microcosm.blue
Fast and robust atproto CAR file processing in rust
fix example
bad-example.com
+24
-11
+7
-1
benches/huge-car.rs
+7
-1
benches/huge-car.rs
···
22
});
23
}
24
25
async fn drive_car(filename: impl AsRef<Path>) -> usize {
26
let reader = tokio::fs::File::open(filename).await.unwrap();
27
let reader = tokio::io::BufReader::new(reader);
28
29
let mut driver =
30
-
match Driver::load_car(reader, |block| block.len().to_le_bytes().to_vec(), 1024)
31
.await
32
.unwrap()
33
.unwrap()
···
22
});
23
}
24
25
+
#[inline(always)]
26
+
fn ser(block: Vec<u8>) -> Vec<u8> {
27
+
let s = block.len();
28
+
usize::to_ne_bytes(s).to_vec()
29
+
}
30
+
31
async fn drive_car(filename: impl AsRef<Path>) -> usize {
32
let reader = tokio::fs::File::open(filename).await.unwrap();
33
let reader = tokio::io::BufReader::new(reader);
34
35
let mut driver =
36
+
match Driver::load_car(reader, ser, 1024)
37
.await
38
.unwrap()
39
.unwrap()
+7
-1
benches/non-huge-cars.rs
+7
-1
benches/non-huge-cars.rs
+4
-3
examples/read-file/main.rs
+4
-3
examples/read-file/main.rs
···
24
let reader = tokio::io::BufReader::new(reader);
25
26
let (commit, mut driver) = match DriverBuilder::new()
27
-
.with_block_processor(|block| block.len().to_ne_bytes().to_vec().into())
28
.load_car(reader)
29
.await?
30
{
31
-
Driver::Memory(commit, mem_driver) => (commit, mem_driver),
32
-
Driver::Disk(_) => panic!("this example doesn't handle big CARs"),
33
};
34
35
log::info!("got commit: {commit:?}");
···
24
let reader = tokio::io::BufReader::new(reader);
25
26
let (commit, mut driver) = match DriverBuilder::new()
27
+
.with_block_processor(|block| block.len().to_ne_bytes().to_vec())
28
.load_car(reader)
29
.await?
30
{
31
+
None => todo!(),
32
+
Some(Driver::Memory(commit, mem_driver)) => (commit, mem_driver),
33
+
Some(Driver::Disk(_)) => panic!("this example doesn't handle big CARs"),
34
};
35
36
log::info!("got commit: {commit:?}");
+6
-6
readme.md
+6
-6
readme.md
···
86
static GLOBAL: MiMalloc = MiMalloc;
87
```
88
89
-
- 450MiB CAR file: `1.1s` (-15%)
90
-
- 128MiB: `310ms` (-13%)
91
-
- 5.0MiB: `6.1ms` (-10%)
92
-
- 279KiB: `160us` (-5%)
93
-
- 3.4KiB: `5.7us` (-9%)
94
-
- empty: `660ns` (-7%)
95
96
processing CARs requires buffering blocks, so it can consume a lot of memory. repo-stream's in-memory driver has minimal memory overhead, but there are two ways to make it work with less mem (you can do either or both!)
97
···
86
static GLOBAL: MiMalloc = MiMalloc;
87
```
88
89
+
- 450MiB CAR file: `1.2s` (-8%)
90
+
- 128MiB: `300ms` (-14%)
91
+
- 5.0MiB: `6.0ms` (-12%)
92
+
- 279KiB: `140us` (-21%)
93
+
- 3.4KiB: `4.7us` (-10%)
94
+
- empty: `640ns` (-4%)
95
96
processing CARs requires buffering blocks, so it can consume a lot of memory. repo-stream's in-memory driver has minimal memory overhead, but there are two ways to make it work with less mem (you can do either or both!)
97