+15 -16

Cargo.lock

··· 1220 1220 checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" 1221 1221 dependencies = [ 1222 1222 "data-encoding", 1223 1223 - "syn 2.0.103", 1223 1223 + "syn 1.0.109", 1224 1224 ] 1225 1225 1226 1226 [[package]] ··· 1569 1569 checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" 1570 1570 dependencies = [ 1571 1571 "libc", 1572 1572 - "windows-sys 0.59.0", 1572 1572 + "windows-sys 0.52.0", 1573 1573 ] 1574 1574 1575 1575 [[package]] ··· 1611 1611 1612 1612 [[package]] 1613 1613 name = "fjall" 1614 1614 - version = "2.8.0" 1615 1615 - source = "registry+https://github.com/rust-lang/crates.io-index" 1616 1616 - checksum = "26b2ced3483989a62b3533c9f99054d73b527c6c0045cf22b00fe87956f1a46f" 1614 1614 + version = "2.11.2" 1615 1615 + source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d" 1617 1616 dependencies = [ 1618 1617 "byteorder", 1619 1618 "byteview", ··· 2595 2594 dependencies = [ 2596 2595 "hermit-abi", 2597 2596 "libc", 2598 2598 - "windows-sys 0.59.0", 2597 2597 + "windows-sys 0.52.0", 2599 2598 ] 2600 2599 2601 2600 [[package]] ··· 2800 2799 checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" 2801 2800 dependencies = [ 2802 2801 "cfg-if", 2803 2803 - "windows-targets 0.52.6", 2802 2802 + "windows-targets 0.48.5", 2804 2803 ] 2805 2804 2806 2805 [[package]] ··· 2931 2930 2932 2931 [[package]] 2933 2932 name = "lsm-tree" 2934 2934 - version = "2.8.0" 2933 2933 + version = "2.10.2" 2935 2934 source = "registry+https://github.com/rust-lang/crates.io-index" 2936 2936 - checksum = "d0a63a5e98a38b51765274137d8aedfbd848da5f4d016867e186b673fcc06a8c" 2935 2935 + checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c" 2937 2936 dependencies = [ 2938 2937 "byteorder", 2939 2938 "crossbeam-skiplist", ··· 4054 4053 "once_cell", 4055 4054 "socket2 0.5.9", 4056 4055 "tracing", 4057 4057 - "windows-sys 0.59.0", 4056 4056 + "windows-sys 0.52.0", 4058 4057 ] 4059 4058 4060 4059 [[package]] ··· 4436 4435 "errno", 4437 4436 "libc", 4438 4437 "linux-raw-sys 0.4.15", 4439 4439 - "windows-sys 0.59.0", 4438 4438 + "windows-sys 0.52.0", 4440 4439 ] 4441 4440 4442 4441 [[package]] ··· 4449 4448 "errno", 4450 4449 "libc", 4451 4450 "linux-raw-sys 0.9.4", 4452 4452 - "windows-sys 0.59.0", 4451 4451 + "windows-sys 0.52.0", 4453 4452 ] 4454 4453 4455 4454 [[package]] ··· 5188 5187 "getrandom 0.3.3", 5189 5188 "once_cell", 5190 5189 "rustix 1.0.5", 5191 5191 - "windows-sys 0.59.0", 5190 5190 + "windows-sys 0.52.0", 5192 5191 ] 5193 5192 5194 5193 [[package]] ··· 5821 5820 5822 5821 [[package]] 5823 5822 name = "value-log" 5824 5824 - version = "1.8.0" 5823 5823 + version = "1.9.0" 5825 5824 source = "registry+https://github.com/rust-lang/crates.io-index" 5826 5826 - checksum = "fd29b17c041f94e0885179637289815cd038f0c9fc19c4549d5a97017404fb7d" 5825 5825 + checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" 5827 5826 dependencies = [ 5828 5827 "byteorder", 5829 5828 "byteview", ··· 6073 6072 source = "registry+https://github.com/rust-lang/crates.io-index" 6074 6073 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" 6075 6074 dependencies = [ 6076 6076 - "windows-sys 0.59.0", 6075 6075 + "windows-sys 0.48.0", 6077 6076 ] 6078 6077 6079 6078 [[package]]

+1 -1

ufos/Cargo.toml

··· 13 13 clap = { version = "4.5.31", features = ["derive"] } 14 14 dropshot = "0.16.0" 15 15 env_logger = "0.11.7" 16 16 - fjall = { version = "2.8.0", features = ["lz4"] } 16 16 + fjall = { git = "https://github.com/fjall-rs/fjall.git", features = ["lz4"] } 17 17 getrandom = "0.3.3" 18 18 http = "1.3.1" 19 19 jetstream = { path = "../jetstream", features = ["metrics"] }

+10 -3

ufos/src/storage_fjall.rs

··· 1222 1222 AllTimeRecordsKey::new(new_creates_count.into(), &nsid).to_db_bytes()?, 1223 1223 ), 1224 1224 }; 1225 1225 - batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better 1225 1225 + // remove_weak is allowed here because the secondary ranking index only ever inserts once at a key 1226 1226 + batch.remove_weak(&self.rollups, &old_k); 1226 1227 batch.insert(&self.rollups, &new_k, ""); 1227 1228 } 1228 1229 ··· 1246 1247 AllTimeDidsKey::new(new_dids_estimate.into(), &nsid).to_db_bytes()?, 1247 1248 ), 1248 1249 }; 1249 1249 - batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better 1250 1250 + // remove_weak is allowed here because the secondary ranking index only ever inserts once at a key 1251 1251 + batch.remove_weak(&self.rollups, &old_k); 1250 1252 batch.insert(&self.rollups, &new_k, ""); 1251 1253 } 1252 1254 ··· 1619 1621 histogram!("storage_trim_dirty_nsids").record(completed.len() as f64); 1620 1622 histogram!("storage_trim_duration").record(dt.as_micros() as f64); 1621 1623 counter!("storage_trim_removed", "dangling" => "true").increment(total_danglers as u64); 1622 1622 - counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64); 1624 1624 + if total_deleted >= total_danglers { 1625 1625 + counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64); 1626 1626 + } else { 1627 1627 + // TODO: probably think through what's happening here 1628 1628 + log::warn!("weird trim case: more danglers than deleted? metric will be missing for dangling=false. deleted={total_deleted} danglers={total_danglers}"); 1629 1629 + } 1623 1630 for c in completed { 1624 1631 dirty_nsids.remove(&c); 1625 1632 }