-1

Cargo.lock

··· 2372 2372 "ed25519-dalek", 2373 2373 "futures", 2374 2374 "futures-lite", 2375 2375 - "genawaiter", 2376 2375 "getrandom 0.2.16", 2377 2376 "getrandom 0.3.4", 2378 2377 "http",

+2

Cargo.toml

··· 48 48 bytes = "1.10" 49 49 smol_str = { version = "0.3", features = ["serde"] } 50 50 url = "2.5" 51 51 + cid = { version = "0.11.1", features = ["serde", "std"] } 52 52 + ipld-core = { version = "0.4.2", features = ["serde"] } 51 53 52 54 # Proc macros 53 55 proc-macro2 = "1.0"

+2 -3

crates/jacquard-common/Cargo.toml

··· 30 30 base64.workspace = true 31 31 bytes.workspace = true 32 32 chrono.workspace = true 33 33 - cid = { version = "0.11.1", features = ["serde", "std"] } 34 34 - ipld-core = { version = "0.4.2", features = ["serde"] } 33 33 + cid.workspace = true 34 34 + ipld-core.workspace = true 35 35 langtag = { version = "0.4.0", features = ["serde"] } 36 36 miette.workspace = true 37 37 multibase = "0.9.1" ··· 58 58 futures = { version = "0.3", optional = true } 59 59 tokio-tungstenite-wasm = { version = "0.4", features = ["rustls-tls-native-roots"], optional = true } 60 60 ciborium = {version = "0.2.0", optional = true } 61 61 - genawaiter = { version = "0.99.1", features = ["futures03"] } 62 61 zstd = { version = "0.13", optional = true } 63 62 64 63 [target.'cfg(target_family = "wasm")'.dependencies]

+4 -3

crates/jacquard-repo/Cargo.toml

··· 24 24 serde_ipld_dagcbor.workspace = true 25 25 serde_bytes = "0.11" 26 26 27 27 - # IPLD primitives (match jacquard-common versions) 28 28 - cid = { version = "0.11.1", features = ["serde", "std"] } 29 29 - ipld-core = { version = "0.4.2", features = ["serde"] } 27 27 + # IPLD primitives 28 28 + cid.workspace = true 29 29 + ipld-core.workspace = true 30 30 multihash = "0.19.3" 31 31 32 32 # CAR file I/O ··· 47 47 trait-variant.workspace = true 48 48 n0-future.workspace = true 49 49 tokio = { workspace = true, default-features = false, features = ["fs", "io-util"] } 50 50 + 50 51 51 52 # Crypto (for commit signing/verification) 52 53 ed25519-dalek = { version = "2", features = ["rand_core"] }

+4

crates/jacquard-repo/src/mst/tree.rs

··· 1153 1153 /// - All leaf record blocks (read from storage) 1154 1154 /// 1155 1155 /// This is suitable for CAR export and avoids loading all blocks into memory. 1156 1156 + /// 1157 1157 + /// TODO: get rid of tokio dependency here if possible 1156 1158 pub async fn write_blocks_to_car<W: tokio::io::AsyncWrite + Send + Unpin>( 1157 1159 &self, 1158 1160 writer: &mut iroh_car::CarWriter<W>, ··· 1177 1179 } 1178 1180 1179 1181 /// Recursively write MST nodes to CAR and collect leaf CIDs 1182 1182 + /// 1183 1183 + /// TODO: get rid of tokio dependency here if possible 1180 1184 fn write_mst_nodes_to_car<'a, W: tokio::io::AsyncWrite + Send + Unpin>( 1181 1185 &'a self, 1182 1186 writer: &'a mut iroh_car::CarWriter<W>,

-325

crates/jacquard-repo/tests/interop.rs

··· 204 204 } 205 205 206 206 #[tokio::test] 207 207 - async fn test_minimal_determinism() { 208 208 - // Minimal test with just a few keys 209 209 - let keys = vec!["A0/501344", "A1/700567", "B0/436099"]; 210 210 - 211 211 - fn test_cid(n: u8) -> cid::Cid { 212 212 - let data = vec![n; 32]; 213 213 - let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); 214 214 - cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) 215 215 - } 216 216 - 217 217 - // Build tree in forward order 218 218 - let storage1 = Arc::new(MemoryBlockStore::new()); 219 219 - let mut mst1 = Mst::new(storage1); 220 220 - for (i, &key) in keys.iter().enumerate() { 221 221 - println!("MST1: Adding {}", key); 222 222 - mst1 = mst1.add(key, test_cid(i as u8)).await.unwrap(); 223 223 - } 224 224 - 225 225 - // Build tree in reverse order 226 226 - let storage2 = Arc::new(MemoryBlockStore::new()); 227 227 - let mut mst2 = Mst::new(storage2); 228 228 - for (i, &key) in keys.iter().rev().enumerate() { 229 229 - let idx = keys.len() - 1 - i; 230 230 - println!("MST2: Adding {}", key); 231 231 - mst2 = mst2.add(key, test_cid(idx as u8)).await.unwrap(); 232 232 - } 233 233 - 234 234 - // Check if all keys exist in both trees 235 235 - for key in keys.iter() { 236 236 - let v1 = mst1.get(key).await.unwrap(); 237 237 - let v2 = mst2.get(key).await.unwrap(); 238 238 - println!( 239 239 - "Key {}: mst1={:?}, mst2={:?}", 240 240 - key, 241 241 - v1.is_some(), 242 242 - v2.is_some() 243 243 - ); 244 244 - assert_eq!(v1.is_some(), v2.is_some(), "Key {} mismatch", key); 245 245 - } 246 246 - 247 247 - // Root CIDs should match 248 248 - println!("mst1 root: {:?}", mst1.root().await.unwrap()); 249 249 - println!("mst2 root: {:?}", mst2.root().await.unwrap()); 250 250 - 251 251 - // Trees should be identical 252 252 - assert_eq!( 253 253 - mst1.root().await.unwrap(), 254 254 - mst2.root().await.unwrap(), 255 255 - "Tree structure should be deterministic" 256 256 - ); 257 257 - } 258 258 - 259 259 - #[tokio::test] 260 260 - async fn test_first_10_keys_determinism() { 261 261 - // Test first 10 keys from example_keys.txt 262 262 - let keys_txt = include_str!("fixtures/example_keys.txt"); 263 263 - let keys: Vec<&str> = keys_txt 264 264 - .lines() 265 265 - .filter(|s| !s.is_empty()) 266 266 - .take(10) 267 267 - .collect(); 268 268 - 269 269 - fn test_cid(n: u8) -> cid::Cid { 270 270 - let data = vec![n; 32]; 271 271 - let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); 272 272 - cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) 273 273 - } 274 274 - 275 275 - let storage1 = Arc::new(MemoryBlockStore::new()); 276 276 - let mut mst1 = Mst::new(storage1); 277 277 - for (i, &key) in keys.iter().enumerate() { 278 278 - mst1 = mst1.add(key, test_cid(i as u8)).await.unwrap(); 279 279 - } 280 280 - 281 281 - let storage2 = Arc::new(MemoryBlockStore::new()); 282 282 - let mut mst2 = Mst::new(storage2); 283 283 - for (i, &key) in keys.iter().rev().enumerate() { 284 284 - let idx = keys.len() - 1 - i; 285 285 - mst2 = mst2.add(key, test_cid(idx as u8)).await.unwrap(); 286 286 - } 287 287 - 288 288 - // Check all keys present 289 289 - for &key in &keys { 290 290 - assert!(mst1.get(key).await.unwrap().is_some()); 291 291 - assert!(mst2.get(key).await.unwrap().is_some()); 292 292 - } 293 293 - 294 294 - eprintln!("mst1 root: {:?}", mst1.root().await.unwrap()); 295 295 - eprintln!("mst2 root: {:?}", mst2.root().await.unwrap()); 296 296 - 297 297 - assert_eq!( 298 298 - mst1.root().await.unwrap(), 299 299 - mst2.root().await.unwrap(), 300 300 - "Tree structure should be deterministic" 301 301 - ); 302 302 - } 303 303 - 304 304 - #[tokio::test] 305 305 - async fn test_minimal_corruption_case() { 306 306 - // Minimal reproduction of the corruption bug 307 307 - let storage = Arc::new(MemoryBlockStore::new()); 308 308 - let mut mst = Mst::new(storage); 309 309 - 310 310 - fn test_cid(n: u8) -> cid::Cid { 311 311 - let data = vec![n; 32]; 312 312 - let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); 313 313 - cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) 314 314 - } 315 315 - 316 316 - // Add N0 (layer 0) first 317 317 - println!("Adding N0/719700 (layer {})", layer_for_key("N0/719700")); 318 318 - mst = mst.add("N0/719700", test_cid(0)).await.unwrap(); 319 319 - 320 320 - // Verify N0 is retrievable 321 321 - assert!( 322 322 - mst.get("N0/719700").await.unwrap().is_some(), 323 323 - "N0 should exist after adding it" 324 324 - ); 325 325 - 326 326 - // Add M5 (layer 5) 327 327 - println!("Adding M5/340624 (layer {})", layer_for_key("M5/340624")); 328 328 - mst = mst.add("M5/340624", test_cid(1)).await.unwrap(); 329 329 - 330 330 - // Verify both are retrievable 331 331 - assert!( 332 332 - mst.get("N0/719700").await.unwrap().is_some(), 333 333 - "N0 should still exist after adding M5" 334 334 - ); 335 335 - assert!( 336 336 - mst.get("M5/340624").await.unwrap().is_some(), 337 337 - "M5 should exist after adding it" 338 338 - ); 339 339 - } 340 340 - 341 341 - #[tokio::test] 342 207 async fn test_generated_keys_at_specific_layers() { 343 208 // Generate keys at different layers and verify they work correctly 344 209 let storage = Arc::new(MemoryBlockStore::new()); ··· 371 236 } 372 237 } 373 238 374 374 - #[tokio::test] 375 375 - async fn test_first_n_keys_determinism() { 376 376 - // Test varying numbers of keys to find breaking point 377 377 - let all_keys = vec![ 378 378 - "A0/501344", 379 379 - "A1/700567", 380 380 - "A2/239654", 381 381 - "A3/570745", 382 382 - "A4/231700", 383 383 - "A5/343219", 384 384 - "B0/436099", 385 385 - "B1/293486", 386 386 - "B2/303249", 387 387 - "B3/690557", 388 388 - ]; 389 389 - 390 390 - fn test_cid(n: u8) -> cid::Cid { 391 391 - let data = vec![n; 32]; 392 392 - let mh = multihash::Multihash::wrap(SHA2_256, &data).unwrap(); 393 393 - cid::Cid::new_v1(DAG_CBOR_CID_CODEC, mh) 394 394 - } 395 395 - 396 396 - for n in 3..=10 { 397 397 - let keys: Vec<&str> = all_keys.iter().take(n).copied().collect(); 398 398 - 399 399 - let storage1 = Arc::new(MemoryBlockStore::new()); 400 400 - let mut mst1 = Mst::new(storage1); 401 401 - for (i, &key) in keys.iter().enumerate() { 402 402 - mst1 = mst1.add(key, test_cid(i as u8)).await.unwrap(); 403 403 - } 404 404 - 405 405 - let storage2 = Arc::new(MemoryBlockStore::new()); 406 406 - let mut mst2 = Mst::new(storage2); 407 407 - for (i, &key) in keys.iter().rev().enumerate() { 408 408 - let idx = keys.len() - 1 - i; 409 409 - mst2 = mst2.add(key, test_cid(idx as u8)).await.unwrap(); 410 410 - } 411 411 - 412 412 - let match_result = mst1.root().await.unwrap() == mst2.root().await.unwrap(); 413 413 - eprintln!( 414 414 - "{} keys - Match: {} (mst1: {:?}, mst2: {:?})", 415 415 - n, 416 416 - match_result, 417 417 - mst1.root().await.unwrap(), 418 418 - mst2.root().await.unwrap() 419 419 - ); 420 420 - 421 421 - if !match_result { 422 422 - panic!("Determinism breaks at {} keys!", n); 423 423 - } 424 424 - } 425 425 - } 426 426 - 427 427 - // ============================================================================ 428 428 - // Commit Proof Fixture Tests (Phase 2.5) 429 429 - // ============================================================================ 430 430 - 431 239 #[derive(Debug, Deserialize)] 432 240 struct CommitProofFixture { 433 241 comment: String, ··· 725 533 } 726 534 727 535 #[tokio::test] 728 728 - async fn test_inspect_single_key_serialization() { 729 729 - // Inspect what we're actually serializing for a single key 730 730 - use jacquard_repo::mst::util::layer_for_key; 731 731 - 732 732 - let key = "com.example.record/3jqfcqzm3ft2j"; 733 733 - let cid1: cid::Cid = "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" 734 734 - .parse() 735 735 - .unwrap(); 736 736 - 737 737 - println!("Key: {}", key); 738 738 - println!("Layer: {}", layer_for_key(key)); 739 739 - println!("Value CID: {}", cid1); 740 740 - 741 741 - let storage = Arc::new(MemoryBlockStore::new()); 742 742 - let mut mst = Mst::new(storage.clone()); 743 743 - 744 744 - mst = mst.add(key, cid1).await.unwrap(); 745 745 - 746 746 - // Persist to storage so we can inspect serialized bytes 747 747 - let root_cid = mst.persist().await.unwrap(); 748 748 - 749 749 - println!("\nRoot CID: {}", root_cid); 750 750 - println!( 751 751 - "Expected: bafyreicphm6sin567zmcmw2yrbguhsdqwzkxs62rcayyk6ylivxfguazgi (from test output)" 752 752 - ); 753 753 - 754 754 - // Fetch the actual serialized bytes from storage 755 755 - let node_bytes = storage.get(&root_cid).await.unwrap().unwrap(); 756 756 - println!("\nSerialized node ({} bytes):", node_bytes.len()); 757 757 - println!("Hex: {}", hex::encode(&node_bytes)); 758 758 - 759 759 - // Deserialize to see structure 760 760 - use jacquard_repo::mst::node::NodeData; 761 761 - let node: NodeData = serde_ipld_dagcbor::from_slice(&node_bytes).unwrap(); 762 762 - println!("\nNodeData:"); 763 763 - println!(" left: {:?}", node.left); 764 764 - println!(" entries: {} entries", node.entries.len()); 765 765 - for (i, entry) in node.entries.iter().enumerate() { 766 766 - println!( 767 767 - " [{}] prefix_len={}, key_suffix={:?}, value={}, tree={:?}", 768 768 - i, 769 769 - entry.prefix_len, 770 770 - String::from_utf8_lossy(&entry.key_suffix), 771 771 - entry.value, 772 772 - entry.tree 773 773 - ); 774 774 - } 775 775 - } 776 776 - 777 777 - #[tokio::test] 778 778 - async fn test_inspect_two_key_serialization() { 779 779 - // Inspect 2-key tree structure 780 780 - use jacquard_repo::mst::util::layer_for_key; 781 781 - 782 782 - let key1 = "com.example.record/3jqfcqzm3ft2j"; // A 783 783 - let key2 = "com.example.record/3jqfcqzm3fz2j"; // C 784 784 - let cid1: cid::Cid = "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454" 785 785 - .parse() 786 786 - .unwrap(); 787 787 - 788 788 - println!("Key 1 (A): {} (layer {})", key1, layer_for_key(key1)); 789 789 - println!("Key 2 (C): {} (layer {})", key2, layer_for_key(key2)); 790 790 - 791 791 - let storage = Arc::new(MemoryBlockStore::new()); 792 792 - let mut mst = Mst::new(storage.clone()); 793 793 - 794 794 - mst = mst.add(key1, cid1).await.unwrap(); 795 795 - mst = mst.add(key2, cid1).await.unwrap(); 796 796 - 797 797 - // Persist to storage so we can inspect serialized bytes 798 798 - let root_cid = mst.persist().await.unwrap(); 799 799 - 800 800 - println!("\nRoot CID: {}", root_cid); 801 801 - println!("Expected: bafyreidfcktqnfmykz2ps3dbul35pepleq7kvv526g47xahuz3rqtptmky"); 802 802 - 803 803 - // Fetch and inspect 804 804 - let node_bytes = storage.get(&root_cid).await.unwrap().unwrap(); 805 805 - println!("\nSerialized node ({} bytes):", node_bytes.len()); 806 806 - println!("Hex: {}", hex::encode(&node_bytes)); 807 807 - 808 808 - use jacquard_repo::mst::node::NodeData; 809 809 - let node: NodeData = serde_ipld_dagcbor::from_slice(&node_bytes).unwrap(); 810 810 - println!("\nNodeData:"); 811 811 - println!(" left: {:?}", node.left); 812 812 - println!(" entries: {} entries", node.entries.len()); 813 813 - for (i, entry) in node.entries.iter().enumerate() { 814 814 - println!( 815 815 - " [{}] prefix_len={}, key_suffix={:?}, value={}, tree={:?}", 816 816 - i, 817 817 - entry.prefix_len, 818 818 - String::from_utf8_lossy(&entry.key_suffix), 819 819 - entry.value, 820 820 - entry.tree 821 821 - ); 822 822 - } 823 823 - 824 824 - // Calculate what prefix compression SHOULD be 825 825 - let prefix_len = jacquard_repo::mst::util::common_prefix_len(key1, key2); 826 826 - println!("\nCommon prefix length between keys: {}", prefix_len); 827 827 - println!("Common prefix: {:?}", &key1[..prefix_len]); 828 828 - println!("Key1 suffix: {:?}", &key1[prefix_len..]); 829 829 - println!("Key2 suffix: {:?}", &key2[prefix_len..]); 830 830 - } 831 831 - 832 832 - #[tokio::test] 833 536 async fn test_real_repo_car_roundtrip() { 834 537 use jacquard_repo::car::{read_car, write_car}; 835 538 use std::path::Path; ··· 893 596 } 894 597 895 598 println!("✓ All {} blocks match after roundtrip", blocks.len()); 896 896 - } 897 897 - 898 898 - #[tokio::test] 899 899 - async fn test_real_repo_car_header() { 900 900 - use jacquard_repo::car::read_car_header; 901 901 - use std::path::Path; 902 902 - 903 903 - let fixture_path = Path::new(concat!( 904 904 - env!("CARGO_MANIFEST_DIR"), 905 905 - "/tests/fixtures/repo-nonbinary.computer-2025-10-21T13_05_55.090Z.car" 906 906 - )); 907 907 - 908 908 - if !fixture_path.exists() { 909 909 - eprintln!("⚠️ Skipping test_real_repo_car_header - fixture not present"); 910 910 - return; 911 911 - } 912 912 - 913 913 - let roots = read_car_header(fixture_path) 914 914 - .await 915 915 - .expect("Failed to read CAR header"); 916 916 - 917 917 - println!("✓ CAR file has {} root(s)", roots.len()); 918 918 - 919 919 - assert!(!roots.is_empty(), "CAR should have at least one root"); 920 920 - 921 921 - for (i, root) in roots.iter().enumerate() { 922 922 - println!(" Root {}: {}", i, root); 923 923 - } 924 599 } 925 600 926 601 #[tokio::test]