+23 -23

src/backfill/mod.rs

··· 9 9 use jacquard::types::did::Did; 10 10 use jacquard::{CowStr, IntoStatic, prelude::*}; 11 11 use jacquard_common::xrpc::XrpcError; 12 12 - use jacquard_repo::MemoryBlockStore; 13 12 use jacquard_repo::mst::Mst; 13 13 + use jacquard_repo::{BlockStore, MemoryBlockStore}; 14 14 use miette::{IntoDiagnostic, Result}; 15 15 use smol_str::{SmolStr, ToSmolStr, format_smolstr}; 16 16 use std::collections::HashMap; ··· 305 305 emit_identity(&state.status); 306 306 307 307 trace!( 308 308 - "fetched {} bytes for {} in {:?}", 308 308 + "fetched {} bytes for {did} in {:?}", 309 309 car_bytes.body.len(), 310 310 - did, 311 310 start.elapsed() 312 311 ); 313 312 ··· 316 315 let parsed = jacquard_repo::car::reader::parse_car_bytes(&car_bytes.body) 317 316 .await 318 317 .into_diagnostic()?; 319 319 - trace!("parsed car for {} in {:?}", did, start.elapsed()); 318 318 + trace!("parsed car for {did} in {:?}", start.elapsed()); 320 319 321 320 let start = Instant::now(); 322 322 - let store = Arc::new(MemoryBlockStore::new()); 323 323 - for (_cid, bytes) in &parsed.blocks { 324 324 - jacquard_repo::BlockStore::put(store.as_ref(), bytes) 325 325 - .await 326 326 - .into_diagnostic()?; 327 327 - } 321 321 + let store = Arc::new(MemoryBlockStore::new_from_blocks(parsed.blocks)); 328 322 trace!( 329 329 - "stored {} blocks in memory for {} in {:?}", 330 330 - parsed.blocks.len(), 331 331 - did, 323 323 + "stored {} blocks in memory for {did} in {:?}", 324 324 + store.len(), 332 325 start.elapsed() 333 326 ); 334 327 335 328 // 4. parse root commit to get mst root 336 336 - let root_bytes = parsed 337 337 - .blocks 329 329 + let root_bytes = store 338 330 .get(&parsed.root) 331 331 + .await 332 332 + .into_diagnostic()? 339 333 .ok_or_else(|| miette::miette!("root block missing from CAR"))?; 340 334 341 341 - let commit = jacquard_repo::commit::Commit::from_cbor(root_bytes).into_diagnostic()?; 342 342 - debug!("backfilling repo at revision {}", commit.rev); 335 335 + let root_commit = 336 336 + jacquard_repo::commit::Commit::from_cbor(&root_bytes).into_diagnostic()?; 337 337 + debug!( 338 338 + "backfilling repo at revision {}, root cid {}", 339 339 + root_commit.rev, root_commit.data 340 340 + ); 343 341 344 342 // 4.5. verify commit signature 345 343 if verify_signatures { 346 344 let pubkey = app_state.resolver.resolve_signing_key(did).await?; 347 347 - commit 345 345 + root_commit 348 346 .verify(&pubkey) 349 347 .map_err(|e| miette::miette!("signature verification failed for {did}: {e}"))?; 350 348 trace!("signature verified for {did}"); ··· 352 350 353 351 // 5. walk mst 354 352 let start = Instant::now(); 355 355 - let mst: Mst<MemoryBlockStore> = Mst::load(store, commit.data, None); 353 353 + let mst: Mst<MemoryBlockStore> = Mst::load(store, root_commit.data, None); 356 354 let leaves = mst.leaves().await.into_diagnostic()?; 357 355 trace!("walked mst for {} in {:?}", did, start.elapsed()); 358 356 ··· 361 359 let (_state, added_records, added_blocks, count) = { 362 360 let app_state = app_state.clone(); 363 361 let did = did.clone(); 364 364 - let rev = commit.rev; 365 365 - let storage = mst.storage().clone(); 362 362 + let rev = root_commit.rev; 366 363 367 364 tokio::task::spawn_blocking(move || { 368 365 let mut count = 0; ··· 370 367 let mut added_blocks = 0; 371 368 let mut collection_counts: HashMap<SmolStr, u64> = HashMap::new(); 372 369 let mut batch = app_state.db.inner.batch(); 370 370 + let store = mst.storage(); 373 371 374 372 // pre-load existing record CIDs for this DID to detect duplicates/updates 375 373 let prefix = keys::record_prefix(&did); ··· 385 383 386 384 for (key, cid) in leaves { 387 385 let val_bytes = tokio::runtime::Handle::current() 388 388 - .block_on(jacquard_repo::BlockStore::get(storage.as_ref(), &cid)) 386 386 + .block_on(store.get(&cid)) 389 387 .into_diagnostic()?; 390 388 391 389 if let Some(val) = val_bytes { ··· 398 396 let (action, is_new) = if let Some(existing_cid) = existing_cids.get(&path) 399 397 { 400 398 if existing_cid == cid.as_str() { 399 399 + debug!("skip {did}/{collection}/{rkey} ({cid})"); 401 400 continue; // skip unchanged record 402 401 } 403 402 ("update", false) 404 403 } else { 405 404 ("create", true) 406 405 }; 406 406 + debug!("{action} {did}/{collection}/{rkey} ({cid})"); 407 407 408 408 let db_key = keys::record_key(&did, &collection, rkey); 409 409 ··· 440 440 // 6. update status to synced 441 441 state.status = RepoStatus::Synced; 442 442 state.rev = Some(rev); 443 443 - state.data = Some(Cid::ipld(parsed.root)); 443 443 + state.data = Some(Cid::ipld(root_commit.data)); 444 444 state.last_updated_at = chrono::Utc::now().timestamp(); 445 445 446 446 batch.insert(

+104

src/bin/mst_dump.rs

··· 1 1 + use std::env; 2 2 + use std::sync::Arc; 3 3 + use std::time::Duration; 4 4 + 5 5 + use hydrant::resolver::Resolver; 6 6 + use jacquard::IntoStatic; // Corrected from jacquard_identity::IntoStatic 7 7 + use jacquard::api::com_atproto::sync::get_repo::GetRepo; 8 8 + use jacquard::prelude::XrpcExt; 9 9 + use jacquard::types::did::Did; 10 10 + use jacquard_common::types::ident::AtIdentifier; 11 11 + use jacquard_repo::MemoryBlockStore; 12 12 + use jacquard_repo::mst::Mst; 13 13 + use miette::{IntoDiagnostic, Result}; 14 14 + use tracing::{Level, info}; 15 15 + use tracing_subscriber::FmtSubscriber; // Restored 16 16 + use url::Url; // Restored 17 17 + 18 18 + #[tokio::main] 19 19 + async fn main() -> Result<()> { 20 20 + // Setup logging 21 21 + let subscriber = FmtSubscriber::builder() 22 22 + .with_max_level(Level::INFO) 23 23 + .finish(); 24 24 + tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); 25 25 + 26 26 + // Parse args 27 27 + let args: Vec<String> = env::args().collect(); 28 28 + if args.len() != 2 { 29 29 + eprintln!("Usage: {} <handle|did>", args[0]); 30 30 + std::process::exit(1); 31 31 + } 32 32 + let identifier_str = &args[1]; 33 33 + 34 34 + // Init resolver 35 35 + let plc_url = Url::parse("https://plc.directory").into_diagnostic()?; 36 36 + let resolver = Resolver::new(plc_url, 100); 37 37 + 38 38 + // Resolve identity 39 39 + info!("Resolving {}...", identifier_str); 40 40 + let identifier = if identifier_str.starts_with("did:") { 41 41 + AtIdentifier::Did(Did::new(identifier_str).map_err(|e| miette::miette!("{}", e))?) 42 42 + } else { 43 43 + AtIdentifier::Handle(identifier_str.parse().into_diagnostic()?) 44 44 + }; 45 45 + 46 46 + let did = match identifier { 47 47 + AtIdentifier::Did(d) => d.into_static(), 48 48 + AtIdentifier::Handle(h) => { 49 49 + let d = resolver.resolve_did(&AtIdentifier::Handle(h)).await?; 50 50 + d 51 51 + } 52 52 + }; 53 53 + info!("Resolved to DID: {}", did); 54 54 + 55 55 + let (pds_url, _) = resolver.resolve_identity_info(&did).await?; 56 56 + info!("PDS URL: {}", pds_url); 57 57 + 58 58 + // Fetch repo 59 59 + info!("Fetching repo..."); 60 60 + let http = reqwest::Client::builder() 61 61 + .timeout(Duration::from_secs(30)) 62 62 + .build() 63 63 + .into_diagnostic()?; 64 64 + 65 65 + let req = GetRepo::new().did(did.clone()).build(); 66 66 + let resp = http.xrpc(pds_url).send(&req).await.into_diagnostic()?; 67 67 + let car_bytes = resp.into_output().map_err(|e| miette::miette!("{}", e))?; // explicit map_err 68 68 + 69 69 + info!("Fetched {} bytes", car_bytes.body.len()); 70 70 + 71 71 + // Parse CAR 72 72 + let parsed = jacquard_repo::car::reader::parse_car_bytes(&car_bytes.body) 73 73 + .await 74 74 + .into_diagnostic()?; 75 75 + 76 76 + let store = Arc::new(MemoryBlockStore::new()); 77 77 + for (_cid, bytes) in &parsed.blocks { 78 78 + jacquard_repo::BlockStore::put(store.as_ref(), bytes) 79 79 + .await 80 80 + .into_diagnostic()?; 81 81 + } 82 82 + 83 83 + // Load MST 84 84 + let root_bytes = parsed 85 85 + .blocks 86 86 + .get(&parsed.root) 87 87 + .ok_or_else(|| miette::miette!("root block missing from CAR"))?; 88 88 + 89 89 + let root_commit = jacquard_repo::commit::Commit::from_cbor(root_bytes).into_diagnostic()?; 90 90 + info!("Repo rev: {}", root_commit.rev); 91 91 + 92 92 + let mst: Mst<MemoryBlockStore> = Mst::load(store.clone(), root_commit.data, None); 93 93 + let leaves = mst.leaves().await.into_diagnostic()?; 94 94 + let root = mst.root().await.into_diagnostic()?; 95 95 + 96 96 + info!("Found {} records", leaves.len()); 97 97 + 98 98 + println!("root -> {}", root); 99 99 + for (key, cid) in leaves { 100 100 + println!("{} -> {}", key, cid); 101 101 + } 102 102 + 103 103 + Ok(()) 104 104 + }

+4 -4

src/ingest/worker.rs

··· 262 262 return Ok(ProcessResult::Ok); 263 263 } 264 264 265 265 - if let (Some(prev_repo), Some(prev_commit)) = (&repo_state.data, &commit.prev_data) 266 266 - && prev_repo != &prev_commit.0 265 265 + if let (Some(repo), Some(prev_commit)) = (&repo_state.data, &commit.prev_data) 266 266 + && repo != &prev_commit.0 267 267 { 268 268 warn!( 269 269 - "gap detected for {}: prev {} != stored {}. triggering backfill", 270 270 - did, prev_repo, prev_commit.0 269 269 + "gap detected for {}: repo {} != commit prev {}. triggering backfill", 270 270 + did, repo, prev_commit.0 271 271 ); 272 272 273 273 let mut batch = state.db.inner.batch();

+10

src/lib.rs

··· 1 1 + pub mod api; 2 2 + pub mod backfill; 3 3 + pub mod config; 4 4 + pub mod crawler; 5 5 + pub mod db; 6 6 + pub mod ingest; 7 7 + pub mod ops; 8 8 + pub mod resolver; 9 9 + pub mod state; 10 10 + pub mod types;

+9 -20

src/main.rs

··· 1 1 - mod api; 2 2 - mod backfill; 3 3 - mod config; 4 4 - mod crawler; 5 5 - mod db; 6 6 - mod ingest; 7 7 - mod ops; 8 8 - mod resolver; 9 9 - mod state; 10 10 - mod types; 11 11 - 12 12 - use crate::config::{Config, SignatureVerification}; 13 13 - use crate::crawler::Crawler; 14 14 - use crate::db::set_firehose_cursor; 15 15 - use crate::ingest::firehose::FirehoseIngestor; 16 16 - use crate::state::AppState; 17 17 - use crate::{backfill::BackfillWorker, ingest::worker::FirehoseWorker}; 1 1 + use hydrant::config::{Config, SignatureVerification}; 2 2 + use hydrant::crawler::Crawler; 3 3 + use hydrant::db::{self, set_firehose_cursor}; 4 4 + use hydrant::ingest::firehose::FirehoseIngestor; 5 5 + use hydrant::state::AppState; 6 6 + use hydrant::{api, backfill::BackfillWorker, ingest::worker::FirehoseWorker}; 18 7 use futures::{FutureExt, TryFutureExt, future::BoxFuture}; 19 8 use miette::IntoDiagnostic; 20 9 use mimalloc::MiMalloc; ··· 81 70 82 71 if let Err(e) = spawn_blocking({ 83 72 let state = state.clone(); 84 84 - move || crate::backfill::manager::queue_pending_backfills(&state) 73 73 + move || hydrant::backfill::manager::queue_pending_backfills(&state) 85 74 }) 86 75 .await 87 76 .into_diagnostic()? ··· 92 81 93 82 if let Err(e) = spawn_blocking({ 94 83 let state = state.clone(); 95 95 - move || crate::backfill::manager::queue_gone_backfills(&state) 84 84 + move || hydrant::backfill::manager::queue_gone_backfills(&state) 96 85 }) 97 86 .await 98 87 .into_diagnostic()? ··· 103 92 104 93 std::thread::spawn({ 105 94 let state = state.clone(); 106 106 - move || crate::backfill::manager::retry_worker(state) 95 95 + move || hydrant::backfill::manager::retry_worker(state) 107 96 }); 108 97 109 98 tokio::spawn({

+8 -7

src/ops.rs

··· 190 190 191 191 trace!("parsed car for {did} in {:?}", start.elapsed()); 192 192 193 193 + let root_bytes = parsed 194 194 + .blocks 195 195 + .get(&parsed.root) 196 196 + .ok_or_else(|| miette::miette!("root block missing from CAR"))?; 197 197 + 198 198 + let repo_commit = jacquard_repo::commit::Commit::from_cbor(root_bytes).into_diagnostic()?; 199 199 + 193 200 if let Some(key) = signing_key { 194 194 - let root_bytes = parsed 195 195 - .blocks 196 196 - .get(&parsed.root) 197 197 - .ok_or_else(|| miette::miette!("root block missing from CAR"))?; 198 198 - 199 199 - let repo_commit = jacquard_repo::commit::Commit::from_cbor(root_bytes).into_diagnostic()?; 200 201 repo_commit 201 202 .verify(key) 202 203 .map_err(|e| miette::miette!("signature verification failed for {did}: {e}"))?; ··· 204 205 } 205 206 206 207 repo_state.rev = Some(commit.rev.clone()); 207 207 - repo_state.data = Some(Cid::ipld(parsed.root)); 208 208 + repo_state.data = Some(Cid::ipld(repo_commit.data)); 208 209 repo_state.last_updated_at = chrono::Utc::now().timestamp(); 209 210 210 211 batch.insert(&db.repos, keys::repo_key(did), ser_repo_state(&repo_state)?);