+3 -11

src/api/repo.rs

··· 1 1 use crate::api::AppState; 2 2 use crate::db::{Db, keys, ser_repo_state}; 3 3 - use crate::ops::send_backfill_req; 4 3 use crate::types::RepoState; 5 4 use axum::{Json, Router, extract::State, http::StatusCode, routing::post}; 6 5 use jacquard::types::did::Did; ··· 25 24 let db = &state.db; 26 25 let mut batch = db.inner.batch(); 27 26 let mut added = 0; 28 28 - let mut to_backfill = Vec::new(); 29 27 30 28 for did_str in req.dids { 31 29 let did = Did::new_owned(did_str.as_str()) ··· 43 41 batch.insert(&db.pending, &did_key, Vec::new()); 44 42 45 43 added += 1; 46 46 - 47 47 - let jacquard_did = Did::new_owned(did.as_str()) 48 48 - .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; 49 49 - to_backfill.push(jacquard_did); 50 44 } 51 45 } 52 46 ··· 55 49 .await 56 50 .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? 57 51 .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?; 52 52 + 58 53 state.db.update_count_async("repos", added).await; 59 54 state.db.update_count_async("pending", added).await; 60 55 61 61 - // trigger backfill 62 62 - for did in to_backfill { 63 63 - send_backfill_req(&state, did) 64 64 - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; 65 65 - } 56 56 + // trigger backfill worker 57 57 + state.notify_backfill(); 66 58 } 67 59 Ok(StatusCode::OK) 68 60 }

+10 -40

src/backfill/manager.rs

··· 7 7 use std::time::Duration; 8 8 use tracing::{debug, error, info}; 9 9 10 10 - pub fn queue_pending_backfills(state: &AppState) -> Result<()> { 11 11 - info!("scanning for pending backfills..."); 12 12 - let mut count = 0; 13 13 - 14 14 - for guard in state.db.pending.iter() { 15 15 - let key = guard.key().into_diagnostic()?; 16 16 - let did = match TrimmedDid::try_from(key.as_ref()) { 17 17 - Ok(did) => did.to_did(), 18 18 - Err(e) => { 19 19 - error!("invalid did in db, skipping: {e}"); 20 20 - continue; 21 21 - } 22 22 - }; 23 23 - 24 24 - debug!("queuing did {did}"); 25 25 - if let Err(e) = state.backfill_tx.send(did.clone()) { 26 26 - error!("failed to queue pending backfill for did:{did}: {e}"); 27 27 - } else { 28 28 - count += 1; 29 29 - } 30 30 - } 31 31 - 32 32 - info!("queued {count} pending backfills"); 33 33 - Ok(()) 34 34 - } 35 35 - 36 10 pub fn queue_gone_backfills(state: &Arc<AppState>) -> Result<()> { 37 37 - info!("scanning for deactivated/takendown repos to retry..."); 11 11 + debug!("scanning for deactivated/takendown repos to retry..."); 38 12 let mut count = 0; 39 13 40 14 for guard in state.db.resync.iter() { ··· 49 23 50 24 if let Ok(resync_state) = rmp_serde::from_slice::<ResyncState>(&val) { 51 25 if matches!(resync_state, ResyncState::Gone { .. }) { 52 52 - info!("queuing retry for gone repo: {did}"); 26 26 + debug!("queuing retry for gone repo: {did}"); 53 27 54 28 // move back to pending 55 29 let mut batch = state.db.inner.batch(); ··· 64 38 batch.insert(&state.db.repos, &repo_key, ser_repo_state(&repo_state)?); 65 39 } 66 40 41 41 + state.db.update_count("resync", -1); 42 42 + state.db.update_count("pending", 1); 67 43 batch.commit().into_diagnostic()?; 68 44 69 69 - if let Err(e) = state.backfill_tx.send(did.clone()) { 70 70 - error!("failed to queue retry for {did}: {e}"); 71 71 - } else { 72 72 - count += 1; 73 73 - } 45 45 + state.notify_backfill(); 46 46 + count += 1; 74 47 } 75 48 } 76 49 } ··· 110 83 rmp_serde::from_slice::<ResyncState>(&value) 111 84 { 112 85 if next_retry <= now { 113 113 - info!("retrying backfill for {did}"); 86 86 + debug!("retrying backfill for {did}"); 114 87 115 88 // move back to pending 89 89 + state.db.update_count("pending", 1); 116 90 if let Err(e) = db.pending.insert(key, Vec::new()) { 117 91 error!("failed to move {did} to pending: {e}"); 118 92 db::check_poisoned(&e); 119 93 continue; 120 94 } 121 95 122 122 - // queue 123 123 - if let Err(e) = state.backfill_tx.send(did.clone()) { 124 124 - error!("failed to queue retry for {did}: {e}"); 125 125 - } else { 126 126 - count += 1; 127 127 - } 96 96 + state.notify_backfill(); 97 97 + count += 1; 128 98 } 129 99 } 130 100 }

+96 -30

src/backfill/mod.rs

··· 1 1 use crate::db::types::{DbAction, DbRkey, DbTid, TrimmedDid}; 2 2 use crate::db::{self, Db, keys, ser_repo_state}; 3 3 use crate::ops; 4 4 - use crate::state::{AppState, BackfillRx}; 4 4 + use crate::state::AppState; 5 5 use crate::types::{AccountEvt, BroadcastEvent, RepoState, RepoStatus, ResyncState, StoredEvent}; 6 6 - use futures::TryFutureExt; 6 6 + 7 7 use jacquard::api::com_atproto::sync::get_repo::{GetRepo, GetRepoError}; 8 8 use jacquard::error::{ClientError, ClientErrorKind}; 9 9 use jacquard::types::cid::Cid; ··· 24 24 25 25 pub mod manager; 26 26 27 27 + use crate::ingest::{BufferTx, IngestMessage}; 28 28 + 27 29 pub struct BackfillWorker { 28 30 state: Arc<AppState>, 29 29 - rx: BackfillRx, 31 31 + buffer_tx: BufferTx, 30 32 http: reqwest::Client, 31 33 semaphore: Arc<Semaphore>, 32 34 verify_signatures: bool, 35 35 + in_flight: Arc<scc::HashSet<Did<'static>>>, 33 36 } 34 37 35 38 impl BackfillWorker { 36 39 pub fn new( 37 40 state: Arc<AppState>, 38 38 - rx: BackfillRx, 41 41 + buffer_tx: BufferTx, 39 42 timeout: Duration, 40 43 concurrency_limit: usize, 41 44 verify_signatures: bool, 42 45 ) -> Self { 43 46 Self { 44 47 state, 45 45 - rx, 48 48 + buffer_tx, 46 49 http: reqwest::Client::builder() 47 50 .timeout(timeout) 48 51 .zstd(true) ··· 52 55 .expect("failed to build http client"), 53 56 semaphore: Arc::new(Semaphore::new(concurrency_limit)), 54 57 verify_signatures, 58 58 + in_flight: Arc::new(scc::HashSet::new()), 55 59 } 56 60 } 61 61 + } 57 62 58 58 - pub async fn run(mut self) { 63 63 + struct InFlightGuard { 64 64 + did: Did<'static>, 65 65 + set: Arc<scc::HashSet<Did<'static>>>, 66 66 + } 67 67 + 68 68 + impl Drop for InFlightGuard { 69 69 + fn drop(&mut self) { 70 70 + let _ = self.set.remove_sync(&self.did); 71 71 + } 72 72 + } 73 73 + 74 74 + impl BackfillWorker { 75 75 + pub async fn run(self) { 59 76 info!("backfill worker started"); 60 60 - while let Some(did) = self.rx.recv().await { 61 61 - let permit = self 62 62 - .semaphore 63 63 - .clone() 64 64 - .acquire_owned() 65 65 - .await 66 66 - .expect("semaphore closed"); 77 77 + loop { 78 78 + let mut spawned = 0; 79 79 + 80 80 + for guard in self.state.db.pending.iter() { 81 81 + let key = match guard.key() { 82 82 + Ok(k) => k, 83 83 + Err(e) => { 84 84 + error!("failed to read pending key: {e}"); 85 85 + db::check_poisoned(&e); 86 86 + continue; 87 87 + } 88 88 + }; 67 89 68 68 - tokio::spawn( 69 69 - Self::process_did_wrapper( 70 70 - self.state.clone(), 71 71 - self.http.clone(), 72 72 - did.clone(), 73 73 - permit, 74 74 - self.verify_signatures, 75 75 - ) 76 76 - .inspect_err(move |e| { 77 77 - error!("backfill process failed for {did}: {e}"); 78 78 - db::check_poisoned_report(e); 79 79 - }), 80 80 - ); 90 90 + let did = match TrimmedDid::try_from(key.as_ref()) { 91 91 + Ok(d) => d.to_did(), 92 92 + Err(e) => { 93 93 + error!("invalid did in pending: {e}"); 94 94 + continue; 95 95 + } 96 96 + }; 97 97 + 98 98 + if self.in_flight.contains_sync(&did) { 99 99 + continue; 100 100 + } 101 101 + let _ = self.in_flight.insert_sync(did.clone().into_static()); 102 102 + 103 103 + let permit = match self.semaphore.clone().try_acquire_owned() { 104 104 + Ok(p) => p, 105 105 + Err(_) => break, 106 106 + }; 107 107 + 108 108 + let guard = InFlightGuard { 109 109 + did: did.clone().into_static(), 110 110 + set: self.in_flight.clone(), 111 111 + }; 112 112 + 113 113 + let state = self.state.clone(); 114 114 + let http = self.http.clone(); 115 115 + let buffer_tx_clone = self.buffer_tx.clone(); 116 116 + let did_clone = did.clone(); 117 117 + let verify = self.verify_signatures; 118 118 + 119 119 + tokio::spawn(async move { 120 120 + let _guard = guard; 121 121 + Self::process_did_wrapper( 122 122 + state, 123 123 + http, 124 124 + buffer_tx_clone, 125 125 + did_clone.clone(), 126 126 + permit, 127 127 + verify, 128 128 + ) 129 129 + .await 130 130 + .inspect_err(move |e| { 131 131 + error!("backfill process failed for {did_clone}: {e}"); 132 132 + db::check_poisoned_report(e); 133 133 + }) 134 134 + }); 135 135 + 136 136 + spawned += 1; 137 137 + } 138 138 + 139 139 + if spawned == 0 { 140 140 + self.state.backfill_notify.notified().await; 141 141 + } 81 142 } 82 143 } 83 144 84 145 async fn process_did_wrapper( 85 146 state: Arc<AppState>, 86 147 http: reqwest::Client, 148 148 + buffer_tx: BufferTx, 87 149 did: Did<'static>, 88 150 _permit: tokio::sync::OwnedSemaphorePermit, 89 151 verify_signatures: bool, ··· 132 194 }) 133 195 .await 134 196 .into_diagnostic()??; 197 197 + 198 198 + // Notify completion to worker shard 199 199 + if let Err(e) = buffer_tx.send(IngestMessage::BackfillFinished(did.clone())) { 200 200 + error!("failed to send BackfillFinished for {did}: {e}"); 201 201 + } 135 202 } 136 203 Err(e) => { 137 204 let mut was_ratelimited = false; ··· 232 299 } 233 300 } 234 301 235 235 - // unblock buffer processing for this DID 236 236 - state.blocked_dids.remove_async(&did).await; 302 302 + // wake worker to pick up more 303 303 + state.backfill_notify.notify_one(); 237 304 Ok(()) 238 305 } 239 306 ··· 512 579 count += 1; 513 580 } 514 581 515 515 - // 6. update status to synced 516 516 - state.status = RepoStatus::Synced; 582 582 + // 6. update data, status is updated in worker shard 517 583 state.rev = Some(rev.clone().into()); 518 584 state.data = Some(root_commit.data); 519 585 state.last_updated_at = chrono::Utc::now().timestamp();

+3 -4

src/crawler/mod.rs

··· 1 1 use crate::db::{Db, keys, ser_repo_state}; 2 2 - use crate::ops::send_backfill_req; 3 2 use crate::state::AppState; 4 3 use crate::types::RepoState; 5 4 use jacquard::api::com_atproto::sync::list_repos::{ListRepos, ListReposOutput}; ··· 120 119 self.state.db.update_count_async("pending", count).await; 121 120 } 122 121 123 123 - // 5. queue for backfill 124 124 - for did in to_queue { 125 125 - send_backfill_req(&self.state, did)?; 122 122 + // 5. notify backfill worker 123 123 + if !to_queue.is_empty() { 124 124 + self.state.notify_backfill(); 126 125 } 127 126 128 127 if cursor.is_none() {

+4 -4

src/ingest/firehose.rs

··· 1 1 use crate::db::{self, Db, keys}; 2 2 - use crate::ingest::BufferTx; 2 2 + use crate::ingest::{BufferTx, IngestMessage}; 3 3 use crate::state::AppState; 4 4 use jacquard::api::com_atproto::sync::subscribe_repos::{SubscribeRepos, SubscribeReposMessage}; 5 5 use jacquard::types::did::Did; ··· 17 17 buffer_tx: BufferTx, 18 18 relay_host: Url, 19 19 full_network: bool, 20 20 - verify_signatures: bool, 20 20 + _verify_signatures: bool, 21 21 } 22 22 23 23 impl FirehoseIngestor { ··· 33 33 buffer_tx, 34 34 relay_host, 35 35 full_network, 36 36 - verify_signatures, 36 36 + _verify_signatures: verify_signatures, 37 37 } 38 38 } 39 39 ··· 114 114 // }); 115 115 // } 116 116 117 117 - if let Err(e) = self.buffer_tx.send(msg) { 117 117 + if let Err(e) = self.buffer_tx.send(IngestMessage::Firehose(msg)) { 118 118 error!("failed to send message to buffer processor: {e}"); 119 119 } 120 120 }

+9 -1

src/ingest/mod.rs

··· 4 4 pub mod firehose; 5 5 pub mod worker; 6 6 7 7 - pub type BufferedMessage = SubscribeReposMessage<'static>; 7 7 + use jacquard::types::did::Did; 8 8 + 9 9 + #[derive(Debug)] 10 10 + pub enum IngestMessage { 11 11 + Firehose(SubscribeReposMessage<'static>), 12 12 + BackfillFinished(Did<'static>), 13 13 + } 14 14 + 15 15 + pub type BufferedMessage = IngestMessage; 8 16 9 17 pub type BufferTx = mpsc::UnboundedSender<BufferedMessage>; 10 18 #[allow(dead_code)]

+405 -248

src/ingest/worker.rs

··· 1 1 use crate::db::{self, keys}; 2 2 - use crate::ingest::BufferedMessage; 3 3 - use crate::ops::{self, send_backfill_req}; 2 2 + use crate::ingest::{BufferedMessage, IngestMessage}; 3 3 + use crate::ops; 4 4 use crate::resolver::NoSigningKeyError; 5 5 use crate::state::AppState; 6 6 use crate::types::{AccountEvt, BroadcastEvent, IdentityEvt, RepoState, RepoStatus}; 7 7 use jacquard::api::com_atproto::sync::subscribe_repos::SubscribeReposMessage; 8 8 9 9 use fjall::OwnedWriteBatch; 10 10 - use futures::future::join_all; 10 10 + 11 11 use jacquard::cowstr::ToCowStr; 12 12 + use jacquard::types::crypto::PublicKey; 12 13 use jacquard::types::did::Did; 14 14 + use jacquard_api::com_atproto::sync::subscribe_repos::Commit; 13 15 use jacquard_common::IntoStatic; 14 14 - use jacquard_common::types::crypto::PublicKey; 15 16 use jacquard_repo::error::CommitError; 16 16 - use miette::{Diagnostic, IntoDiagnostic, Result}; 17 17 + use miette::{Context, Diagnostic, IntoDiagnostic, Result}; 17 18 use smol_str::ToSmolStr; 18 18 - use std::collections::{HashMap, HashSet}; 19 19 + use std::collections::{HashMap, HashSet, hash_map::DefaultHasher}; 20 20 + use std::hash::{Hash, Hasher}; 19 21 use std::sync::Arc; 20 20 - use std::time::Duration; 21 22 use tokio::sync::mpsc; 22 22 - use tracing::{debug, error, trace, warn}; 23 23 + use tracing::{debug, error, info, trace, warn}; 23 24 24 25 #[derive(Debug)] 25 26 struct KeyFetchError(miette::Report); ··· 70 71 } 71 72 } 72 73 73 73 - #[derive(Debug, Clone, Copy)] 74 74 - enum ProcessResult { 74 74 + #[derive(Debug)] 75 75 + enum RepoProcessResult<'s, 'c> { 75 76 Deleted, 76 76 - Ok, 77 77 - } 78 78 - 79 79 - enum RepoCheckResult { 80 80 - Syncing, 81 81 - Ok(RepoState<'static>), 77 77 + Syncing(Option<&'c Commit<'c>>), 78 78 + Ok(RepoState<'s>), 82 79 } 83 80 84 81 pub struct FirehoseWorker { 85 82 state: Arc<AppState>, 86 83 rx: mpsc::UnboundedReceiver<BufferedMessage>, 87 84 verify_signatures: bool, 85 85 + num_shards: usize, 86 86 + } 87 87 + 88 88 + struct WorkerContext<'a> { 89 89 + verify_signatures: bool, 90 90 + state: &'a AppState, 91 91 + repo_cache: &'a mut HashMap<Did<'static>, RepoState<'static>>, 92 92 + batch: &'a mut OwnedWriteBatch, 93 93 + added_blocks: &'a mut i64, 94 94 + records_delta: &'a mut i64, 95 95 + broadcast_events: &'a mut Vec<BroadcastEvent>, 96 96 + handle: &'a tokio::runtime::Handle, 88 97 } 89 98 90 99 impl FirehoseWorker { ··· 97 106 state, 98 107 rx, 99 108 verify_signatures, 109 109 + num_shards: 64, 100 110 } 101 111 } 102 112 113 113 + // starts the worker threads and the main dispatch loop 114 114 + // the dispatch loop reads from the firehose channel and distributes messages to shards 115 115 + // based on the consistent hash of the DID 103 116 pub fn run(mut self, handle: tokio::runtime::Handle) -> Result<()> { 104 104 - const BUF_SIZE: usize = 500; 105 105 - let mut buf = Vec::<BufferedMessage>::with_capacity(BUF_SIZE); 106 106 - let mut failed = Vec::<BufferedMessage>::new(); 117 117 + let mut shards = Vec::with_capacity(self.num_shards); 118 118 + 119 119 + for i in 0..self.num_shards { 120 120 + let (tx, rx) = mpsc::unbounded_channel(); 121 121 + shards.push(tx); 122 122 + 123 123 + let state = self.state.clone(); 124 124 + let verify = self.verify_signatures; 125 125 + let handle = handle.clone(); 126 126 + 127 127 + std::thread::Builder::new() 128 128 + .name(format!("ingest-shard-{}", i)) 129 129 + .spawn(move || { 130 130 + Self::worker_thread(i, rx, state, verify, handle); 131 131 + }) 132 132 + .into_diagnostic()?; 133 133 + } 134 134 + 135 135 + info!("started {} ingest shards", self.num_shards); 107 136 108 137 let _g = handle.enter(); 138 138 + 139 139 + // dispatch loop 140 140 + while let Some(msg) = self.rx.blocking_recv() { 141 141 + let did = match &msg { 142 142 + IngestMessage::Firehose(m) => match m { 143 143 + SubscribeReposMessage::Commit(c) => &c.repo, 144 144 + SubscribeReposMessage::Identity(i) => &i.did, 145 145 + SubscribeReposMessage::Account(a) => &a.did, 146 146 + SubscribeReposMessage::Sync(s) => &s.did, 147 147 + _ => continue, 148 148 + }, 149 149 + IngestMessage::BackfillFinished(did) => did, 150 150 + }; 151 151 + 152 152 + let mut hasher = DefaultHasher::new(); 153 153 + did.hash(&mut hasher); 154 154 + let hash = hasher.finish(); 155 155 + let shard_idx = (hash as usize) % self.num_shards; 156 156 + 157 157 + if let Err(e) = shards[shard_idx].send(msg) { 158 158 + error!("failed to send message to shard {shard_idx}: {e}"); 159 159 + // break if send fails; receiver likely closed 160 160 + break; 161 161 + } 162 162 + } 163 163 + 164 164 + error!("firehose worker dispatcher shutting down"); 165 165 + 166 166 + Ok(()) 167 167 + } 168 168 + 169 169 + // synchronous worker loop running on a dedicated thread 170 170 + // pulls messages from the channel, builds batches, and processes them 171 171 + // enters the tokio runtime only when necessary (key resolution) 172 172 + fn worker_thread( 173 173 + id: usize, 174 174 + mut rx: mpsc::UnboundedReceiver<BufferedMessage>, 175 175 + state: Arc<AppState>, 176 176 + verify_signatures: bool, 177 177 + handle: tokio::runtime::Handle, 178 178 + ) { 179 179 + let _guard = handle.enter(); 180 180 + debug!("shard {id} started"); 181 181 + 109 182 let mut repo_cache = HashMap::new(); 110 183 let mut deleted = HashSet::new(); 111 111 - let mut broadcast_events = Vec::<BroadcastEvent>::with_capacity(BUF_SIZE); 184 184 + let mut broadcast_events = Vec::new(); 112 185 113 113 - loop { 114 114 - let mut batch = self.state.db.inner.batch(); 186 186 + while let Some(msg) = rx.blocking_recv() { 187 187 + let mut batch = state.db.inner.batch(); 115 188 repo_cache.clear(); 116 189 deleted.clear(); 117 190 broadcast_events.clear(); 118 191 119 119 - // resolve signing keys for commits and syncs if verification is enabled 120 120 - let keys = if self.verify_signatures { 121 121 - let dids: HashSet<Did> = buf 122 122 - .iter() 123 123 - .filter_map(|msg| match msg { 124 124 - SubscribeReposMessage::Commit(c) => Some(c.repo.clone()), 125 125 - SubscribeReposMessage::Sync(s) => Some(s.did.clone()), 126 126 - _ => None, 127 127 - }) 128 128 - .collect(); 192 192 + let mut added_blocks = 0; 193 193 + let mut records_delta = 0; 129 194 130 130 - let futures = dids.into_iter().map(|did| async { 131 131 - let res = self.state.resolver.resolve_signing_key(&did).await; 132 132 - (did, res) 133 133 - }); 195 195 + let mut ctx = WorkerContext { 196 196 + state: &state, 197 197 + repo_cache: &mut repo_cache, 198 198 + batch: &mut batch, 199 199 + added_blocks: &mut added_blocks, 200 200 + records_delta: &mut records_delta, 201 201 + broadcast_events: &mut broadcast_events, 202 202 + handle: &handle, 203 203 + verify_signatures, 204 204 + }; 134 205 135 135 - handle.block_on(join_all(futures)).into_iter().collect() 136 136 - } else { 137 137 - HashMap::new() 138 138 - }; 206 206 + match msg { 207 207 + IngestMessage::BackfillFinished(did) => { 208 208 + debug!("backfill finished for {did}, verifying state and draining buffer"); 139 209 140 140 - let mut added_blocks = 0; 141 141 - let mut records_delta = 0; 142 142 - for msg in buf.drain(..) { 143 143 - let (did, seq) = match &msg { 144 144 - SubscribeReposMessage::Commit(c) => (&c.repo, c.seq), 145 145 - SubscribeReposMessage::Identity(i) => (&i.did, i.seq), 146 146 - SubscribeReposMessage::Account(a) => (&a.did, a.seq), 147 147 - SubscribeReposMessage::Sync(s) => (&s.did, s.seq), 148 148 - _ => continue, 149 149 - }; 210 210 + // load repo state to transition status and draining buffer 211 211 + let repo_key = keys::repo_key(&did); 212 212 + if let Ok(Some(state_bytes)) = state.db.repos.get(&repo_key).into_diagnostic() { 213 213 + match crate::db::deser_repo_state(&state_bytes) { 214 214 + Ok(repo_state) => { 215 215 + let repo_state = repo_state.into_static(); 150 216 151 151 - if self.state.blocked_dids.contains_sync(did) { 152 152 - failed.push(msg); 153 153 - continue; 154 154 - } 155 155 - if deleted.contains(did) { 156 156 - continue; 217 217 + match Self::drain_resync_buffer(&mut ctx, &did, repo_state) { 218 218 + Ok(res) => match res { 219 219 + RepoProcessResult::Ok(s) => { 220 220 + // TODO: there might be a race condition here where we get a new commit 221 221 + // while the resync buffer is being drained, we should handle that probably 222 222 + // but also it should still be fine since we'll sync eventually anyway 223 223 + match ops::update_repo_status( 224 224 + &mut batch, 225 225 + &state.db, 226 226 + &did, 227 227 + s, 228 228 + RepoStatus::Synced, 229 229 + ) { 230 230 + Ok(s) => { 231 231 + repo_cache.insert(did.clone(), s.into_static()); 232 232 + } 233 233 + Err(e) => { 234 234 + // this can only fail if serde retry fails which would be really weird 235 235 + error!( 236 236 + "failed to transition {did} to synced: {e}" 237 237 + ); 238 238 + } 239 239 + } 240 240 + } 241 241 + RepoProcessResult::Deleted => { 242 242 + deleted.insert(did.clone()); 243 243 + } 244 244 + // we don't have to handle this since drain_resync_buffer doesn't delete 245 245 + // the commits from the resync buffer so they will get retried later 246 246 + RepoProcessResult::Syncing(_) => {} 247 247 + }, 248 248 + Err(e) => { 249 249 + error!("failed to drain resync buffer for {did}: {e}") 250 250 + } 251 251 + }; 252 252 + } 253 253 + Err(e) => error!("failed to deser repo state for {did}: {e}"), 254 254 + } 255 255 + } 157 256 } 257 257 + IngestMessage::Firehose(msg) => { 258 258 + let (did, seq) = match &msg { 259 259 + SubscribeReposMessage::Commit(c) => (&c.repo, c.seq), 260 260 + SubscribeReposMessage::Identity(i) => (&i.did, i.seq), 261 261 + SubscribeReposMessage::Account(a) => (&a.did, a.seq), 262 262 + SubscribeReposMessage::Sync(s) => (&s.did, s.seq), 263 263 + _ => continue, 264 264 + }; 158 265 159 159 - match self.process_message( 160 160 - &mut repo_cache, 161 161 - &mut batch, 162 162 - &mut added_blocks, 163 163 - &mut records_delta, 164 164 - &mut broadcast_events, 165 165 - &msg, 166 166 - did, 167 167 - &keys, 168 168 - ) { 169 169 - Ok(ProcessResult::Ok) => {} 170 170 - Ok(ProcessResult::Deleted) => { 171 171 - deleted.insert(did.clone()); 266 266 + if deleted.contains(did) { 267 267 + continue; 172 268 } 173 173 - Err(e) => { 174 174 - error!("error processing message for {did}: {e}"); 175 175 - db::check_poisoned_report(&e); 176 176 - // dont retry commit or sync on key fetch errors 177 177 - // since we'll just try again later if we get commit or sync again 178 178 - if e.downcast_ref::<KeyFetchError>().is_none() 179 179 - && e.downcast_ref::<CommitError>().is_none() 180 180 - && e.downcast_ref::<NoSigningKeyError>().is_none() 181 181 - { 182 182 - failed.push(msg); 269 269 + 270 270 + match Self::process_message(&mut ctx, &msg, did) { 271 271 + Ok(RepoProcessResult::Ok(_)) => {} 272 272 + Ok(RepoProcessResult::Deleted) => { 273 273 + deleted.insert(did.clone()); 274 274 + } 275 275 + Ok(RepoProcessResult::Syncing(Some(commit))) => { 276 276 + if let Err(e) = ops::persist_to_resync_buffer(&state.db, did, commit) { 277 277 + error!("failed to persist commit to resync_buffer for {did}: {e}"); 278 278 + } 279 279 + } 280 280 + Ok(RepoProcessResult::Syncing(None)) => {} 281 281 + Err(e) => { 282 282 + error!("error processing message for {did}: {e}"); 283 283 + db::check_poisoned_report(&e); 284 284 + if Self::check_if_retriable_failure(&e) { 285 285 + if let SubscribeReposMessage::Commit(commit) = &msg { 286 286 + if let Err(e) = 287 287 + ops::persist_to_resync_buffer(&state.db, did, commit) 288 288 + { 289 289 + error!( 290 290 + "failed to persist commit to resync_buffer for {did}: {e}" 291 291 + ); 292 292 + } 293 293 + } 294 294 + } 183 295 } 184 296 } 185 185 - } 186 297 187 187 - self.state 188 188 - .cur_firehose 189 189 - .store(seq, std::sync::atomic::Ordering::SeqCst); 298 298 + state 299 299 + .cur_firehose 300 300 + .store(seq, std::sync::atomic::Ordering::SeqCst); 301 301 + } 190 302 } 191 303 192 192 - // commit all changes to db 193 193 - batch.commit().into_diagnostic()?; 304 304 + if let Err(e) = batch.commit() { 305 305 + error!("failed to commit batch in shard {id}: {e}"); 306 306 + } 194 307 195 308 if added_blocks > 0 { 196 196 - self.state.db.update_count("blocks", added_blocks); 309 309 + state.db.update_count("blocks", added_blocks); 197 310 } 198 311 if records_delta != 0 { 199 199 - self.state.db.update_count("records", records_delta); 312 312 + state.db.update_count("records", records_delta); 200 313 } 201 314 for evt in broadcast_events.drain(..) { 202 202 - let _ = self.state.db.event_tx.send(evt); 203 203 - } 204 204 - 205 205 - self.state 206 206 - .db 207 207 - .inner 208 208 - .persist(fjall::PersistMode::Buffer) 209 209 - .into_diagnostic()?; 210 210 - 211 211 - // add failed back to buf here so the ordering is preserved 212 212 - if !failed.is_empty() { 213 213 - buf.append(&mut failed); 315 315 + let _ = state.db.event_tx.send(evt); 214 316 } 215 317 216 216 - // wait until we receive some messages 217 217 - // this does mean we will have an up to 1 second delay, before we send events to consumers 218 218 - // but thats reasonable imo, could also be configured of course 219 219 - let _ = handle.block_on(async { 220 220 - tokio::time::timeout( 221 221 - Duration::from_secs(1), 222 222 - self.rx.recv_many(&mut buf, BUF_SIZE), 223 223 - ) 224 224 - .await 225 225 - }); 226 226 - if buf.is_empty() { 227 227 - if self.rx.is_closed() { 228 228 - error!("ingestor crashed? shutting down buffer processor"); 229 229 - break; 230 230 - } 231 231 - continue; 232 232 - } 318 318 + state.db.inner.persist(fjall::PersistMode::Buffer).ok(); 233 319 } 320 320 + } 234 321 235 235 - Ok(()) 322 322 + // dont retry commit or sync on key fetch errors 323 323 + // since we'll just try again later if we get commit or sync again 324 324 + fn check_if_retriable_failure(e: &miette::Report) -> bool { 325 325 + e.downcast_ref::<KeyFetchError>().is_none() 326 326 + && e.downcast_ref::<CommitError>().is_none() 327 327 + && e.downcast_ref::<NoSigningKeyError>().is_none() 236 328 } 237 329 238 238 - fn process_message( 239 239 - &self, 240 240 - repo_cache: &mut HashMap<Did<'static>, RepoState<'static>>, 241 241 - batch: &mut OwnedWriteBatch, 242 242 - added_blocks: &mut i64, 243 243 - records_delta: &mut i64, 244 244 - broadcast_events: &mut Vec<BroadcastEvent>, 245 245 - msg: &BufferedMessage, 330 330 + fn process_message<'s, 'c>( 331 331 + ctx: &mut WorkerContext, 332 332 + msg: &'c SubscribeReposMessage<'static>, 246 333 did: &Did, 247 247 - keys: &HashMap<Did<'static>, Result<PublicKey<'static>>>, 248 248 - ) -> Result<ProcessResult> { 249 249 - let state = &self.state; 250 250 - let verify_signatures = self.verify_signatures; 251 251 - 252 252 - let RepoCheckResult::Ok(repo_state) = 253 253 - Self::check_repo_state(repo_cache, batch, state, did, msg)? 254 254 - else { 255 255 - return Ok(ProcessResult::Ok); 256 256 - }; 257 257 - 258 258 - let get_key = || { 259 259 - if verify_signatures { 260 260 - let key = keys.get(did).ok_or_else(|| { 261 261 - KeyFetchError(miette::miette!( 262 262 - "!!! THIS IS A BUG !!! missing pubkey for {did}" 263 263 - )) 264 264 - })?; 265 265 - match key { 266 266 - Ok(key) => Ok(Some(key)), 267 267 - Err(e) => { 268 268 - return Err(KeyFetchError(miette::miette!( 269 269 - "failed to get pubkey for {did}: {e}" 270 270 - ))); 271 271 - } 272 272 - } 273 273 - } else { 274 274 - Ok(None) 334 334 + ) -> Result<RepoProcessResult<'s, 'c>> { 335 335 + let check_repo_res = Self::check_repo_state(ctx, did, msg)?; 336 336 + let mut repo_state = match check_repo_res { 337 337 + RepoProcessResult::Syncing(_) | RepoProcessResult::Deleted => { 338 338 + return Ok(check_repo_res); 275 339 } 340 340 + RepoProcessResult::Ok(s) => s, 276 341 }; 277 342 278 343 match msg { 279 344 SubscribeReposMessage::Commit(commit) => { 280 345 trace!("processing buffered commit for {did}"); 281 346 282 282 - if matches!(repo_state.rev, Some(ref rev) if commit.rev.as_str() <= rev.to_tid().as_str()) 283 283 - { 284 284 - debug!( 285 285 - "skipping replayed event for {}: {} <= {}", 286 286 - did, 287 287 - commit.rev, 288 288 - repo_state 289 289 - .rev 290 290 - .as_ref() 291 291 - .map(|r| r.to_tid()) 292 292 - .expect("we checked in if") 293 293 - ); 294 294 - return Ok(ProcessResult::Ok); 295 295 - } 296 296 - 297 297 - if let (Some(repo), Some(prev_commit)) = (&repo_state.data, &commit.prev_data) 298 298 - && repo != &prev_commit.0.to_ipld().expect("valid cid") 299 299 - { 300 300 - warn!( 301 301 - "gap detected for {}: repo {} != commit prev {}. triggering backfill", 302 302 - did, repo, prev_commit.0 303 303 - ); 304 304 - 305 305 - let mut batch = state.db.inner.batch(); 306 306 - ops::update_repo_status( 307 307 - &mut batch, 308 308 - &state.db, 309 309 - did, 310 310 - repo_state, 311 311 - RepoStatus::Backfilling, 312 312 - )?; 313 313 - batch.commit().into_diagnostic()?; 314 314 - send_backfill_req(state, did.clone().into_static())?; 315 315 - 316 316 - return Ok(ProcessResult::Ok); 317 317 - } 318 318 - 319 319 - let res = ops::apply_commit(batch, &state.db, repo_state, &commit, get_key()?)?; 320 320 - repo_cache.insert(did.clone().into_static(), res.repo_state); 321 321 - *added_blocks += res.blocks_count; 322 322 - *records_delta += res.records_delta; 323 323 - broadcast_events.push(BroadcastEvent::Persisted( 324 324 - self.state 325 325 - .db 326 326 - .next_event_id 327 327 - .load(std::sync::atomic::Ordering::SeqCst) 328 328 - - 1, 329 329 - )); 347 347 + return Self::process_commit(ctx, did, repo_state, commit); 330 348 } 331 349 SubscribeReposMessage::Sync(sync) => { 332 350 debug!("processing buffered sync for {did}"); 333 351 334 334 - match ops::verify_sync_event(sync.blocks.as_ref(), get_key()?) { 352 352 + match ops::verify_sync_event( 353 353 + sync.blocks.as_ref(), 354 354 + Self::fetch_key(ctx, did)?.as_ref(), 355 355 + ) { 335 356 Ok((root, rev)) => { 336 357 if let Some(current_data) = &repo_state.data { 337 358 if current_data == &root.to_ipld().expect("valid cid") { 338 359 debug!("skipping noop sync for {did}"); 339 339 - return Ok(ProcessResult::Ok); 360 360 + return Ok(RepoProcessResult::Ok(repo_state)); 340 361 } 341 362 } 342 363 343 364 if let Some(current_rev) = &repo_state.rev { 344 365 if rev.as_str() <= current_rev.to_tid().as_str() { 345 366 debug!("skipping replayed sync for {did}"); 346 346 - return Ok(ProcessResult::Ok); 367 367 + return Ok(RepoProcessResult::Ok(repo_state)); 347 368 } 348 369 } 349 370 350 371 warn!("sync event for {did}: triggering backfill"); 351 351 - let mut batch = state.db.inner.batch(); 352 352 - ops::update_repo_status( 372 372 + let mut batch = ctx.state.db.inner.batch(); 373 373 + repo_state = ops::update_repo_status( 353 374 &mut batch, 354 354 - &state.db, 375 375 + &ctx.state.db, 355 376 did, 356 377 repo_state, 357 378 RepoStatus::Backfilling, 358 379 )?; 380 380 + ctx.state.db.update_count("pending", 1); 359 381 batch.commit().into_diagnostic()?; 360 360 - 361 361 - send_backfill_req(state, did.clone().into_static())?; 362 362 - return Ok(ProcessResult::Ok); 382 382 + ctx.state.notify_backfill(); 383 383 + return Ok(RepoProcessResult::Ok(repo_state)); 363 384 } 364 385 Err(e) => { 365 386 error!("failed to process sync event for {did}: {e}"); ··· 377 398 did: did.clone().into_static(), 378 399 handle, 379 400 }; 380 380 - broadcast_events.push(ops::make_identity_event(&state.db, evt)); 401 401 + ctx.broadcast_events 402 402 + .push(ops::make_identity_event(&ctx.state.db, evt)); 381 403 } 382 404 SubscribeReposMessage::Account(account) => { 383 405 debug!("processing buffered account for {did}"); ··· 392 414 match &account.status { 393 415 Some(AccountStatus::Deleted) => { 394 416 debug!("account {did} deleted, wiping data"); 395 395 - ops::delete_repo(batch, &state.db, did)?; 396 396 - return Ok(ProcessResult::Deleted); 417 417 + ops::delete_repo(ctx.batch, &ctx.state.db, did)?; 418 418 + return Ok(RepoProcessResult::Deleted); 397 419 } 398 420 status => { 399 421 let target_status = match status { ··· 425 447 426 448 if repo_state.status == target_status { 427 449 debug!("account status unchanged for {did}: {target_status:?}"); 428 428 - return Ok(ProcessResult::Ok); 450 450 + return Ok(RepoProcessResult::Ok(repo_state)); 429 451 } 430 452 431 431 - let new_state = ops::update_repo_status( 432 432 - batch, 433 433 - &state.db, 453 453 + repo_state = ops::update_repo_status( 454 454 + ctx.batch, 455 455 + &ctx.state.db, 434 456 did, 435 457 repo_state, 436 458 target_status, 437 459 )?; 438 438 - repo_cache.insert(did.clone().into_static(), new_state); 460 460 + ctx.repo_cache.insert( 461 461 + did.clone().into_static(), 462 462 + repo_state.clone().into_static(), 463 463 + ); 439 464 } 440 465 } 441 466 } else { ··· 444 469 // 1. we handle changing repo status to Synced before this (in check repo state) 445 470 // 2. initiating backfilling is also handled there 446 471 } 447 447 - 448 448 - broadcast_events.push(ops::make_account_event(&state.db, evt)); 472 472 + ctx.broadcast_events 473 473 + .push(ops::make_account_event(&ctx.state.db, evt)); 449 474 } 450 475 _ => { 451 476 warn!("unknown message type in buffer for {did}"); 452 477 } 453 478 } 454 479 455 455 - Ok(ProcessResult::Ok) 480 480 + Ok(RepoProcessResult::Ok(repo_state)) 456 481 } 457 482 458 458 - fn check_repo_state( 459 459 - repo_cache: &mut HashMap<Did<'static>, RepoState<'static>>, 460 460 - batch: &mut OwnedWriteBatch, 461 461 - state: &AppState, 483 483 + fn process_commit<'c, 'ns, 's: 'ns>( 484 484 + ctx: &mut WorkerContext, 485 485 + did: &Did, 486 486 + repo_state: RepoState<'s>, 487 487 + commit: &'c Commit<'c>, 488 488 + ) -> Result<RepoProcessResult<'ns, 'c>> { 489 489 + // check for replayed events (already seen revision) 490 490 + if matches!(repo_state.rev, Some(ref rev) if commit.rev.as_str() <= rev.to_tid().as_str()) { 491 491 + debug!( 492 492 + "skipping replayed event for {}: {} <= {}", 493 493 + did, 494 494 + commit.rev, 495 495 + repo_state 496 496 + .rev 497 497 + .as_ref() 498 498 + .map(|r| r.to_tid()) 499 499 + .expect("we checked in if") 500 500 + ); 501 501 + return Ok(RepoProcessResult::Ok(repo_state)); 502 502 + } 503 503 + 504 504 + if let (Some(repo), Some(prev_commit)) = (&repo_state.data, &commit.prev_data) 505 505 + && repo 506 506 + != &prev_commit 507 507 + .0 508 508 + .to_ipld() 509 509 + .into_diagnostic() 510 510 + .wrap_err("invalid cid from relay")? 511 511 + { 512 512 + warn!( 513 513 + "gap detected for {}: repo {} != commit prev {}. triggering backfill", 514 514 + did, repo, prev_commit.0 515 515 + ); 516 516 + 517 517 + let mut batch = ctx.state.db.inner.batch(); 518 518 + let repo_state = ops::update_repo_status( 519 519 + &mut batch, 520 520 + &ctx.state.db, 521 521 + did, 522 522 + repo_state, 523 523 + RepoStatus::Backfilling, 524 524 + )?; 525 525 + ctx.state.db.update_count("pending", 1); 526 526 + batch.commit().into_diagnostic()?; 527 527 + ctx.repo_cache 528 528 + .insert(did.clone().into_static(), repo_state.clone().into_static()); 529 529 + ctx.state.notify_backfill(); 530 530 + return Ok(RepoProcessResult::Syncing(Some(commit))); 531 531 + } 532 532 + 533 533 + let res = ops::apply_commit( 534 534 + ctx.batch, 535 535 + &ctx.state.db, 536 536 + repo_state, 537 537 + &commit, 538 538 + Self::fetch_key(ctx, did)?.as_ref(), 539 539 + )?; 540 540 + let repo_state = res.repo_state; 541 541 + ctx.repo_cache 542 542 + .insert(did.clone().into_static(), repo_state.clone().into_static()); 543 543 + *ctx.added_blocks += res.blocks_count; 544 544 + *ctx.records_delta += res.records_delta; 545 545 + ctx.broadcast_events.push(BroadcastEvent::Persisted( 546 546 + ctx.state 547 547 + .db 548 548 + .next_event_id 549 549 + .load(std::sync::atomic::Ordering::SeqCst) 550 550 + - 1, 551 551 + )); 552 552 + 553 553 + Ok(RepoProcessResult::Ok(repo_state)) 554 554 + } 555 555 + 556 556 + // checks the current state of the repo in the database 557 557 + // if the repo is new, creates initial state and triggers backfill 558 558 + // handles transitions between states (backfilling -> synced, etc) 559 559 + fn check_repo_state<'s, 'c>( 560 560 + ctx: &mut WorkerContext, 462 561 did: &Did<'_>, 463 463 - msg: &BufferedMessage, 464 464 - ) -> Result<RepoCheckResult> { 562 562 + msg: &'c SubscribeReposMessage<'static>, 563 563 + ) -> Result<RepoProcessResult<'s, 'c>> { 465 564 // check if we have this repo 466 466 - if let Some(state) = repo_cache.get(did) { 467 467 - return Ok(RepoCheckResult::Ok(state.clone())); 565 565 + if let Some(state) = ctx.repo_cache.get(did) { 566 566 + return Ok(RepoProcessResult::Ok(state.clone())); 468 567 } 469 568 470 569 let repo_key = keys::repo_key(&did); 471 471 - let Some(state_bytes) = state.db.repos.get(&repo_key).into_diagnostic()? else { 570 570 + let Some(state_bytes) = ctx.state.db.repos.get(&repo_key).into_diagnostic()? else { 472 571 // we don't know this repo, but we are receiving events for it 473 572 // this means we should backfill it before processing its events 474 573 debug!("discovered new account {did} from firehose, queueing backfill"); ··· 477 576 // using a separate batch here since we want to make it known its being backfilled 478 577 // immediately. we could use the batch for the unit of work we are doing but 479 578 // then we wouldn't be able to start backfilling until the unit of work is done 480 480 - let mut batch = state.db.inner.batch(); 481 579 580 580 + let mut batch = ctx.state.db.inner.batch(); 482 581 batch.insert( 483 483 - &state.db.repos, 582 582 + &ctx.state.db.repos, 484 583 &repo_key, 485 584 crate::db::ser_repo_state(&new_state)?, 486 585 ); 487 487 - batch.insert(&state.db.pending, &repo_key, &[]); 586 586 + batch.insert(&ctx.state.db.pending, &repo_key, &[]); 587 587 + ctx.state.db.update_count("repos", 1); 588 588 + ctx.state.db.update_count("pending", 1); 488 589 batch.commit().into_diagnostic()?; 489 590 490 490 - send_backfill_req(state, did.clone().into_static())?; 591 591 + ctx.state.notify_backfill(); 491 592 492 492 - return Ok(RepoCheckResult::Syncing); 593 593 + return Ok(RepoProcessResult::Syncing(None)); 493 594 }; 494 595 let mut repo_state = crate::db::deser_repo_state(&state_bytes)?.into_static(); 495 596 496 597 // if we are backfilling or it is new, DON'T mark it as synced yet 497 598 // the backfill worker will do that when it finishes 498 599 match &repo_state.status { 499 499 - RepoStatus::Synced => Ok(RepoCheckResult::Ok(repo_state)), 600 600 + RepoStatus::Synced => { 601 601 + // lazy drain: if there are buffered commits, drain them now 602 602 + if ops::has_buffered_commits(&ctx.state.db, did) { 603 603 + Self::drain_resync_buffer(ctx, did, repo_state) 604 604 + } else { 605 605 + Ok(RepoProcessResult::Ok(repo_state)) 606 606 + } 607 607 + } 500 608 RepoStatus::Backfilling | RepoStatus::Error(_) => { 501 501 - // repo is being backfilled or is in error state 502 502 - // we dont touch the state because the backfill worker will do that 503 503 - // we should not really get here because the backfill worker should have marked it as 504 504 - // being worked on (blocked repos) meaning we would have returned earlier 505 609 debug!( 506 610 "ignoring active status for {did} as it is {:?}", 507 611 repo_state.status 508 612 ); 509 509 - Ok(RepoCheckResult::Syncing) 613 613 + Ok(RepoProcessResult::Syncing(None)) 510 614 } 511 615 RepoStatus::Deactivated | RepoStatus::Suspended | RepoStatus::Takendown => { 512 616 // if it was in deactivated/takendown/suspended state, we can mark it as synced ··· 514 618 // UNLESS it is an account status event that keeps it deactivated 515 619 if let SubscribeReposMessage::Account(acc) = msg { 516 620 if !acc.active { 517 517 - return Ok(RepoCheckResult::Ok(repo_state)); 621 621 + return Ok(RepoProcessResult::Ok(repo_state)); 518 622 } 519 623 } 520 520 - 521 624 repo_state = ops::update_repo_status( 522 522 - batch, 523 523 - &state.db, 524 524 - &did, 625 625 + ctx.batch, 626 626 + &ctx.state.db, 627 627 + did, 525 628 repo_state, 526 629 RepoStatus::Synced, 527 630 )?; 528 528 - repo_cache.insert(did.clone().into_static(), repo_state.clone()); 529 529 - Ok(RepoCheckResult::Ok(repo_state)) 631 631 + ctx.repo_cache 632 632 + .insert(did.clone().into_static(), repo_state.clone()); 633 633 + Ok(RepoProcessResult::Ok(repo_state)) 634 634 + } 635 635 + } 636 636 + } 637 637 + 638 638 + fn drain_resync_buffer<'s>( 639 639 + ctx: &mut WorkerContext, 640 640 + did: &Did, 641 641 + mut repo_state: RepoState<'s>, 642 642 + ) -> Result<RepoProcessResult<'s, 'static>> { 643 643 + let prefix = keys::resync_buffer_prefix(did); 644 644 + 645 645 + for guard in ctx.state.db.resync_buffer.prefix(&prefix) { 646 646 + let (key, value) = guard.into_inner().into_diagnostic()?; 647 647 + let commit: Commit = rmp_serde::from_slice(&value).into_diagnostic()?; 648 648 + 649 649 + let res = Self::process_commit(ctx, did, repo_state, &commit); 650 650 + let res = match res { 651 651 + Ok(r) => r, 652 652 + Err(e) => { 653 653 + if !Self::check_if_retriable_failure(&e) { 654 654 + ctx.batch.remove(&ctx.state.db.resync_buffer, key); 655 655 + } 656 656 + return Err(e); 657 657 + } 658 658 + }; 659 659 + match res { 660 660 + RepoProcessResult::Ok(rs) => { 661 661 + ctx.batch.remove(&ctx.state.db.resync_buffer, key); 662 662 + repo_state = rs; 663 663 + } 664 664 + RepoProcessResult::Syncing(_) => { 665 665 + return Ok(RepoProcessResult::Syncing(None)); 666 666 + } 667 667 + RepoProcessResult::Deleted => { 668 668 + ctx.batch.remove(&ctx.state.db.resync_buffer, key); 669 669 + return Ok(RepoProcessResult::Deleted); 670 670 + } 530 671 } 672 672 + } 673 673 + 674 674 + Ok(RepoProcessResult::Ok(repo_state)) 675 675 + } 676 676 + 677 677 + fn fetch_key(ctx: &WorkerContext, did: &Did) -> Result<Option<PublicKey<'static>>> { 678 678 + if ctx.verify_signatures { 679 679 + let key = ctx 680 680 + .handle 681 681 + .block_on(ctx.state.resolver.resolve_signing_key(did)) 682 682 + .map_err(|e| { 683 683 + KeyFetchError(miette::miette!("failed to get pubkey for {did}: {e}")) 684 684 + })?; 685 685 + Ok(Some(key)) 686 686 + } else { 687 687 + Ok(None) 531 688 } 532 689 } 533 690 }

+19 -25

src/main.rs

··· 24 24 25 25 info!("{cfg}"); 26 26 27 27 - let (state, backfill_rx) = AppState::new(&cfg)?; 27 27 + let state = AppState::new(&cfg)?; 28 28 let (buffer_tx, buffer_rx) = mpsc::unbounded_channel(); 29 29 let state = Arc::new(state); 30 30 - 31 31 - tokio::spawn( 32 32 - api::serve(state.clone(), cfg.api_port).inspect_err(|e| error!("API server failed: {e}")), 33 33 - ); 34 34 - 35 35 - if cfg.enable_debug { 36 36 - tokio::spawn( 37 37 - api::serve_debug(state.clone(), cfg.debug_port) 38 38 - .inspect_err(|e| error!("debug server failed: {e}")), 39 39 - ); 40 40 - } 41 30 42 31 if !cfg.disable_backfill { 43 32 tokio::spawn({ ··· 45 34 let timeout = cfg.repo_fetch_timeout; 46 35 BackfillWorker::new( 47 36 state, 48 48 - backfill_rx, 37 37 + buffer_tx.clone(), 49 38 timeout, 50 39 cfg.backfill_concurrency_limit, 51 40 matches!( ··· 55 44 ) 56 45 .run() 57 46 }); 58 58 - } 59 59 - 60 60 - if let Err(e) = spawn_blocking({ 61 61 - let state = state.clone(); 62 62 - move || hydrant::backfill::manager::queue_pending_backfills(&state) 63 63 - }) 64 64 - .await 65 65 - .into_diagnostic()? 66 66 - { 67 67 - error!("failed to queue pending backfills: {e}"); 68 68 - db::check_poisoned_report(&e); 69 47 } 70 48 71 49 if let Err(e) = spawn_blocking({ ··· 156 134 ); 157 135 } 158 136 159 159 - let tasks = if !cfg.disable_firehose { 137 137 + let mut tasks = if !cfg.disable_firehose { 160 138 let firehose_worker = std::thread::spawn({ 161 139 let state = state.clone(); 162 140 let handle = tokio::runtime::Handle::current(); ··· 195 173 // essentially we just want to keep the main thread alive for the other components 196 174 vec![Box::pin(futures::future::pending::<miette::Result<()>>()) as BoxFuture<_>] 197 175 }; 176 176 + 177 177 + let state_api = state.clone(); 178 178 + tasks.push(Box::pin(async move { 179 179 + api::serve(state_api, cfg.api_port) 180 180 + .await 181 181 + .map_err(|e| miette::miette!("API server failed: {e}")) 182 182 + }) as BoxFuture<_>); 183 183 + 184 184 + if cfg.enable_debug { 185 185 + let state_debug = state.clone(); 186 186 + tasks.push(Box::pin(async move { 187 187 + api::serve_debug(state_debug, cfg.debug_port) 188 188 + .await 189 189 + .map_err(|e| miette::miette!("debug server failed: {e}")) 190 190 + }) as BoxFuture<_>); 191 191 + } 198 192 199 193 let res = futures::future::select_all(tasks); 200 194 if let (Err(e), _, _) = res.await {

+21 -7

src/ops.rs

··· 1 1 use crate::db::types::{DbAction, DbRkey, DbTid, TrimmedDid}; 2 2 use crate::db::{self, Db, keys, ser_repo_state}; 3 3 - use crate::state::AppState; 4 3 use crate::types::{ 5 4 AccountEvt, BroadcastEvent, IdentityEvt, MarshallableEvt, RepoState, RepoStatus, ResyncState, 6 5 StoredEvent, ··· 10 9 use jacquard::IntoStatic; 11 10 12 11 use jacquard::types::cid::Cid; 12 12 + use jacquard::types::did::Did; 13 13 use jacquard_api::com_atproto::sync::subscribe_repos::Commit; 14 14 use jacquard_common::types::crypto::PublicKey; 15 15 use jacquard_repo::car::reader::parse_car_bytes; ··· 19 19 use std::time::Instant; 20 20 use tracing::{debug, trace, warn}; 21 21 22 22 - pub fn send_backfill_req(state: &AppState, did: jacquard::types::did::Did<'static>) -> Result<()> { 23 23 - state 24 24 - .backfill_tx 25 25 - .send(did.clone()) 26 26 - .map_err(|_| miette::miette!("failed to send backfill request for {did}"))?; 27 27 - let _ = state.blocked_dids.insert_sync(did); 22 22 + pub fn persist_to_resync_buffer(db: &Db, did: &Did, commit: &Commit) -> Result<()> { 23 23 + let key = keys::resync_buffer_key(did, DbTid::from(&commit.rev)); 24 24 + let value = rmp_serde::to_vec(commit).into_diagnostic()?; 25 25 + db.resync_buffer.insert(key, value).into_diagnostic()?; 26 26 + debug!( 27 27 + "buffered commit seq {} for {did} to resync_buffer", 28 28 + commit.seq 29 29 + ); 28 30 Ok(()) 31 31 + } 32 32 + 33 33 + pub fn has_buffered_commits(db: &Db, did: &Did) -> bool { 34 34 + let prefix = keys::resync_buffer_prefix(did); 35 35 + db.resync_buffer.prefix(&prefix).next().is_some() 29 36 } 30 37 31 38 // emitting identity is ephemeral ··· 66 73 batch.remove(&db.repos, &repo_key); 67 74 batch.remove(&db.pending, &repo_key); 68 75 batch.remove(&db.resync, &repo_key); 76 76 + 77 77 + let resync_prefix = keys::resync_buffer_prefix(did); 78 78 + for guard in db.resync_buffer.prefix(&resync_prefix) { 79 79 + let k = guard.key().into_diagnostic()?; 80 80 + batch.remove(&db.resync_buffer, k); 81 81 + } 69 82 70 83 // 2. delete from records (all partitions) 71 84 let mut partitions = Vec::new(); ··· 151 164 152 165 Ok(repo_state) 153 166 } 167 167 + 154 168 pub fn verify_sync_event(blocks: &[u8], key: Option<&PublicKey>) -> Result<(Cid<'static>, String)> { 155 169 let parsed = tokio::task::block_in_place(|| { 156 170 tokio::runtime::Handle::current()

+13 -20

src/state.rs

··· 1 1 use std::sync::atomic::AtomicI64; 2 2 3 3 - use jacquard_common::types::string::Did; 4 4 - use tokio::sync::mpsc; 5 5 - 6 3 use miette::Result; 4 4 + use tokio::sync::Notify; 7 5 8 6 use crate::{config::Config, db::Db, resolver::Resolver}; 9 7 10 10 - pub type BackfillTx = mpsc::UnboundedSender<Did<'static>>; 11 11 - pub type BackfillRx = mpsc::UnboundedReceiver<Did<'static>>; 12 12 - 13 8 pub struct AppState { 14 9 pub db: Db, 15 15 - pub backfill_tx: BackfillTx, 16 10 pub resolver: Resolver, 17 11 pub cur_firehose: AtomicI64, 18 18 - pub blocked_dids: scc::HashSet<Did<'static>>, 12 12 + pub backfill_notify: Notify, 19 13 } 20 14 21 15 impl AppState { 22 22 - pub fn new(config: &Config) -> Result<(Self, BackfillRx)> { 16 16 + pub fn new(config: &Config) -> Result<Self> { 23 17 let db = Db::open( 24 18 &config.database_path, 25 19 config.cache_size, 26 20 config.disable_lz4_compression, 27 21 )?; 28 22 let resolver = Resolver::new(config.plc_url.clone(), config.identity_cache_size); 29 29 - let (backfill_tx, backfill_rx) = mpsc::unbounded_channel(); 23 23 + 24 24 + Ok(Self { 25 25 + db, 26 26 + resolver, 27 27 + cur_firehose: AtomicI64::new(0), 28 28 + backfill_notify: Notify::new(), 29 29 + }) 30 30 + } 30 31 31 31 - Ok(( 32 32 - Self { 33 33 - db, 34 34 - backfill_tx, 35 35 - resolver, 36 36 - cur_firehose: AtomicI64::new(0), 37 37 - blocked_dids: scc::HashSet::new(), 38 38 - }, 39 39 - backfill_rx, 40 40 - )) 32 32 + pub fn notify_backfill(&self) { 33 33 + self.backfill_notify.notify_one(); 41 34 } 42 35 }