+18 -16

src/api/repo.rs

··· 1 1 use crate::api::AppState; 2 2 use crate::db::{Db, keys, ser_repo_state}; 3 3 + use crate::ops; 3 4 use crate::types::{GaugeState, RepoState}; 4 5 use axum::{Json, Router, extract::State, http::StatusCode, routing::post}; 5 6 use jacquard::types::did::Did; 7 7 + use rand::Rng; 6 8 use serde::Deserialize; 7 9 use std::sync::Arc; 8 10 ··· 33 35 .await 34 36 .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? 35 37 { 36 36 - let repo_state = RepoState::backfilling(&did); 38 38 + let repo_state = RepoState::backfilling(rand::rng().next_u64()); 37 39 let bytes = ser_repo_state(&repo_state) 38 40 .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; 39 41 40 42 batch.insert(&db.repos, &did_key, bytes); 41 41 - batch.insert(&db.pending, &did_key, Vec::new()); 43 43 + batch.insert( 44 44 + &db.pending, 45 45 + keys::pending_key(repo_state.index_id), 46 46 + &did_key, 47 47 + ); 42 48 43 49 added += 1; 44 50 } ··· 88 94 .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; 89 95 90 96 let was_pending = matches!(repo_state.status, crate::types::RepoStatus::Backfilling); 91 91 - let _was_resync = matches!( 92 92 - repo_state.status, 93 93 - crate::types::RepoStatus::Error(_) 94 94 - | crate::types::RepoStatus::Deactivated 95 95 - | crate::types::RepoStatus::Takendown 96 96 - | crate::types::RepoStatus::Suspended 97 97 - ); 97 97 + // todo: idk 98 98 + // let was_resync = matches!( 99 99 + // repo_state.status, 100 100 + // crate::types::RepoStatus::Error(_) 101 101 + // | crate::types::RepoStatus::Deactivated 102 102 + // | crate::types::RepoStatus::Takendown 103 103 + // | crate::types::RepoStatus::Suspended 104 104 + // ); 98 105 99 106 let old_gauge = if was_pending { 100 107 GaugeState::Pending ··· 115 122 GaugeState::Synced 116 123 }; 117 124 118 118 - batch.remove(&db.repos, &did_key); 119 119 - if was_pending { 120 120 - batch.remove(&db.pending, &did_key); 121 121 - } 122 122 - if old_gauge.is_resync() { 123 123 - batch.remove(&db.resync, &did_key); 124 124 - } 125 125 + ops::delete_repo(&mut batch, db, &did, repo_state) 126 126 + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; 125 127 126 128 state 127 129 .db

+85 -31

src/backfill/manager.rs

··· 1 1 use crate::db::types::TrimmedDid; 2 2 - use crate::db::{self, deser_repo_state, keys, ser_repo_state}; 2 2 + use crate::db::{self, deser_repo_state}; 3 3 + use crate::ops; 3 4 use crate::state::AppState; 4 5 use crate::types::{RepoStatus, ResyncState}; 5 6 use miette::{IntoDiagnostic, Result}; ··· 11 12 debug!("scanning for deactivated/takendown repos to retry..."); 12 13 let mut count = 0; 13 14 15 15 + let mut batch = state.db.inner.batch(); 16 16 + 14 17 for guard in state.db.resync.iter() { 15 18 let (key, val) = guard.into_inner().into_diagnostic()?; 16 19 let did = match TrimmedDid::try_from(key.as_ref()) { ··· 25 28 if matches!(resync_state, ResyncState::Gone { .. }) { 26 29 debug!("queuing retry for gone repo: {did}"); 27 30 28 28 - // move back to pending 29 29 - let mut batch = state.db.inner.batch(); 30 30 - batch.remove(&state.db.resync, key.clone()); 31 31 - batch.insert(&state.db.pending, key.clone(), Vec::new()); 31 31 + let Some(state_bytes) = state.db.repos.get(&key).into_diagnostic()? else { 32 32 + error!("repo state not found for {did}"); 33 33 + continue; 34 34 + }; 32 35 33 36 // update repo state back to backfilling 34 34 - if let Some(state_bytes) = state.db.repos.get(&key).into_diagnostic()? { 35 35 - let mut repo_state = deser_repo_state(&state_bytes)?; 36 36 - repo_state.status = RepoStatus::Backfilling; 37 37 - batch.insert(&state.db.repos, key, ser_repo_state(&repo_state)?); 38 38 - } 39 39 - 40 40 - batch.commit().into_diagnostic()?; 41 41 - state.db.update_count("resync", -1); 42 42 - state.db.update_count("pending", 1); 37 37 + let repo_state = deser_repo_state(&state_bytes)?; 38 38 + ops::update_repo_status( 39 39 + &mut batch, 40 40 + &state.db, 41 41 + &did, 42 42 + repo_state, 43 43 + RepoStatus::Backfilling, 44 44 + )?; 43 45 44 44 - state.notify_backfill(); 45 46 count += 1; 46 47 } 47 48 } 48 49 } 49 50 51 51 + if count == 0 { 52 52 + return Ok(()); 53 53 + } 54 54 + 55 55 + batch.commit().into_diagnostic()?; 56 56 + 57 57 + state.db.update_count("resync", -count); 58 58 + state.db.update_count("pending", count); 59 59 + 60 60 + state.notify_backfill(); 61 61 + 50 62 info!("queued {count} gone backfills"); 51 63 Ok(()) 52 64 } ··· 60 72 61 73 let now = chrono::Utc::now().timestamp(); 62 74 let mut count = 0; 75 75 + 76 76 + let mut batch = state.db.inner.batch(); 63 77 64 78 for guard in db.resync.iter() { 65 79 let (key, value) = match guard.into_inner() { ··· 78 92 } 79 93 }; 80 94 81 81 - if let Ok(ResyncState::Error { next_retry, .. }) = 82 82 - rmp_serde::from_slice::<ResyncState>(&value) 83 83 - { 84 84 - if next_retry <= now { 85 85 - debug!("retrying backfill for {did}"); 95 95 + match rmp_serde::from_slice::<ResyncState>(&value) { 96 96 + Ok(ResyncState::Error { next_retry, .. }) => { 97 97 + if next_retry <= now { 98 98 + debug!("retrying backfill for {did}"); 86 99 87 87 - // move back to pending 88 88 - if let Err(e) = db.pending.insert(keys::repo_key(&did), Vec::new()) { 89 89 - error!("failed to move {did} to pending: {e}"); 90 90 - db::check_poisoned(&e); 91 91 - continue; 92 92 - } 93 93 - state.db.update_count("pending", 1); 100 100 + let state_bytes = match state.db.repos.get(&key).into_diagnostic() { 101 101 + Ok(b) => b, 102 102 + Err(err) => { 103 103 + error!("failed to get repo state for {did}: {err}"); 104 104 + continue; 105 105 + } 106 106 + }; 107 107 + let Some(state_bytes) = state_bytes else { 108 108 + error!("repo state not found for {did}"); 109 109 + continue; 110 110 + }; 94 111 95 95 - state.notify_backfill(); 96 96 - count += 1; 112 112 + let repo_state = match deser_repo_state(&state_bytes) { 113 113 + Ok(s) => s, 114 114 + Err(e) => { 115 115 + error!("failed to deserialize repo state for {did}: {e}"); 116 116 + continue; 117 117 + } 118 118 + }; 119 119 + let res = ops::update_repo_status( 120 120 + &mut batch, 121 121 + &state.db, 122 122 + &did, 123 123 + repo_state, 124 124 + RepoStatus::Backfilling, 125 125 + ); 126 126 + if let Err(e) = res { 127 127 + error!("failed to update repo status for {did}: {e}"); 128 128 + continue; 129 129 + } 130 130 + 131 131 + count += 1; 132 132 + } 133 133 + } 134 134 + Ok(_) => { 135 135 + // not an error state, do nothing 136 136 + } 137 137 + Err(e) => { 138 138 + error!("failed to deserialize resync state for {did}: {e}"); 139 139 + continue; 97 140 } 98 141 } 99 142 } 100 143 101 101 - if count > 0 { 102 102 - info!("queued {count} retries"); 144 144 + if count == 0 { 145 145 + continue; 103 146 } 147 147 + 148 148 + if let Err(e) = batch.commit() { 149 149 + error!("failed to commit batch: {e}"); 150 150 + db::check_poisoned(&e); 151 151 + continue; 152 152 + } 153 153 + 154 154 + state.db.update_count("resync", -count); 155 155 + state.db.update_count("pending", count); 156 156 + state.notify_backfill(); 157 157 + info!("queued {count} retries"); 104 158 } 105 159 }

+7 -10

src/backfill/mod.rs

··· 84 84 let mut spawned = 0; 85 85 86 86 for guard in self.state.db.pending.iter() { 87 87 - let key = match guard.key() { 88 88 - Ok(k) => k, 87 87 + let (key, value) = match guard.into_inner() { 88 88 + Ok(kv) => kv, 89 89 Err(e) => { 90 90 - error!("failed to read pending key: {e}"); 90 90 + error!("failed to read pending entry: {e}"); 91 91 db::check_poisoned(&e); 92 92 continue; 93 93 } 94 94 }; 95 95 96 96 - let did = match TrimmedDid::try_from(key.as_ref()) { 96 96 + let did = match TrimmedDid::try_from(value.as_ref()) { 97 97 Ok(d) => d.to_did(), 98 98 Err(e) => { 99 99 - error!("invalid did '{key:?}' in pending: {e}"); 99 99 + error!("invalid did in pending value: {e}"); 100 100 continue; 101 101 } 102 102 }; ··· 168 168 169 169 // determine old gauge state 170 170 // if it was error/suspended etc, we need to know which error kind it was to decrement correctly. 171 171 - // we have to peek at the resync state. `previous_state` is the repo state, which tells us the Status. 172 171 // we have to peek at the resync state. `previous_state` is the repo state, which tells us the Status. 173 172 let old_gauge = match previous_state.status { 174 173 RepoStatus::Backfilling => GaugeState::Pending, ··· 393 392 start.elapsed() 394 393 ); 395 394 396 396 - if let Some(h) = handle { 397 397 - state.handle = Some(h.to_smolstr()); 398 398 - } 395 395 + state.handle = handle.map(|h| h.into_static()); 399 396 400 397 let emit_identity = |status: &RepoStatus| { 401 398 let evt = AccountEvt { ··· 428 425 if matches!(e, GetRepoError::RepoNotFound(_)) { 429 426 warn!("repo {did} not found, deleting"); 430 427 let mut batch = db.inner.batch(); 431 431 - ops::delete_repo(&mut batch, db, did)?; 428 428 + ops::delete_repo(&mut batch, db, did, state)?; 432 429 batch.commit().into_diagnostic()?; 433 430 return Ok(previous_state); // stop backfill 434 431 }

+6 -2

src/crawler/mod.rs

··· 5 5 use jacquard::prelude::*; 6 6 use jacquard_common::CowStr; 7 7 use miette::{IntoDiagnostic, Result}; 8 8 + use rand::Rng; 9 9 + use rand::rngs::SmallRng; 8 10 use smol_str::SmolStr; 9 11 use std::sync::Arc; 10 12 use std::time::Duration; ··· 37 39 38 40 pub async fn run(self) -> Result<()> { 39 41 info!("crawler started"); 42 42 + 43 43 + let mut rng: SmallRng = rand::make_rng(); 40 44 41 45 let db = &self.state.db; 42 46 ··· 140 144 if !Db::contains_key(db.repos.clone(), &did_key).await? { 141 145 trace!("crawler found new repo: {}", repo.did); 142 146 143 143 - let state = RepoState::backfilling(&repo.did); 147 147 + let state = RepoState::backfilling(rng.next_u64()); 144 148 batch.insert(&db.repos, &did_key, ser_repo_state(&state)?); 145 145 - batch.insert(&db.pending, &did_key, Vec::new()); 149 149 + batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key); 146 150 to_queue.push(repo.did.clone()); 147 151 } 148 152 }

+4

src/db/keys.rs

··· 15 15 vec 16 16 } 17 17 18 18 + pub fn pending_key(id: u64) -> [u8; 8] { 19 19 + id.to_be_bytes() 20 20 + } 21 21 + 18 22 // prefix format: {DID}| (DID trimmed) 19 23 pub fn record_prefix_did(did: &Did) -> Vec<u8> { 20 24 let repo = TrimmedDid::from(did);

+25 -51

src/ingest/worker.rs

··· 15 15 use jacquard_common::IntoStatic; 16 16 use jacquard_repo::error::CommitError; 17 17 use miette::{Context, Diagnostic, IntoDiagnostic, Result}; 18 18 + use rand::Rng; 18 19 use smol_str::ToSmolStr; 19 19 - use std::collections::{HashMap, HashSet, hash_map::DefaultHasher}; 20 20 + use std::collections::hash_map::DefaultHasher; 20 21 use std::hash::{Hash, Hasher}; 21 22 use std::sync::Arc; 22 23 use thiserror::Error; ··· 64 65 struct WorkerContext<'a> { 65 66 verify_signatures: bool, 66 67 state: &'a AppState, 67 67 - repo_cache: &'a mut HashMap<Did<'static>, RepoState<'static>>, 68 68 batch: &'a mut OwnedWriteBatch, 69 69 added_blocks: &'a mut i64, 70 70 records_delta: &'a mut i64, ··· 156 156 let _guard = handle.enter(); 157 157 debug!("shard {id} started"); 158 158 159 159 - let mut repo_cache = HashMap::new(); 160 160 - let mut deleted = HashSet::new(); 161 159 let mut broadcast_events = Vec::new(); 162 160 163 161 while let Some(msg) = rx.blocking_recv() { 164 162 let mut batch = state.db.inner.batch(); 165 165 - repo_cache.clear(); 166 166 - deleted.clear(); 167 163 broadcast_events.clear(); 168 164 169 165 let mut added_blocks = 0; ··· 171 167 172 168 let mut ctx = WorkerContext { 173 169 state: &state, 174 174 - repo_cache: &mut repo_cache, 175 170 batch: &mut batch, 176 171 added_blocks: &mut added_blocks, 177 172 records_delta: &mut records_delta, ··· 197 192 // TODO: there might be a race condition here where we get a new commit 198 193 // while the resync buffer is being drained, we should handle that probably 199 194 // but also it should still be fine since we'll sync eventually anyway 200 200 - match ops::update_repo_status( 195 195 + let res = ops::update_repo_status( 201 196 &mut batch, 202 197 &state.db, 203 198 &did, 204 199 s, 205 200 RepoStatus::Synced, 206 206 - ) { 207 207 - Ok(s) => { 208 208 - repo_cache.insert(did.clone(), s.into_static()); 209 209 - } 210 210 - Err(e) => { 211 211 - // this can only fail if serde retry fails which would be really weird 212 212 - error!( 213 213 - "failed to transition {did} to synced: {e}" 214 214 - ); 215 215 - } 201 201 + ); 202 202 + if let Err(e) = res { 203 203 + // this can only fail if serde retry fails which would be really weird 204 204 + error!("failed to transition {did} to synced: {e}"); 216 205 } 217 217 - } 218 218 - RepoProcessResult::Deleted => { 219 219 - deleted.insert(did.clone()); 220 206 } 221 207 // we don't have to handle this since drain_resync_buffer doesn't delete 222 208 // the commits from the resync buffer so they will get retried later 223 209 RepoProcessResult::Syncing(_) => {} 210 210 + RepoProcessResult::Deleted => {} 224 211 }, 225 212 Err(e) => { 226 213 error!("failed to drain resync buffer for {did}: {e}") ··· 240 227 _ => continue, 241 228 }; 242 229 243 243 - if deleted.contains(did) { 244 244 - continue; 245 245 - } 246 246 - 247 230 match Self::process_message(&mut ctx, &msg, did) { 248 231 Ok(RepoProcessResult::Ok(_)) => {} 249 249 - Ok(RepoProcessResult::Deleted) => { 250 250 - deleted.insert(did.clone()); 251 251 - } 232 232 + Ok(RepoProcessResult::Deleted) => {} 252 233 Ok(RepoProcessResult::Syncing(Some(commit))) => { 253 234 if let Err(e) = ops::persist_to_resync_buffer(&state.db, did, commit) { 254 235 error!("failed to persist commit to resync_buffer for {did}: {e}"); ··· 359 340 repo_state, 360 341 RepoStatus::Backfilling, 361 342 )?; 362 362 - batch.insert(&ctx.state.db.pending, keys::repo_key(did), &[]); 363 343 batch.commit().into_diagnostic()?; 364 344 ctx.state 365 345 .db ··· 399 379 match &account.status { 400 380 Some(AccountStatus::Deleted) => { 401 381 debug!("account {did} deleted, wiping data"); 402 402 - ops::delete_repo(ctx.batch, &ctx.state.db, did)?; 382 382 + ops::delete_repo(ctx.batch, &ctx.state.db, did, repo_state)?; 403 383 return Ok(RepoProcessResult::Deleted); 404 384 } 405 385 status => { ··· 442 422 repo_state, 443 423 target_status, 444 424 )?; 445 445 - ctx.repo_cache.insert( 446 446 - did.clone().into_static(), 447 447 - repo_state.clone().into_static(), 448 448 - ); 425 425 + ctx.state 426 426 + .db 427 427 + .update_gauge_diff(&GaugeState::Synced, &GaugeState::Resync(None)); 449 428 } 450 429 } 451 430 } else { ··· 500 479 ); 501 480 502 481 let mut batch = ctx.state.db.inner.batch(); 503 503 - let repo_state = ops::update_repo_status( 482 482 + let _repo_state = ops::update_repo_status( 504 483 &mut batch, 505 484 &ctx.state.db, 506 485 did, 507 486 repo_state, 508 487 RepoStatus::Backfilling, 509 488 )?; 510 510 - batch.insert(&ctx.state.db.pending, keys::repo_key(did), &[]); 511 489 batch.commit().into_diagnostic()?; 512 490 ctx.state.db.update_gauge_diff( 513 491 &crate::types::GaugeState::Synced, 514 492 &crate::types::GaugeState::Pending, 515 493 ); 516 516 - ctx.repo_cache 517 517 - .insert(did.clone().into_static(), repo_state.clone().into_static()); 518 494 ctx.state.notify_backfill(); 519 495 return Ok(RepoProcessResult::Syncing(Some(commit))); 520 496 } ··· 527 503 Self::fetch_key(ctx, did)?.as_ref(), 528 504 )?; 529 505 let repo_state = res.repo_state; 530 530 - ctx.repo_cache 531 531 - .insert(did.clone().into_static(), repo_state.clone().into_static()); 532 506 *ctx.added_blocks += res.blocks_count; 533 507 *ctx.records_delta += res.records_delta; 534 508 ctx.broadcast_events.push(BroadcastEvent::Persisted( ··· 550 524 did: &Did<'_>, 551 525 msg: &'c SubscribeReposMessage<'static>, 552 526 ) -> Result<RepoProcessResult<'s, 'c>, IngestError> { 553 553 - // check if we have this repo 554 554 - if let Some(state) = ctx.repo_cache.get(did) { 555 555 - return Ok(RepoProcessResult::Ok(state.clone())); 556 556 - } 557 557 - 558 527 let repo_key = keys::repo_key(&did); 559 528 let Some(state_bytes) = ctx.state.db.repos.get(&repo_key).into_diagnostic()? else { 560 529 // we don't know this repo, but we are receiving events for it 561 530 // this means we should backfill it before processing its events 562 531 debug!("discovered new account {did} from firehose, queueing backfill"); 563 532 564 564 - let new_state = RepoState::backfilling(did); 533 533 + let repo_state = RepoState::backfilling(rand::rng().next_u64()); 565 534 // using a separate batch here since we want to make it known its being backfilled 566 535 // immediately. we could use the batch for the unit of work we are doing but 567 536 // then we wouldn't be able to start backfilling until the unit of work is done 568 568 - 569 537 let mut batch = ctx.state.db.inner.batch(); 570 538 batch.insert( 571 539 &ctx.state.db.repos, 572 540 &repo_key, 573 573 - crate::db::ser_repo_state(&new_state)?, 541 541 + crate::db::ser_repo_state(&repo_state)?, 542 542 + ); 543 543 + batch.insert( 544 544 + &ctx.state.db.pending, 545 545 + keys::pending_key(repo_state.index_id), 546 546 + &repo_key, 574 547 ); 575 575 - batch.insert(&ctx.state.db.pending, repo_key, &[]); 576 548 batch.commit().into_diagnostic()?; 577 549 578 550 ctx.state.db.update_count("repos", 1); ··· 621 593 repo_state, 622 594 RepoStatus::Synced, 623 595 )?; 624 624 - ctx.repo_cache 625 625 - .insert(did.clone().into_static(), repo_state.clone()); 596 596 + ctx.state.db.update_gauge_diff( 597 597 + &crate::types::GaugeState::Resync(None), 598 598 + &crate::types::GaugeState::Synced, 599 599 + ); 626 600 Ok(RepoProcessResult::Ok(repo_state)) 627 601 } 628 602 }

+42 -15

src/ops.rs

··· 14 14 use jacquard_common::types::crypto::PublicKey; 15 15 use jacquard_repo::car::reader::parse_car_bytes; 16 16 use miette::{Context, IntoDiagnostic, Result}; 17 17 + use rand::{Rng, rng}; 17 18 use std::collections::HashMap; 18 19 use std::sync::atomic::Ordering; 19 20 use std::time::Instant; ··· 65 66 batch: &'batch mut OwnedWriteBatch, 66 67 db: &Db, 67 68 did: &jacquard::types::did::Did, 69 69 + repo_state: RepoState, 68 70 ) -> Result<()> { 69 71 debug!("deleting repo {did}"); 72 72 + 70 73 let repo_key = keys::repo_key(did); 74 74 + let pending_key = keys::pending_key(repo_state.index_id); 71 75 72 76 // 1. delete from repos, pending, resync 73 77 batch.remove(&db.repos, &repo_key); 74 74 - batch.remove(&db.pending, &repo_key); 75 75 - batch.remove(&db.resync, &repo_key); 78 78 + match repo_state.status { 79 79 + RepoStatus::Synced => {} 80 80 + RepoStatus::Backfilling => { 81 81 + batch.remove(&db.pending, &pending_key); 82 82 + } 83 83 + _ => { 84 84 + batch.remove(&db.resync, &repo_key); 85 85 + } 86 86 + } 76 87 88 88 + // 2. delete from resync buffer 77 89 let resync_prefix = keys::resync_buffer_prefix(did); 78 90 for guard in db.resync_buffer.prefix(&resync_prefix) { 79 91 let k = guard.key().into_diagnostic()?; 80 92 batch.remove(&db.resync_buffer, k); 81 93 } 82 94 83 83 - // 2. delete from records 95 95 + // 3. delete from records 84 96 let records_prefix = keys::record_prefix_did(did); 85 97 for guard in db.records.prefix(&records_prefix) { 86 98 let k = guard.key().into_diagnostic()?; 87 99 batch.remove(&db.records, k); 88 100 } 89 101 90 90 - // 3. reset collection counts 102 102 + // 4. reset collection counts 91 103 let mut count_prefix = Vec::new(); 92 104 count_prefix.push(b'r'); 93 105 count_prefix.push(keys::SEP); ··· 111 123 ) -> Result<RepoState<'s>> { 112 124 debug!("updating repo status for {did} to {new_status:?}"); 113 125 114 114 - let key = keys::repo_key(did); 126 126 + let repo_key = keys::repo_key(did); 127 127 + let pending_key = keys::pending_key(repo_state.index_id); 115 128 116 129 // manage queues 117 130 match &new_status { 118 131 RepoStatus::Synced => { 119 119 - batch.remove(&db.pending, &key); 120 120 - batch.remove(&db.resync, &key); 132 132 + batch.remove(&db.pending, &pending_key); 133 133 + // we dont have to remove from resync here because it has to transition resync -> pending first 121 134 } 122 135 RepoStatus::Backfilling => { 123 123 - batch.insert(&db.pending, &key, &[]); 124 124 - batch.remove(&db.resync, &key); 136 136 + // if we are coming from an error state, remove from resync 137 137 + if !matches!(repo_state.status, RepoStatus::Synced) { 138 138 + batch.remove(&db.resync, &repo_key); 139 139 + } 140 140 + // remove the old entry 141 141 + batch.remove(&db.pending, &pending_key); 142 142 + // add as new entry 143 143 + repo_state.index_id = rng().next_u64(); 144 144 + batch.insert( 145 145 + &db.pending, 146 146 + keys::pending_key(repo_state.index_id), 147 147 + &repo_key, 148 148 + ); 125 149 } 126 150 RepoStatus::Error(_msg) => { 127 127 - batch.remove(&db.pending, &key); 151 151 + batch.remove(&db.pending, &pending_key); 152 152 + // TODO: we need to make errors have kind instead of "message" in repo status 153 153 + // and then pass it to resync error kind 128 154 let resync_state = crate::types::ResyncState::Error { 129 129 - kind: crate::types::ResyncErrorKind::Generic, // ops errors are usually generic logic errors? or transport? 155 155 + kind: crate::types::ResyncErrorKind::Generic, 130 156 retry_count: 0, 131 157 next_retry: chrono::Utc::now().timestamp(), 132 158 }; 133 159 batch.insert( 134 160 &db.resync, 135 135 - &key, 161 161 + &repo_key, 136 162 rmp_serde::to_vec(&resync_state).into_diagnostic()?, 137 163 ); 138 164 } 139 165 RepoStatus::Deactivated | RepoStatus::Takendown | RepoStatus::Suspended => { 140 140 - batch.remove(&db.pending, &key); 166 166 + // this shouldnt be needed since a repo wont be in a pending state when it gets to any of these states 167 167 + // batch.remove(&db.pending, &pending_key); 141 168 let resync_state = ResyncState::Gone { 142 169 status: new_status.clone(), 143 170 }; 144 171 batch.insert( 145 172 &db.resync, 146 146 - &key, 173 173 + &repo_key, 147 174 rmp_serde::to_vec(&resync_state).into_diagnostic()?, 148 175 ); 149 176 } ··· 152 179 repo_state.status = new_status; 153 180 repo_state.last_updated_at = chrono::Utc::now().timestamp(); 154 181 155 155 - batch.insert(&db.repos, &key, ser_repo_state(&repo_state)?); 182 182 + batch.insert(&db.repos, &repo_key, ser_repo_state(&repo_state)?); 156 183 157 184 Ok(repo_state) 158 185 }

+8 -10

src/types.rs

··· 1 1 use std::fmt::Display; 2 2 3 3 - use jacquard::{CowStr, IntoStatic}; 3 3 + use jacquard::{CowStr, IntoStatic, types::string::Handle}; 4 4 use jacquard_common::types::string::Did; 5 5 use serde::{Deserialize, Serialize}; 6 6 use serde_json::Value; ··· 35 35 #[derive(Debug, Clone, Serialize, Deserialize)] 36 36 #[serde(bound(deserialize = "'i: 'de"))] 37 37 pub struct RepoState<'i> { 38 38 - #[serde(borrow)] 39 39 - pub did: TrimmedDid<'i>, 40 38 pub status: RepoStatus, 41 39 pub rev: Option<DbTid>, 42 40 pub data: Option<IpldCid>, 43 41 pub last_seq: Option<i64>, 44 42 pub last_updated_at: i64, // unix timestamp 45 45 - pub handle: Option<SmolStr>, 43 43 + #[serde(borrow)] 44 44 + pub handle: Option<Handle<'i>>, 45 45 + pub index_id: u64, 46 46 } 47 47 48 48 impl<'i> RepoState<'i> { 49 49 - pub fn backfilling(did: &'i Did<'i>) -> Self { 49 49 + pub fn backfilling(index_id: u64) -> Self { 50 50 Self { 51 51 - did: TrimmedDid::from(did), 52 51 status: RepoStatus::Backfilling, 53 52 rev: None, 54 53 data: None, 55 54 last_seq: None, 56 55 last_updated_at: chrono::Utc::now().timestamp(), 57 56 handle: None, 57 57 + index_id, 58 58 } 59 59 } 60 60 } ··· 64 64 65 65 fn into_static(self) -> Self::Output { 66 66 RepoState { 67 67 - did: self.did.into_static(), 68 67 status: self.status, 69 68 rev: self.rev, 70 69 data: self.data, 71 70 last_seq: self.last_seq, 72 71 last_updated_at: self.last_updated_at, 73 73 - handle: self.handle, 72 72 + handle: self.handle.map(|s| s.into_static()), 73 73 + index_id: self.index_id, 74 74 } 75 75 } 76 76 } 77 77 - 78 78 - // from src/backfill/resync_state.rs 79 77 80 78 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] 81 79 pub enum ResyncErrorKind {