+15

.sqlx/query-03faaf7b8676e0af1bf620759425632560dabfd5748d0383971c10f9b2847d7d.json

··· 1 1 + { 2 2 + "db_name": "PostgreSQL", 3 3 + "query": "\n DELETE FROM user_blocks\n WHERE user_id = $1\n AND block_cid = ANY($2)\n ", 4 4 + "describe": { 5 5 + "columns": [], 6 6 + "parameters": { 7 7 + "Left": [ 8 8 + "Uuid", 9 9 + "ByteaArray" 10 10 + ] 11 11 + }, 12 12 + "nullable": [] 13 13 + }, 14 14 + "hash": "03faaf7b8676e0af1bf620759425632560dabfd5748d0383971c10f9b2847d7d" 15 15 + }

+15

.sqlx/query-d71881b1dd8111b2afff6a7af8829651379afbe050dcc8a93e0b91eced31ca89.json

··· 1 1 + { 2 2 + "db_name": "PostgreSQL", 3 3 + "query": "\n INSERT INTO user_blocks (user_id, block_cid)\n SELECT $1, block_cid FROM UNNEST($2::bytea[]) AS t(block_cid)\n ON CONFLICT (user_id, block_cid) DO NOTHING\n ", 4 4 + "describe": { 5 5 + "columns": [], 6 6 + "parameters": { 7 7 + "Left": [ 8 8 + "Uuid", 9 9 + "ByteaArray" 10 10 + ] 11 11 + }, 12 12 + "nullable": [] 13 13 + }, 14 14 + "hash": "d71881b1dd8111b2afff6a7af8829651379afbe050dcc8a93e0b91eced31ca89" 15 15 + }

+22

.sqlx/query-e70fc3dced4eb7dc220ca2a18cdfcbd5f2d66dff2262bb083fd4118b032ff978.json

··· 1 1 + { 2 2 + "db_name": "PostgreSQL", 3 3 + "query": "SELECT block_cid FROM user_blocks WHERE user_id = $1", 4 4 + "describe": { 5 5 + "columns": [ 6 6 + { 7 7 + "ordinal": 0, 8 8 + "name": "block_cid", 9 9 + "type_info": "Bytea" 10 10 + } 11 11 + ], 12 12 + "parameters": { 13 13 + "Left": [ 14 14 + "Uuid" 15 15 + ] 16 16 + }, 17 17 + "nullable": [ 18 18 + false 19 19 + ] 20 20 + }, 21 21 + "hash": "e70fc3dced4eb7dc220ca2a18cdfcbd5f2d66dff2262bb083fd4118b032ff978" 22 22 + }

+1

migrations/20260106_clear_user_blocks.sql

··· 1 1 + TRUNCATE TABLE user_blocks;

+9 -8

src/api/backup.rs

··· 220 220 } 221 221 }; 222 222 223 223 - let car_bytes = match generate_full_backup(&state.block_store, &head_cid).await { 224 224 - Ok(bytes) => bytes, 225 225 - Err(e) => { 226 226 - error!("Failed to generate CAR: {:?}", e); 227 227 - return ApiError::InternalError(Some("Failed to generate backup".into())) 228 228 - .into_response(); 229 229 - } 230 230 - }; 223 223 + let car_bytes = 224 224 + match generate_full_backup(&state.db, &state.block_store, user.id, &head_cid).await { 225 225 + Ok(bytes) => bytes, 226 226 + Err(e) => { 227 227 + error!("Failed to generate CAR: {:?}", e); 228 228 + return ApiError::InternalError(Some("Failed to generate backup".into())) 229 229 + .into_response(); 230 230 + } 231 231 + }; 231 232 232 233 let block_count = crate::scheduled::count_car_blocks(&car_bytes); 233 234 let size_bytes = car_bytes.len() as i64;

+34 -16

src/api/repo/record/batch.rs

··· 388 388 return ApiError::InternalError(Some("Failed to persist MST".into())).into_response(); 389 389 } 390 390 }; 391 391 - let mut relevant_blocks = std::collections::BTreeMap::new(); 391 391 + let mut new_mst_blocks = std::collections::BTreeMap::new(); 392 392 + let mut old_mst_blocks = std::collections::BTreeMap::new(); 392 393 for key in &modified_keys { 393 393 - if mst 394 394 - .blocks_for_path(key, &mut relevant_blocks) 395 395 - .await 396 396 - .is_err() 397 397 - { 394 394 + if mst.blocks_for_path(key, &mut new_mst_blocks).await.is_err() { 398 395 return ApiError::InternalError(Some("Failed to get new MST blocks for path".into())) 399 396 .into_response(); 400 397 } 401 398 if original_mst 402 402 - .blocks_for_path(key, &mut relevant_blocks) 399 399 + .blocks_for_path(key, &mut old_mst_blocks) 403 400 .await 404 401 .is_err() 405 402 { ··· 407 404 .into_response(); 408 405 } 409 406 } 410 410 - let mut written_cids = tracking_store.get_all_relevant_cids(); 411 411 - for cid in relevant_blocks.keys() { 412 412 - if !written_cids.contains(cid) { 413 413 - written_cids.push(*cid); 407 407 + let mut relevant_blocks = new_mst_blocks.clone(); 408 408 + relevant_blocks.extend(old_mst_blocks.iter().map(|(k, v)| (*k, v.clone()))); 409 409 + let written_cids: Vec<Cid> = tracking_store 410 410 + .get_all_relevant_cids() 411 411 + .into_iter() 412 412 + .chain(relevant_blocks.keys().copied()) 413 413 + .collect::<std::collections::HashSet<_>>() 414 414 + .into_iter() 415 415 + .collect(); 416 416 + let written_cids_str: Vec<String> = written_cids.iter().map(|c| c.to_string()).collect(); 417 417 + let prev_record_cids = ops.iter().filter_map(|op| match op { 418 418 + RecordOp::Update { 419 419 + prev: Some(cid), .. 414 420 } 415 415 - } 416 416 - let written_cids_str = written_cids 417 417 - .iter() 418 418 - .map(|c| c.to_string()) 419 419 - .collect::<Vec<_>>(); 421 421 + | RecordOp::Delete { 422 422 + prev: Some(cid), .. 423 423 + } => Some(*cid), 424 424 + _ => None, 425 425 + }); 426 426 + let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid) 427 427 + .chain( 428 428 + old_mst_blocks 429 429 + .keys() 430 430 + .filter(|cid| !new_mst_blocks.contains_key(*cid)) 431 431 + .copied(), 432 432 + ) 433 433 + .chain(prev_record_cids) 434 434 + .collect::<std::collections::HashSet<_>>() 435 435 + .into_iter() 436 436 + .collect(); 420 437 let commit_res = match commit_and_log( 421 438 &state, 422 439 CommitParams { ··· 428 445 ops, 429 446 blocks_cids: &written_cids_str, 430 447 blobs: &all_blob_cids, 448 448 + obsolete_cids, 431 449 }, 432 450 ) 433 451 .await

+24 -13

src/api/repo/record/delete.rs

··· 129 129 rkey: rkey_for_audit.clone(), 130 130 prev: prev_record_cid, 131 131 }; 132 132 - let mut relevant_blocks = std::collections::BTreeMap::new(); 132 132 + let mut new_mst_blocks = std::collections::BTreeMap::new(); 133 133 + let mut old_mst_blocks = std::collections::BTreeMap::new(); 133 134 if new_mst 134 134 - .blocks_for_path(&key, &mut relevant_blocks) 135 135 + .blocks_for_path(&key, &mut new_mst_blocks) 135 136 .await 136 137 .is_err() 137 138 { ··· 139 140 .into_response(); 140 141 } 141 142 if mst 142 142 - .blocks_for_path(&key, &mut relevant_blocks) 143 143 + .blocks_for_path(&key, &mut old_mst_blocks) 143 144 .await 144 145 .is_err() 145 146 { 146 147 return ApiError::InternalError(Some("Failed to get old MST blocks for path".into())) 147 148 .into_response(); 148 149 } 149 149 - let mut written_cids = tracking_store.get_all_relevant_cids(); 150 150 - for cid in relevant_blocks.keys() { 151 151 - if !written_cids.contains(cid) { 152 152 - written_cids.push(*cid); 153 153 - } 154 154 - } 155 155 - let written_cids_str = written_cids 156 156 - .iter() 157 157 - .map(|c| c.to_string()) 158 158 - .collect::<Vec<_>>(); 150 150 + let mut relevant_blocks = new_mst_blocks.clone(); 151 151 + relevant_blocks.extend(old_mst_blocks.iter().map(|(k, v)| (*k, v.clone()))); 152 152 + let written_cids: Vec<Cid> = tracking_store 153 153 + .get_all_relevant_cids() 154 154 + .into_iter() 155 155 + .chain(relevant_blocks.keys().copied()) 156 156 + .collect::<std::collections::HashSet<_>>() 157 157 + .into_iter() 158 158 + .collect(); 159 159 + let written_cids_str: Vec<String> = written_cids.iter().map(|c| c.to_string()).collect(); 160 160 + let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid) 161 161 + .chain( 162 162 + old_mst_blocks 163 163 + .keys() 164 164 + .filter(|cid| !new_mst_blocks.contains_key(*cid)) 165 165 + .copied(), 166 166 + ) 167 167 + .chain(prev_record_cid) 168 168 + .collect(); 159 169 let commit_result = match commit_and_log( 160 170 &state, 161 171 CommitParams { ··· 167 177 ops: vec![op], 168 178 blocks_cids: &written_cids_str, 169 179 blobs: &[], 180 180 + obsolete_cids, 170 181 }, 171 182 ) 172 183 .await

+39 -9

src/api/repo/record/utils.rs

··· 92 92 pub ops: Vec<RecordOp>, 93 93 pub blocks_cids: &'a [String], 94 94 pub blobs: &'a [String], 95 95 + pub obsolete_cids: Vec<Cid>, 95 96 } 96 97 97 98 pub async fn commit_and_log( ··· 107 108 ops, 108 109 blocks_cids, 109 110 blobs, 111 111 + obsolete_cids, 110 112 } = params; 111 113 let key_row = sqlx::query!( 112 114 "SELECT key_bytes, encryption_version FROM user_keys WHERE user_id = $1", ··· 200 202 .execute(&mut *tx) 201 203 .await 202 204 .map_err(|e| format!("DB Error (user_blocks): {}", e))?; 205 205 + } 206 206 + if !obsolete_cids.is_empty() { 207 207 + let obsolete_bytes: Vec<Vec<u8>> = obsolete_cids.iter().map(|c| c.to_bytes()).collect(); 208 208 + sqlx::query!( 209 209 + r#" 210 210 + DELETE FROM user_blocks 211 211 + WHERE user_id = $1 212 212 + AND block_cid = ANY($2) 213 213 + "#, 214 214 + user_id, 215 215 + &obsolete_bytes as &[Vec<u8>] 216 216 + ) 217 217 + .execute(&mut *tx) 218 218 + .await 219 219 + .map_err(|e| format!("DB Error (user_blocks delete obsolete): {}", e))?; 203 220 } 204 221 let mut upsert_collections: Vec<String> = Vec::new(); 205 222 let mut upsert_rkeys: Vec<String> = Vec::new(); ··· 404 421 rkey: rkey.to_string(), 405 422 cid: record_cid, 406 423 }; 407 407 - let mut relevant_blocks = std::collections::BTreeMap::new(); 424 424 + let mut new_mst_blocks = std::collections::BTreeMap::new(); 425 425 + let mut old_mst_blocks = std::collections::BTreeMap::new(); 408 426 new_mst 409 409 - .blocks_for_path(&key, &mut relevant_blocks) 427 427 + .blocks_for_path(&key, &mut new_mst_blocks) 410 428 .await 411 429 .map_err(|e| format!("Failed to get new MST blocks for path: {:?}", e))?; 412 412 - mst.blocks_for_path(&key, &mut relevant_blocks) 430 430 + mst.blocks_for_path(&key, &mut old_mst_blocks) 413 431 .await 414 432 .map_err(|e| format!("Failed to get old MST blocks for path: {:?}", e))?; 433 433 + let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid) 434 434 + .chain( 435 435 + old_mst_blocks 436 436 + .keys() 437 437 + .filter(|cid| !new_mst_blocks.contains_key(*cid)) 438 438 + .copied(), 439 439 + ) 440 440 + .collect(); 441 441 + let mut relevant_blocks = new_mst_blocks; 442 442 + relevant_blocks.extend(old_mst_blocks); 415 443 relevant_blocks.insert(record_cid, bytes::Bytes::from(record_bytes)); 416 416 - let mut written_cids = tracking_store.get_all_relevant_cids(); 417 417 - for cid in relevant_blocks.keys() { 418 418 - if !written_cids.contains(cid) { 419 419 - written_cids.push(*cid); 420 420 - } 421 421 - } 444 444 + let written_cids: Vec<Cid> = tracking_store 445 445 + .get_all_relevant_cids() 446 446 + .into_iter() 447 447 + .chain(relevant_blocks.keys().copied()) 448 448 + .collect::<std::collections::HashSet<_>>() 449 449 + .into_iter() 450 450 + .collect(); 422 451 let written_cids_str: Vec<String> = written_cids.iter().map(|c| c.to_string()).collect(); 423 452 let blob_cids = extract_blob_cids(record); 424 453 let result = commit_and_log( ··· 432 461 ops: vec![op], 433 462 blocks_cids: &written_cids_str, 434 463 blobs: &blob_cids, 464 464 + obsolete_cids, 435 465 }, 436 466 ) 437 467 .await?;

+47 -26

src/api/repo/record/write.rs

··· 266 266 rkey: rkey.to_string(), 267 267 cid: record_cid, 268 268 }; 269 269 - let mut relevant_blocks = std::collections::BTreeMap::new(); 269 269 + let mut new_mst_blocks = std::collections::BTreeMap::new(); 270 270 + let mut old_mst_blocks = std::collections::BTreeMap::new(); 270 271 if new_mst 271 271 - .blocks_for_path(&key, &mut relevant_blocks) 272 272 + .blocks_for_path(&key, &mut new_mst_blocks) 272 273 .await 273 274 .is_err() 274 275 { ··· 276 277 .into_response(); 277 278 } 278 279 if mst 279 279 - .blocks_for_path(&key, &mut relevant_blocks) 280 280 + .blocks_for_path(&key, &mut old_mst_blocks) 280 281 .await 281 282 .is_err() 282 283 { 283 284 return ApiError::InternalError(Some("Failed to get old MST blocks for path".into())) 284 285 .into_response(); 285 286 } 287 287 + let mut relevant_blocks = new_mst_blocks.clone(); 288 288 + relevant_blocks.extend(old_mst_blocks.iter().map(|(k, v)| (*k, v.clone()))); 286 289 relevant_blocks.insert(record_cid, bytes::Bytes::from(record_bytes)); 287 287 - let mut written_cids = tracking_store.get_all_relevant_cids(); 288 288 - for cid in relevant_blocks.keys() { 289 289 - if !written_cids.contains(cid) { 290 290 - written_cids.push(*cid); 291 291 - } 292 292 - } 293 293 - let written_cids_str = written_cids 294 294 - .iter() 295 295 - .map(|c| c.to_string()) 296 296 - .collect::<Vec<_>>(); 290 290 + let written_cids: Vec<Cid> = tracking_store 291 291 + .get_all_relevant_cids() 292 292 + .into_iter() 293 293 + .chain(relevant_blocks.keys().copied()) 294 294 + .collect::<std::collections::HashSet<_>>() 295 295 + .into_iter() 296 296 + .collect(); 297 297 + let written_cids_str: Vec<String> = written_cids.iter().map(|c| c.to_string()).collect(); 297 298 let blob_cids = extract_blob_cids(&input.record); 299 299 + let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid) 300 300 + .chain( 301 301 + old_mst_blocks 302 302 + .keys() 303 303 + .filter(|cid| !new_mst_blocks.contains_key(*cid)) 304 304 + .copied(), 305 305 + ) 306 306 + .collect(); 298 307 let commit_result = match commit_and_log( 299 308 &state, 300 309 CommitParams { ··· 306 315 ops: vec![op], 307 316 blocks_cids: &written_cids_str, 308 317 blobs: &blob_cids, 318 318 + obsolete_cids, 309 319 }, 310 320 ) 311 321 .await ··· 512 522 cid: record_cid, 513 523 } 514 524 }; 515 515 - let mut relevant_blocks = std::collections::BTreeMap::new(); 525 525 + let mut new_mst_blocks = std::collections::BTreeMap::new(); 526 526 + let mut old_mst_blocks = std::collections::BTreeMap::new(); 516 527 if new_mst 517 517 - .blocks_for_path(&key, &mut relevant_blocks) 528 528 + .blocks_for_path(&key, &mut new_mst_blocks) 518 529 .await 519 530 .is_err() 520 531 { ··· 522 533 .into_response(); 523 534 } 524 535 if mst 525 525 - .blocks_for_path(&key, &mut relevant_blocks) 536 536 + .blocks_for_path(&key, &mut old_mst_blocks) 526 537 .await 527 538 .is_err() 528 539 { 529 540 return ApiError::InternalError(Some("Failed to get old MST blocks for path".into())) 530 541 .into_response(); 531 542 } 543 543 + let mut relevant_blocks = new_mst_blocks.clone(); 544 544 + relevant_blocks.extend(old_mst_blocks.iter().map(|(k, v)| (*k, v.clone()))); 532 545 relevant_blocks.insert(record_cid, bytes::Bytes::from(record_bytes)); 533 533 - let mut written_cids = tracking_store.get_all_relevant_cids(); 534 534 - for cid in relevant_blocks.keys() { 535 535 - if !written_cids.contains(cid) { 536 536 - written_cids.push(*cid); 537 537 - } 538 538 - } 539 539 - let written_cids_str = written_cids 540 540 - .iter() 541 541 - .map(|c| c.to_string()) 542 542 - .collect::<Vec<_>>(); 546 546 + let written_cids: Vec<Cid> = tracking_store 547 547 + .get_all_relevant_cids() 548 548 + .into_iter() 549 549 + .chain(relevant_blocks.keys().copied()) 550 550 + .collect::<std::collections::HashSet<_>>() 551 551 + .into_iter() 552 552 + .collect(); 553 553 + let written_cids_str: Vec<String> = written_cids.iter().map(|c| c.to_string()).collect(); 543 554 let is_update = existing_cid.is_some(); 544 555 let blob_cids = extract_blob_cids(&input.record); 556 556 + let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid) 557 557 + .chain( 558 558 + old_mst_blocks 559 559 + .keys() 560 560 + .filter(|cid| !new_mst_blocks.contains_key(*cid)) 561 561 + .copied(), 562 562 + ) 563 563 + .chain(existing_cid) 564 564 + .collect(); 545 565 let commit_result = match commit_and_log( 546 566 &state, 547 567 CommitParams { ··· 553 573 ops: vec![op], 554 574 blocks_cids: &written_cids_str, 555 575 blobs: &blob_cids, 576 576 + obsolete_cids, 556 577 }, 557 578 ) 558 579 .await

+159 -101

src/scheduled.rs

··· 226 226 } 227 227 }; 228 228 229 229 - let mut block_cids: Vec<Vec<u8>> = Vec::new(); 230 230 - let mut to_visit = vec![root_cid]; 231 231 - let mut visited = std::collections::HashSet::new(); 229 229 + match collect_current_repo_blocks(&block_store, &root_cid).await { 230 230 + Ok(block_cids) => { 231 231 + if block_cids.is_empty() { 232 232 + failed += 1; 233 233 + continue; 234 234 + } 232 235 233 233 - while let Some(cid) = to_visit.pop() { 234 234 - if visited.contains(&cid) { 235 235 - continue; 236 236 + if let Err(e) = sqlx::query!( 237 237 + r#" 238 238 + INSERT INTO user_blocks (user_id, block_cid) 239 239 + SELECT $1, block_cid FROM UNNEST($2::bytea[]) AS t(block_cid) 240 240 + ON CONFLICT (user_id, block_cid) DO NOTHING 241 241 + "#, 242 242 + user.user_id, 243 243 + &block_cids 244 244 + ) 245 245 + .execute(db) 246 246 + .await 247 247 + { 248 248 + warn!(user_id = %user.user_id, error = %e, "Failed to backfill user_blocks"); 249 249 + failed += 1; 250 250 + } else { 251 251 + info!(user_id = %user.user_id, block_count = block_cids.len(), "Backfilled user_blocks"); 252 252 + success += 1; 253 253 + } 236 254 } 237 237 - visited.insert(cid); 238 238 - block_cids.push(cid.to_bytes()); 239 239 - 240 240 - let block = match block_store.get(&cid).await { 241 241 - Ok(Some(b)) => b, 242 242 - _ => continue, 243 243 - }; 244 244 - 245 245 - if let Ok(commit) = Commit::from_cbor(&block) { 246 246 - to_visit.push(commit.data); 247 247 - if let Some(prev) = commit.prev { 248 248 - to_visit.push(prev); 249 249 - } 250 250 - } else if let Ok(Ipld::Map(ref obj)) = serde_ipld_dagcbor::from_slice::<Ipld>(&block) { 251 251 - if let Some(Ipld::Link(left_cid)) = obj.get("l") { 252 252 - to_visit.push(*left_cid); 253 253 - } 254 254 - if let Some(Ipld::List(entries)) = obj.get("e") { 255 255 - for entry in entries { 256 256 - if let Ipld::Map(entry_obj) = entry { 257 257 - if let Some(Ipld::Link(tree_cid)) = entry_obj.get("t") { 258 258 - to_visit.push(*tree_cid); 259 259 - } 260 260 - if let Some(Ipld::Link(val_cid)) = entry_obj.get("v") { 261 261 - to_visit.push(*val_cid); 262 262 - } 263 263 - } 264 264 - } 265 265 - } 255 255 + Err(e) => { 256 256 + warn!(user_id = %user.user_id, error = %e, "Failed to collect repo blocks for backfill"); 257 257 + failed += 1; 266 258 } 267 259 } 260 260 + } 268 261 269 269 - if block_cids.is_empty() { 270 270 - failed += 1; 262 262 + info!(success, failed, "Completed user_blocks backfill"); 263 263 + } 264 264 + 265 265 + pub async fn collect_current_repo_blocks( 266 266 + block_store: &PostgresBlockStore, 267 267 + head_cid: &Cid, 268 268 + ) -> Result<Vec<Vec<u8>>, String> { 269 269 + let mut block_cids: Vec<Vec<u8>> = Vec::new(); 270 270 + let mut to_visit = vec![*head_cid]; 271 271 + let mut visited = std::collections::HashSet::new(); 272 272 + 273 273 + while let Some(cid) = to_visit.pop() { 274 274 + if visited.contains(&cid) { 271 275 continue; 272 276 } 277 277 + visited.insert(cid); 278 278 + block_cids.push(cid.to_bytes()); 273 279 274 274 - if let Err(e) = sqlx::query!( 275 275 - r#" 276 276 - INSERT INTO user_blocks (user_id, block_cid) 277 277 - SELECT $1, block_cid FROM UNNEST($2::bytea[]) AS t(block_cid) 278 278 - ON CONFLICT (user_id, block_cid) DO NOTHING 279 279 - "#, 280 280 - user.user_id, 281 281 - &block_cids 282 282 - ) 283 283 - .execute(db) 284 284 - .await 285 285 - { 286 286 - warn!(user_id = %user.user_id, error = %e, "Failed to backfill user_blocks"); 287 287 - failed += 1; 288 288 - } else { 289 289 - info!(user_id = %user.user_id, block_count = block_cids.len(), "Backfilled user_blocks"); 290 290 - success += 1; 280 280 + let block = match block_store.get(&cid).await { 281 281 + Ok(Some(b)) => b, 282 282 + Ok(None) => continue, 283 283 + Err(e) => return Err(format!("Failed to get block {}: {:?}", cid, e)), 284 284 + }; 285 285 + 286 286 + if let Ok(commit) = Commit::from_cbor(&block) { 287 287 + to_visit.push(commit.data); 288 288 + } else if let Ok(Ipld::Map(ref obj)) = serde_ipld_dagcbor::from_slice::<Ipld>(&block) { 289 289 + if let Some(Ipld::Link(left_cid)) = obj.get("l") { 290 290 + to_visit.push(*left_cid); 291 291 + } 292 292 + if let Some(Ipld::List(entries)) = obj.get("e") { 293 293 + to_visit.extend( 294 294 + entries 295 295 + .iter() 296 296 + .filter_map(|entry| match entry { 297 297 + Ipld::Map(entry_obj) => Some(entry_obj), 298 298 + _ => None, 299 299 + }) 300 300 + .flat_map(|entry_obj| { 301 301 + [entry_obj.get("t"), entry_obj.get("v")] 302 302 + .into_iter() 303 303 + .flatten() 304 304 + .filter_map(|v| match v { 305 305 + Ipld::Link(cid) => Some(*cid), 306 306 + _ => None, 307 307 + }) 308 308 + }), 309 309 + ); 310 310 + } 291 311 } 292 312 } 293 313 294 294 - info!(success, failed, "Completed user_blocks backfill"); 314 314 + Ok(block_cids) 295 315 } 296 316 297 317 pub async fn backfill_record_blobs(db: &PgPool, block_store: PostgresBlockStore) { ··· 664 684 } 665 685 }; 666 686 667 667 - let car_result = generate_full_backup(block_store, &head_cid).await; 687 687 + let car_result = generate_full_backup(db, block_store, user.user_id, &head_cid).await; 668 688 let car_bytes = match car_result { 669 689 Ok(bytes) => bytes, 670 690 Err(e) => { ··· 736 756 head_cid: &Cid, 737 757 ) -> Result<Vec<u8>, String> { 738 758 use jacquard_repo::storage::BlockStore; 739 739 - use std::io::Write; 740 759 741 741 - let mut car_bytes = 760 760 + let block_cids_bytes = collect_current_repo_blocks(block_store, head_cid).await?; 761 761 + let block_cids: Vec<Cid> = block_cids_bytes 762 762 + .iter() 763 763 + .filter_map(|b| Cid::try_from(b.as_slice()).ok()) 764 764 + .collect(); 765 765 + 766 766 + let car_bytes = 742 767 encode_car_header(head_cid).map_err(|e| format!("Failed to encode CAR header: {}", e))?; 743 768 744 744 - let mut stack = vec![*head_cid]; 745 745 - let mut visited = std::collections::HashSet::new(); 769 769 + let blocks = block_store 770 770 + .get_many(&block_cids) 771 771 + .await 772 772 + .map_err(|e| format!("Failed to fetch blocks: {:?}", e))?; 746 773 747 747 - while let Some(cid) = stack.pop() { 748 748 - if visited.contains(&cid) { 749 749 - continue; 750 750 - } 751 751 - visited.insert(cid); 774 774 + let car_bytes = block_cids 775 775 + .iter() 776 776 + .zip(blocks.iter()) 777 777 + .filter_map(|(cid, block_opt)| block_opt.as_ref().map(|block| (cid, block))) 778 778 + .fold(car_bytes, |mut acc, (cid, block)| { 779 779 + acc.extend(encode_car_block(cid, block)); 780 780 + acc 781 781 + }); 752 782 753 753 - if let Ok(Some(block)) = block_store.get(&cid).await { 754 754 - let cid_bytes = cid.to_bytes(); 755 755 - let total_len = cid_bytes.len() + block.len(); 756 756 - let mut writer = Vec::new(); 757 757 - crate::sync::car::write_varint(&mut writer, total_len as u64) 758 758 - .expect("Writing to Vec<u8> should never fail"); 759 759 - writer 760 760 - .write_all(&cid_bytes) 761 761 - .expect("Writing to Vec<u8> should never fail"); 762 762 - writer 763 763 - .write_all(&block) 764 764 - .expect("Writing to Vec<u8> should never fail"); 765 765 - car_bytes.extend_from_slice(&writer); 783 783 + Ok(car_bytes) 784 784 + } 785 785 + 786 786 + fn encode_car_block(cid: &Cid, block: &[u8]) -> Vec<u8> { 787 787 + use std::io::Write; 788 788 + let cid_bytes = cid.to_bytes(); 789 789 + let total_len = cid_bytes.len() + block.len(); 790 790 + let mut writer = Vec::new(); 791 791 + crate::sync::car::write_varint(&mut writer, total_len as u64) 792 792 + .expect("Writing to Vec<u8> should never fail"); 793 793 + writer 794 794 + .write_all(&cid_bytes) 795 795 + .expect("Writing to Vec<u8> should never fail"); 796 796 + writer 797 797 + .write_all(block) 798 798 + .expect("Writing to Vec<u8> should never fail"); 799 799 + writer 800 800 + } 766 801 767 767 - if let Ok(value) = serde_ipld_dagcbor::from_slice::<Ipld>(&block) { 768 768 - extract_links(&value, &mut stack); 769 769 - } 802 802 + pub async fn generate_repo_car_from_user_blocks( 803 803 + db: &PgPool, 804 804 + block_store: &PostgresBlockStore, 805 805 + user_id: uuid::Uuid, 806 806 + head_cid: &Cid, 807 807 + ) -> Result<Vec<u8>, String> { 808 808 + use jacquard_repo::storage::BlockStore; 809 809 + 810 810 + let block_cid_bytes: Vec<Vec<u8>> = sqlx::query_scalar!( 811 811 + "SELECT block_cid FROM user_blocks WHERE user_id = $1", 812 812 + user_id 813 813 + ) 814 814 + .fetch_all(db) 815 815 + .await 816 816 + .map_err(|e| format!("Failed to fetch user_blocks: {}", e))?; 817 817 + 818 818 + if block_cid_bytes.is_empty() { 819 819 + let cids = collect_current_repo_blocks(block_store, head_cid).await?; 820 820 + if cids.is_empty() { 821 821 + return Err("No blocks found for repo".to_string()); 770 822 } 823 823 + return generate_repo_car(block_store, head_cid).await; 771 824 } 772 825 826 826 + let block_cids: Vec<Cid> = block_cid_bytes 827 827 + .iter() 828 828 + .filter_map(|bytes| Cid::try_from(bytes.as_slice()).ok()) 829 829 + .collect(); 830 830 + 831 831 + let car_bytes = 832 832 + encode_car_header(head_cid).map_err(|e| format!("Failed to encode CAR header: {}", e))?; 833 833 + 834 834 + let blocks = block_store 835 835 + .get_many(&block_cids) 836 836 + .await 837 837 + .map_err(|e| format!("Failed to fetch blocks: {:?}", e))?; 838 838 + 839 839 + let car_bytes = block_cids 840 840 + .iter() 841 841 + .zip(blocks.iter()) 842 842 + .filter_map(|(cid, block_opt)| block_opt.as_ref().map(|block| (cid, block))) 843 843 + .fold(car_bytes, |mut acc, (cid, block)| { 844 844 + acc.extend(encode_car_block(cid, block)); 845 845 + acc 846 846 + }); 847 847 + 773 848 Ok(car_bytes) 774 849 } 775 850 776 851 pub async fn generate_full_backup( 852 852 + db: &PgPool, 777 853 block_store: &PostgresBlockStore, 854 854 + user_id: uuid::Uuid, 778 855 head_cid: &Cid, 779 856 ) -> Result<Vec<u8>, String> { 780 780 - generate_repo_car(block_store, head_cid).await 781 781 - } 782 782 - 783 783 - fn extract_links(value: &Ipld, stack: &mut Vec<Cid>) { 784 784 - match value { 785 785 - Ipld::Link(cid) => { 786 786 - stack.push(*cid); 787 787 - } 788 788 - Ipld::Map(map) => { 789 789 - for v in map.values() { 790 790 - extract_links(v, stack); 791 791 - } 792 792 - } 793 793 - Ipld::List(arr) => { 794 794 - for v in arr { 795 795 - extract_links(v, stack); 796 796 - } 797 797 - } 798 798 - _ => {} 799 799 - } 857 857 + generate_repo_car_from_user_blocks(db, block_store, user_id, head_cid).await 800 858 } 801 859 802 860 pub fn count_car_blocks(car_bytes: &[u8]) -> i32 {

+12 -55

src/sync/repo.rs

··· 1 1 use crate::api::error::ApiError; 2 2 + use crate::scheduled::generate_repo_car_from_user_blocks; 2 3 use crate::state::AppState; 3 4 use crate::sync::car::encode_car_header; 4 5 use crate::sync::util::assert_repo_availability; ··· 8 9 response::{IntoResponse, Response}, 9 10 }; 10 11 use cid::Cid; 11 11 - use ipld_core::ipld::Ipld; 12 12 use jacquard_repo::storage::BlockStore; 13 13 use serde::Deserialize; 14 14 use std::io::Write; 15 15 use std::str::FromStr; 16 16 use tracing::error; 17 17 - 18 18 - const MAX_REPO_BLOCKS_TRAVERSAL: usize = 20_000; 19 17 20 18 fn parse_get_blocks_query(query_string: &str) -> Result<(String, Vec<String>), String> { 21 19 let did = crate::util::parse_repeated_query_param(Some(query_string), "did") ··· 138 136 return get_repo_since(&state, &query.did, &head_cid, since).await; 139 137 } 140 138 141 141 - let mut car_bytes = match encode_car_header(&head_cid) { 142 142 - Ok(h) => h, 139 139 + let car_bytes = match generate_repo_car_from_user_blocks( 140 140 + &state.db, 141 141 + &state.block_store, 142 142 + account.user_id, 143 143 + &head_cid, 144 144 + ) 145 145 + .await 146 146 + { 147 147 + Ok(bytes) => bytes, 143 148 Err(e) => { 144 144 - error!("Failed to encode CAR header: {}", e); 149 149 + error!("Failed to generate repo CAR: {}", e); 145 150 return ApiError::InternalError(None).into_response(); 146 151 } 147 152 }; 148 148 - let mut stack = vec![head_cid]; 149 149 - let mut visited = std::collections::HashSet::new(); 150 150 - let mut remaining = MAX_REPO_BLOCKS_TRAVERSAL; 151 151 - while let Some(cid) = stack.pop() { 152 152 - if visited.contains(&cid) { 153 153 - continue; 154 154 - } 155 155 - visited.insert(cid); 156 156 - if remaining == 0 { 157 157 - break; 158 158 - } 159 159 - remaining -= 1; 160 160 - if let Ok(Some(block)) = state.block_store.get(&cid).await { 161 161 - let cid_bytes = cid.to_bytes(); 162 162 - let total_len = cid_bytes.len() + block.len(); 163 163 - let mut writer = Vec::new(); 164 164 - crate::sync::car::write_varint(&mut writer, total_len as u64) 165 165 - .expect("Writing to Vec<u8> should never fail"); 166 166 - writer 167 167 - .write_all(&cid_bytes) 168 168 - .expect("Writing to Vec<u8> should never fail"); 169 169 - writer 170 170 - .write_all(&block) 171 171 - .expect("Writing to Vec<u8> should never fail"); 172 172 - car_bytes.extend_from_slice(&writer); 173 173 - if let Ok(value) = serde_ipld_dagcbor::from_slice::<Ipld>(&block) { 174 174 - extract_links_ipld(&value, &mut stack); 175 175 - } 176 176 - } 177 177 - } 153 153 + 178 154 ( 179 155 StatusCode::OK, 180 156 [(axum::http::header::CONTENT_TYPE, "application/vnd.ipld.car")], ··· 273 249 car_bytes, 274 250 ) 275 251 .into_response() 276 276 - } 277 277 - 278 278 - fn extract_links_ipld(value: &Ipld, stack: &mut Vec<Cid>) { 279 279 - match value { 280 280 - Ipld::Link(cid) => { 281 281 - stack.push(*cid); 282 282 - } 283 283 - Ipld::Map(map) => { 284 284 - for v in map.values() { 285 285 - extract_links_ipld(v, stack); 286 286 - } 287 287 - } 288 288 - Ipld::List(arr) => { 289 289 - for v in arr { 290 290 - extract_links_ipld(v, stack); 291 291 - } 292 292 - } 293 293 - _ => {} 294 294 - } 295 252 } 296 253 297 254 #[derive(Deserialize)]

+8 -2

tests/account_lifecycle.rs

··· 93 93 let body3: Value = status3.json().await.unwrap(); 94 94 let after_delete_blocks = body3["repoBlocks"].as_i64().unwrap(); 95 95 assert!( 96 96 - after_delete_blocks >= after_create_blocks, 97 97 - "Block count should not decrease after deleting a record (was {}, now {})", 96 96 + after_delete_blocks <= after_create_blocks, 97 97 + "Block count should decrease or stay same after deleting a record (was {}, now {})", 98 98 after_create_blocks, 99 99 + after_delete_blocks 100 100 + ); 101 101 + assert!( 102 102 + after_delete_blocks >= initial_blocks, 103 103 + "Block count after delete should be at least initial count (initial {}, now {})", 104 104 + initial_blocks, 99 105 after_delete_blocks 100 106 ); 101 107 }

+3 -3

tests/delete_account.rs

··· 174 174 .send() 175 175 .await 176 176 .expect("Failed to send delete request"); 177 177 - assert_eq!(delete_res.status(), StatusCode::BAD_REQUEST); 177 177 + assert_eq!(delete_res.status(), StatusCode::UNAUTHORIZED); 178 178 let body: Value = delete_res.json().await.unwrap(); 179 179 assert_eq!(body["error"], "InvalidToken"); 180 180 } ··· 228 228 .send() 229 229 .await 230 230 .expect("Failed to send delete request"); 231 231 - assert_eq!(delete_res.status(), StatusCode::BAD_REQUEST); 231 231 + assert_eq!(delete_res.status(), StatusCode::UNAUTHORIZED); 232 232 let body: Value = delete_res.json().await.unwrap(); 233 233 assert_eq!(body["error"], "ExpiredToken"); 234 234 } ··· 280 280 .send() 281 281 .await 282 282 .expect("Failed to send delete request"); 283 283 - assert_eq!(delete_res.status(), StatusCode::BAD_REQUEST); 283 283 + assert_eq!(delete_res.status(), StatusCode::UNAUTHORIZED); 284 284 let body: Value = delete_res.json().await.unwrap(); 285 285 assert_eq!(body["error"], "InvalidToken"); 286 286 }

+2 -2

tests/email_update.rs

··· 193 193 .send() 194 194 .await 195 195 .expect("Failed to attempt email update"); 196 196 - assert_eq!(res.status(), StatusCode::BAD_REQUEST); 196 196 + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); 197 197 let body: Value = res.json().await.expect("Invalid JSON"); 198 198 assert_eq!(body["error"], "InvalidToken"); 199 199 } ··· 390 390 .send() 391 391 .await 392 392 .expect("Failed to confirm email"); 393 393 - assert_eq!(res.status(), StatusCode::BAD_REQUEST); 393 393 + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); 394 394 let body: Value = res.json().await.expect("Invalid JSON"); 395 395 assert_eq!(body["error"], "InvalidToken"); 396 396 }

+16 -7

tests/oauth.rs

··· 261 261 .to_string(); 262 262 } 263 263 assert!( 264 264 - location.starts_with(redirect_uri), 265 265 - "Redirect to wrong URI: {}", 264 264 + location.contains("code="), 265 265 + "No code in redirect URI: {}", 266 266 location 267 267 ); 268 268 - assert!(location.contains("code="), "No code in redirect"); 269 268 assert!( 270 270 - location.contains(&format!("state={}", state)), 271 271 - "Wrong state" 269 269 + location.contains(&format!("state={}", state)) 270 270 + || location.contains(&format!("state%3D{}", state)), 271 271 + "Wrong state in redirect: {}", 272 272 + location 272 273 ); 273 274 let code = location 274 275 .split("code=") ··· 527 528 ); 528 529 let twofa_body: Value = twofa_res.json().await.unwrap(); 529 530 let final_location = twofa_body["redirect_uri"].as_str().unwrap(); 530 530 - assert!(final_location.starts_with(redirect_uri) && final_location.contains("code=")); 531 531 + assert!( 532 532 + final_location.contains("code="), 533 533 + "No code in redirect URI: {}", 534 534 + final_location 535 535 + ); 531 536 let auth_code = final_location 532 537 .split("code=") 533 538 .nth(1) ··· 805 810 ); 806 811 let twofa_body: Value = twofa_res.json().await.unwrap(); 807 812 let final_location = twofa_body["redirect_uri"].as_str().unwrap(); 808 808 - assert!(final_location.starts_with(redirect_uri) && final_location.contains("code=")); 813 813 + assert!( 814 814 + final_location.contains("code="), 815 815 + "No code in redirect URI: {}", 816 816 + final_location 817 817 + ); 809 818 let final_code = final_location 810 819 .split("code=") 811 820 .nth(1)

+2 -2

tests/password_reset.rs

··· 177 177 .send() 178 178 .await 179 179 .expect("Failed to reset password"); 180 180 - assert_eq!(res.status(), StatusCode::BAD_REQUEST); 180 180 + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); 181 181 let body: Value = res.json().await.expect("Invalid JSON"); 182 182 assert_eq!(body["error"], "InvalidToken"); 183 183 } ··· 241 241 .send() 242 242 .await 243 243 .expect("Failed to reset password"); 244 244 - assert_eq!(res.status(), StatusCode::BAD_REQUEST); 244 244 + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); 245 245 let body: Value = res.json().await.expect("Invalid JSON"); 246 246 assert_eq!(body["error"], "ExpiredToken"); 247 247 }

+1 -1

tests/plc_operations.rs

··· 76 76 .send() 77 77 .await 78 78 .unwrap(); 79 79 - assert_eq!(res.status(), StatusCode::BAD_REQUEST); 79 79 + assert_eq!(res.status(), StatusCode::UNAUTHORIZED); 80 80 let body: serde_json::Value = res.json().await.unwrap(); 81 81 assert!(body["error"] == "InvalidToken" || body["error"] == "ExpiredToken"); 82 82 }