+7 -3

src/backfill.rs

··· 13 13 dest: mpsc::Sender<ExportPage>, 14 14 source_workers: usize, 15 15 until: Option<Dt>, 16 16 - ) -> anyhow::Result<()> { 16 16 + ) -> anyhow::Result<&'static str> { 17 17 // queue up the week bundles that should be available 18 18 let weeks = Arc::new(Mutex::new( 19 19 until ··· 55 55 res.inspect_err(|e| log::error!("problem joining source workers: {e}"))? 56 56 .inspect_err(|e| log::error!("problem *from* source worker: {e}"))?; 57 57 } 58 58 - log::info!("finished fetching backfill in {:?}", t_step.elapsed()); 59 59 - Ok(()) 58 58 + log::info!( 59 59 + "finished fetching backfill in {:?}. senders remaining: {}", 60 60 + t_step.elapsed(), 61 61 + dest.strong_count() 62 62 + ); 63 63 + Ok("backfill") 60 64 }

+10 -4

src/bin/backfill.rs

··· 66 66 catch_up, 67 67 }: Args, 68 68 ) -> anyhow::Result<()> { 69 69 - let mut tasks = JoinSet::new(); 69 69 + let mut tasks = JoinSet::<anyhow::Result<&'static str>>::new(); 70 70 71 71 let (bulk_tx, bulk_out) = mpsc::channel(32); // bulk uses big pages 72 72 ··· 147 147 bulk_out, 148 148 found_last_tx, 149 149 )); 150 150 - tasks.spawn(pages_to_pg(db, full_out)); 150 150 + if catch_up { 151 151 + tasks.spawn(pages_to_pg(db, full_out)); 152 152 + } 151 153 } else { 152 154 tasks.spawn(pages_to_stdout(bulk_out, found_last_tx)); 153 153 - tasks.spawn(pages_to_stdout(full_out, None)); 155 155 + if catch_up { 156 156 + tasks.spawn(pages_to_stdout(full_out, None)); 157 157 + } 154 158 } 155 159 } 156 160 ··· 168 172 log::error!("a joinset task completed with error: {e}"); 169 173 return Err(e); 170 174 } 171 171 - _ => {} 175 175 + Ok(Ok(name)) => { 176 176 + log::trace!("a task completed: {name:?}. {} left", tasks.len()); 177 177 + } 172 178 } 173 179 } 174 180

+5 -5

src/lib.rs

··· 83 83 pub async fn full_pages( 84 84 mut rx: mpsc::Receiver<ExportPage>, 85 85 tx: mpsc::Sender<ExportPage>, 86 86 - ) -> anyhow::Result<()> { 86 86 + ) -> anyhow::Result<&'static str> { 87 87 while let Some(page) = rx.recv().await { 88 88 let n = page.ops.len(); 89 89 if n < 900 { 90 90 let last_age = page.ops.last().map(|op| chrono::Utc::now() - op.created_at); 91 91 let Some(age) = last_age else { 92 92 log::info!("full_pages done, empty final page"); 93 93 - return Ok(()); 93 93 + return Ok("full pages (hmm)"); 94 94 }; 95 95 if age <= chrono::TimeDelta::hours(6) { 96 96 log::info!("full_pages done, final page of {n} ops"); 97 97 } else { 98 98 log::warn!("full_pages finished with small page of {n} ops, but it's {age} old"); 99 99 } 100 100 - return Ok(()); 100 100 + return Ok("full pages (cool)"); 101 101 } 102 102 log::trace!("full_pages: continuing with page of {n} ops"); 103 103 tx.send(page).await?; ··· 110 110 pub async fn pages_to_stdout( 111 111 mut rx: mpsc::Receiver<ExportPage>, 112 112 notify_last_at: Option<oneshot::Sender<Option<Dt>>>, 113 113 - ) -> anyhow::Result<()> { 113 113 + ) -> anyhow::Result<&'static str> { 114 114 let mut last_at = None; 115 115 while let Some(page) = rx.recv().await { 116 116 for op in &page.ops { ··· 128 128 log::error!("receiver for last_at dropped, can't notify"); 129 129 }; 130 130 } 131 131 - Ok(()) 131 131 + Ok("pages_to_stdout") 132 132 } 133 133 134 134 pub fn logo(name: &str) -> String {

+8 -4

src/plc_pg.rs

··· 133 133 } 134 134 } 135 135 136 136 - pub async fn pages_to_pg(db: Db, mut pages: mpsc::Receiver<ExportPage>) -> anyhow::Result<()> { 136 136 + pub async fn pages_to_pg( 137 137 + db: Db, 138 138 + mut pages: mpsc::Receiver<ExportPage>, 139 139 + ) -> anyhow::Result<&'static str> { 137 140 let mut client = db.connect().await?; 138 141 139 142 let ops_stmt = client ··· 176 179 "no more pages. inserted {ops_inserted} ops and {dids_inserted} dids in {:?}", 177 180 t0.elapsed() 178 181 ); 179 179 - Ok(()) 182 182 + Ok("pages_to_pg") 180 183 } 181 184 182 185 /// Dump rows into an empty operations table quickly ··· 197 200 reset: bool, 198 201 mut pages: mpsc::Receiver<ExportPage>, 199 202 notify_last_at: Option<oneshot::Sender<Option<Dt>>>, 200 200 - ) -> anyhow::Result<()> { 203 203 + ) -> anyhow::Result<&'static str> { 201 204 let mut client = db.connect().await?; 202 205 203 206 let t0 = Instant::now(); ··· 272 275 last_at = last_at.filter(|&l| l >= s.last_at).or(Some(s.last_at)); 273 276 } 274 277 } 278 278 + log::debug!("finished receiving bulk pages"); 275 279 276 280 if let Some(notify) = notify_last_at { 277 281 log::trace!("notifying last_at: {last_at:?}"); ··· 314 318 tx.commit().await?; 315 319 log::info!("total backfill time: {:?}", t0.elapsed()); 316 320 317 317 - Ok(()) 321 321 + Ok("backfill_to_pg") 318 322 }

+1 -1

src/poll.rs

··· 155 155 after: Option<Dt>, 156 156 base: Url, 157 157 dest: mpsc::Sender<ExportPage>, 158 158 - ) -> anyhow::Result<()> { 158 158 + ) -> anyhow::Result<&'static str> { 159 159 let mut tick = tokio::time::interval(UPSTREAM_REQUEST_INTERVAL); 160 160 let mut prev_last: Option<LastOp> = after.map(Into::into); 161 161 let mut boundary_state: Option<PageBoundaryState> = None;