ptr.pet
at protocol indexer with flexible filtering, xrpc queries, and a cursor-backed event stream, built on fjall
at-protocol
atproto
indexer
rust
fjall
[crawler] also log how many repos were crawled alongside processed
+17
-7
src/crawler/mod.rs
+17
-7
src/crawler/mod.rs
···
202
202
max_pending: usize,
203
203
resume_pending: usize,
204
204
count: Arc<AtomicUsize>,
205
+
crawled_count: Arc<AtomicUsize>,
205
206
}
206
207
207
208
impl Crawler {
···
230
231
max_pending,
231
232
resume_pending,
232
233
count: Arc::new(AtomicUsize::new(0)),
234
+
crawled_count: Arc::new(AtomicUsize::new(0)),
233
235
}
234
236
}
235
237
···
237
239
tokio::spawn({
238
240
use std::time::Instant;
239
241
let count = self.count.clone();
242
+
let crawled_count = self.crawled_count.clone();
240
243
let mut last_time = Instant::now();
241
244
let mut interval = tokio::time::interval(Duration::from_secs(60));
242
245
async move {
243
246
loop {
244
247
interval.tick().await;
245
-
let delta = count.swap(0, Ordering::Relaxed);
246
-
if delta == 0 {
247
-
debug!("no repos processed in 60s");
248
+
let delta_processed = count.swap(0, Ordering::Relaxed);
249
+
let delta_crawled = crawled_count.swap(0, Ordering::Relaxed);
250
+
251
+
if delta_processed == 0 && delta_crawled == 0 {
252
+
debug!("no repos crawled or processed in 60s");
248
253
continue;
249
254
}
255
+
250
256
let elapsed = last_time.elapsed().as_secs_f64();
251
-
let rate = (elapsed > 0.0)
252
-
.then(|| delta as f64 / elapsed)
253
-
.unwrap_or(0.0);
254
-
info!(rate, delta, elapsed, "crawler progress");
257
+
info!(
258
+
processed = delta_processed,
259
+
crawled = delta_crawled,
260
+
elapsed,
261
+
"crawler progress"
262
+
);
255
263
last_time = Instant::now();
256
264
}
257
265
}
···
378
386
}
379
387
380
388
debug!(count = output.repos.len(), "fetched repos");
389
+
self.crawled_count
390
+
.fetch_add(output.repos.len(), Ordering::Relaxed);
381
391
382
392
let mut batch = db.inner.batch();
383
393
let mut to_queue = Vec::new();