ptr.pet
at protocol indexer with flexible filtering, xrpc queries, and a cursor-backed event stream, built on fjall
at-protocol
atproto
indexer
rust
fjall
[crawler] add NoTlsRetry strategy, log retries, reduce max retries to 5
+69
-8
src/crawler/mod.rs
+69
-8
src/crawler/mod.rs
···
11
use rand::rngs::SmallRng;
12
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
13
use reqwest_retry::Jitter;
14
-
use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff};
15
-
use smol_str::SmolStr;
16
use std::sync::Arc;
17
use std::sync::atomic::{AtomicUsize, Ordering};
18
use std::time::Duration;
19
-
use tracing::{debug, error, info, trace};
20
use url::Url;
21
22
enum CrawlCheckResult {
···
26
Failed,
27
}
28
29
pub struct Crawler {
30
state: Arc<AppState>,
31
relay_host: Url,
···
44
) -> Self {
45
let retry_policy = ExponentialBackoff::builder()
46
.jitter(Jitter::Bounded)
47
-
.build_with_max_retries(8);
48
let reqwest_client = reqwest::Client::builder()
49
.user_agent(concat!(
50
env!("CARGO_PKG_NAME"),
···
56
.expect("that reqwest will build");
57
58
let http = ClientBuilder::new(reqwest_client)
59
-
.with(RetryTransientMiddleware::new_with_policy(retry_policy))
60
.build();
61
let http = Arc::new(http);
62
···
242
let did_key = keys::repo_key(did);
243
trace!("crawler found new repo: {did}");
244
245
-
let state = RepoState::backfilling(rng.next_u64());
246
batch.insert(&db.repos, &did_key, ser_repo_state(&state)?);
247
batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key);
248
to_queue.push(did.clone());
···
294
let resolver = self.state.resolver.clone();
295
let filter = filter.clone();
296
set.spawn(async move {
297
-
const MAX_RETRIES: u32 = 8;
298
let mut rng: SmallRng = rand::make_rng();
299
300
let pds_url = {
···
448
continue;
449
}
450
451
-
let state = RepoState::backfilling(rng.next_u64());
452
batch.insert(&db.repos, &did_key, ser_repo_state(&state)?);
453
batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key);
454
}
···
11
use rand::rngs::SmallRng;
12
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
13
use reqwest_retry::Jitter;
14
+
use reqwest_retry::{
15
+
RetryTransientMiddleware, Retryable, RetryableStrategy, default_on_request_failure,
16
+
default_on_request_success, policies::ExponentialBackoff,
17
+
};
18
+
use smol_str::{SmolStr, ToSmolStr};
19
+
use std::error::Error;
20
use std::sync::Arc;
21
use std::sync::atomic::{AtomicUsize, Ordering};
22
use std::time::Duration;
23
+
use tracing::{debug, error, info, trace, warn};
24
use url::Url;
25
26
enum CrawlCheckResult {
···
30
Failed,
31
}
32
33
+
struct NoTlsRetry;
34
+
35
+
impl RetryableStrategy for NoTlsRetry {
36
+
fn handle(
37
+
&self,
38
+
res: &Result<reqwest::Response, reqwest_middleware::Error>,
39
+
) -> Option<Retryable> {
40
+
match res {
41
+
Ok(success) => default_on_request_success(success),
42
+
Err(error) => {
43
+
if let reqwest_middleware::Error::Reqwest(e) = error {
44
+
if e.is_timeout() {
45
+
return Some(Retryable::Fatal);
46
+
}
47
+
let mut src = e.source();
48
+
while let Some(s) = src {
49
+
if let Some(io_err) = s.downcast_ref::<std::io::Error>() {
50
+
if is_tls_cert_error(io_err) {
51
+
return Some(Retryable::Fatal);
52
+
}
53
+
}
54
+
src = s.source();
55
+
}
56
+
}
57
+
let retryable = default_on_request_failure(error);
58
+
if retryable == Some(Retryable::Transient) {
59
+
if let reqwest_middleware::Error::Reqwest(e) = error {
60
+
let url = e.url().map(|u| u.as_str()).unwrap_or("unknown url");
61
+
let status = e
62
+
.status()
63
+
.map(|s| s.to_smolstr())
64
+
.unwrap_or_else(|| "unknown status".into());
65
+
warn!("retrying request {url}: {status}");
66
+
}
67
+
}
68
+
retryable
69
+
}
70
+
}
71
+
}
72
+
}
73
+
74
+
fn is_tls_cert_error(io_err: &std::io::Error) -> bool {
75
+
let Some(inner) = io_err.get_ref() else {
76
+
return false;
77
+
};
78
+
if let Some(rustls_err) = inner.downcast_ref::<rustls::Error>() {
79
+
return matches!(rustls_err, rustls::Error::InvalidCertificate(_));
80
+
}
81
+
if let Some(nested_io) = inner.downcast_ref::<std::io::Error>() {
82
+
return is_tls_cert_error(nested_io);
83
+
}
84
+
false
85
+
}
86
+
87
pub struct Crawler {
88
state: Arc<AppState>,
89
relay_host: Url,
···
102
) -> Self {
103
let retry_policy = ExponentialBackoff::builder()
104
.jitter(Jitter::Bounded)
105
+
.build_with_max_retries(5);
106
let reqwest_client = reqwest::Client::builder()
107
.user_agent(concat!(
108
env!("CARGO_PKG_NAME"),
···
114
.expect("that reqwest will build");
115
116
let http = ClientBuilder::new(reqwest_client)
117
+
.with(RetryTransientMiddleware::new_with_policy_and_strategy(
118
+
retry_policy,
119
+
NoTlsRetry,
120
+
))
121
.build();
122
let http = Arc::new(http);
123
···
303
let did_key = keys::repo_key(did);
304
trace!("crawler found new repo: {did}");
305
306
+
let state = RepoState::backfilling_untracked(rng.next_u64());
307
batch.insert(&db.repos, &did_key, ser_repo_state(&state)?);
308
batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key);
309
to_queue.push(did.clone());
···
355
let resolver = self.state.resolver.clone();
356
let filter = filter.clone();
357
set.spawn(async move {
358
+
const MAX_RETRIES: u32 = 5;
359
let mut rng: SmallRng = rand::make_rng();
360
361
let pds_url = {
···
509
continue;
510
}
511
512
+
let state = RepoState::backfilling_untracked(rng.next_u64());
513
batch.insert(&db.repos, &did_key, ser_repo_state(&state)?);
514
batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key);
515
}