ptr.pet
at protocol indexer with flexible filtering, xrpc queries, and a cursor-backed event stream, built on fjall
at-protocol
atproto
indexer
rust
fjall
[db,appwide] add poisoning checks on errors and crash early
+3
src/backfill/manager.rs
+3
src/backfill/manager.rs
···
66
66
.into_diagnostic()
67
67
.unwrap_or_else(|e| {
68
68
warn!("failed to scan errors: {e}");
69
+
Db::check_poisoned_report(&e);
69
70
Ok(Vec::new())
70
71
})
71
72
.unwrap_or_else(|e| {
72
73
warn!("failed to scan errors: {e}");
74
+
Db::check_poisoned_report(&e);
73
75
Vec::new()
74
76
});
75
77
···
86
88
// move back to pending
87
89
if let Err(e) = Db::insert(db.pending.clone(), key, Vec::new()).await {
88
90
warn!("failed to move {did} to pending: {e}");
91
+
Db::check_poisoned_report(&e);
89
92
continue;
90
93
}
91
94
+5
-1
src/backfill/mod.rs
+5
-1
src/backfill/mod.rs
···
66
66
did.clone(),
67
67
permit,
68
68
)
69
-
.inspect_err(move |e| error!("backfill process failed for {did}: {e}")),
69
+
.inspect_err(move |e| {
70
+
error!("backfill process failed for {did}: {e}");
71
+
Db::check_poisoned_report(e);
72
+
}),
70
73
);
71
74
}
72
75
}
···
405
408
406
409
if let Err(e) = ops::apply_commit(&state.db, &commit, true) {
407
410
error!("failed to apply buffered commit for {did}: {e}");
411
+
Db::check_poisoned_report(&e);
408
412
}
409
413
410
414
// delete from buffer
+16
-3
src/db/mod.rs
+16
-3
src/db/mod.rs
···
9
9
10
10
use std::sync::atomic::AtomicU64;
11
11
use tokio::sync::broadcast;
12
+
use tracing::error;
12
13
13
14
#[derive(Clone)]
14
15
pub struct Db {
···
98
99
}
99
100
100
101
pub fn persist(&self) -> Result<()> {
101
-
self.inner
102
-
.persist(PersistMode::SyncData)
103
-
.into_diagnostic()?;
102
+
self.inner.persist(PersistMode::SyncAll).into_diagnostic()?;
104
103
Ok(())
105
104
}
106
105
···
189
188
.await
190
189
.into_diagnostic()
191
190
.flatten()
191
+
}
192
+
193
+
pub fn check_poisoned(e: &fjall::Error) {
194
+
if matches!(e, fjall::Error::Poisoned) {
195
+
error!("!!! DATABASE POISONED !!! exiting");
196
+
std::process::exit(10);
197
+
}
198
+
}
199
+
200
+
pub fn check_poisoned_report(e: &miette::Report) {
201
+
let Some(err) = e.downcast_ref::<fjall::Error>() else {
202
+
return;
203
+
};
204
+
Self::check_poisoned(err);
192
205
}
193
206
}
+7
-2
src/ingest/mod.rs
+7
-2
src/ingest/mod.rs
···
57
57
let res = tokio::task::spawn_blocking(move || batch.commit()).await;
58
58
match res {
59
59
Ok(Ok(_)) => {}
60
-
Ok(Err(e)) => error!("failed to persist buffer batch: {}", e),
60
+
Ok(Err(e)) => {
61
+
Db::check_poisoned(&e);
62
+
error!("failed to persist buffer batch: {}", e)
63
+
}
61
64
Err(e) => error!("buffer worker join error: {}", e),
62
65
}
63
66
}
···
146
149
147
150
if let Err(e) = self.process_commit(&commit).await {
148
151
error!("failed to process commit {}: {e}", commit.seq);
152
+
Db::check_poisoned_report(&e);
149
153
// buffer for later inspection/retry
150
154
let _ = self.buffer_event(&commit).await;
151
155
}
···
241
245
.into_diagnostic()?;
242
246
243
247
if let Err(e) = res {
244
-
error!("failed to apply live commit for {}: {}", did_static, e);
248
+
error!("failed to apply live commit for {did_static}: {e}");
249
+
Db::check_poisoned_report(&e);
245
250
self.buffer_event(commit).await?;
246
251
} else {
247
252
debug!("synced event for {}, {} ops", did_static, commit.ops.len());
+9
-1
src/main.rs
+9
-1
src/main.rs
···
56
56
57
57
if let Err(e) = crate::backfill::manager::queue_pending_backfills(&state).await {
58
58
error!("failed to queue pending backfills: {e}");
59
+
Db::check_poisoned_report(&e);
59
60
}
60
61
61
62
tokio::spawn({
···
113
114
.await
114
115
{
115
116
error!("failed to save cursor: {e}");
117
+
Db::check_poisoned_report(&e);
116
118
}
117
119
118
120
let state = state.clone();
···
121
123
match res {
122
124
Ok(Err(e)) => {
123
125
error!("db persist failed: {e}");
126
+
Db::check_poisoned_report(&e);
124
127
}
125
128
Err(e) => {
126
129
error!("persistence task join failed: {e}");
···
139
142
let crawler = Crawler::new(state, crawler_host);
140
143
if let Err(e) = crawler.run().await {
141
144
error!("crawler died: {e}");
145
+
Db::check_poisoned_report(&e);
142
146
}
143
147
}
144
148
});
···
148
152
149
153
if let Err(e) = ingestor.run().await {
150
154
error!("ingestor died: {e}");
155
+
Db::check_poisoned_report(&e);
151
156
}
152
157
153
-
state.db.persist()?;
158
+
if let Err(e) = state.db.persist() {
159
+
Db::check_poisoned_report(&e);
160
+
return Err(e);
161
+
}
154
162
155
163
Ok(())
156
164
}