ptr.pet
at protocol indexer with flexible filtering, xrpc queries, and a cursor-backed event stream, built on fjall
at-protocol
atproto
indexer
rust
fjall
[crawler] respect dids filters
+17
src/crawler/mod.rs
+17
src/crawler/mod.rs
···
1
use crate::db::{Db, keys, ser_repo_state};
2
use crate::state::AppState;
3
use crate::types::RepoState;
4
use jacquard::api::com_atproto::sync::list_repos::{ListRepos, ListReposOutput};
···
136
let mut batch = db.inner.batch();
137
let mut to_queue = Vec::new();
138
139
// 3. process repos
140
for repo in output.repos {
141
let did_key = keys::repo_key(&repo.did);
142
143
// check if known
144
if !Db::contains_key(db.repos.clone(), &did_key).await? {
···
1
use crate::db::{Db, keys, ser_repo_state};
2
+
use crate::filter::FilterMode;
3
use crate::state::AppState;
4
use crate::types::RepoState;
5
use jacquard::api::com_atproto::sync::list_repos::{ListRepos, ListReposOutput};
···
137
let mut batch = db.inner.batch();
138
let mut to_queue = Vec::new();
139
140
+
let filter = self.state.filter.load();
141
+
142
// 3. process repos
143
for repo in output.repos {
144
let did_key = keys::repo_key(&repo.did);
145
+
146
+
let excl_key =
147
+
crate::filter::filter_key(crate::filter::EXCLUDE_PREFIX, repo.did.as_str());
148
+
if db.filter.contains_key(&excl_key).into_diagnostic()? {
149
+
continue;
150
+
}
151
+
152
+
if filter.mode != FilterMode::Full {
153
+
let did_filter_key =
154
+
crate::filter::filter_key(crate::filter::DID_PREFIX, repo.did.as_str());
155
+
if !db.filter.contains_key(&did_filter_key).into_diagnostic()? {
156
+
continue;
157
+
}
158
+
}
159
160
// check if known
161
if !Db::contains_key(db.repos.clone(), &did_key).await? {