vitorpy.com
Noreposts Feed
Add automatic backfill of follows on first feed request
- Created backfill module to fetch existing follows from Bluesky API
- Automatically backfill follows when user has none in database
- Uses public.api.bsky.app to fetch follow list
- Runs backfill in background task to not block feed response
- Make database pool public for backfill access
+83
-1
+59
src/backfill.rs
+59
src/backfill.rs
···
1
+
use anyhow::Result;
2
+
use std::sync::Arc;
3
+
use tracing::{info, warn};
4
+
5
+
use crate::{database::Database, types::Follow};
6
+
7
+
pub async fn backfill_follows(db: Arc<Database>, user_did: &str) -> Result<()> {
8
+
info!("Starting backfill of follows for {}", user_did);
9
+
10
+
let client = reqwest::Client::new();
11
+
let mut cursor: Option<String> = None;
12
+
let mut total_follows = 0;
13
+
14
+
loop {
15
+
let mut url = format!("https://public.api.bsky.app/xrpc/app.bsky.graph.getFollows?actor={}&limit=100", user_did);
16
+
if let Some(ref c) = cursor {
17
+
url.push_str(&format!("&cursor={}", c));
18
+
}
19
+
20
+
let response: serde_json::Value = client.get(&url)
21
+
.send()
22
+
.await?
23
+
.json()
24
+
.await?;
25
+
26
+
let follows = response["follows"].as_array();
27
+
if follows.is_none() {
28
+
break;
29
+
}
30
+
31
+
for follow in follows.unwrap() {
32
+
let target_did = follow["did"].as_str().unwrap_or("");
33
+
if target_did.is_empty() {
34
+
continue;
35
+
}
36
+
37
+
let follow_record = Follow {
38
+
uri: format!("at://{}/app.bsky.graph.follow/{}", user_did, uuid::Uuid::new_v4()),
39
+
follower_did: user_did.to_string(),
40
+
target_did: target_did.to_string(),
41
+
created_at: chrono::Utc::now(),
42
+
indexed_at: chrono::Utc::now(),
43
+
};
44
+
45
+
match db.insert_follow(&follow_record).await {
46
+
Ok(_) => total_follows += 1,
47
+
Err(e) => warn!("Failed to insert follow {}: {}", target_did, e),
48
+
}
49
+
}
50
+
51
+
cursor = response["cursor"].as_str().map(|s| s.to_string());
52
+
if cursor.is_none() {
53
+
break;
54
+
}
55
+
}
56
+
57
+
info!("Backfilled {} follows for {}", total_follows, user_did);
58
+
Ok(())
59
+
}
+1
-1
src/database.rs
+1
-1
src/database.rs
+23
src/main.rs
+23
src/main.rs
···
7
7
Router,
8
8
};
9
9
use clap::Parser;
10
+
use sqlx::Row;
10
11
use std::sync::Arc;
11
12
use tokio::net::TcpListener;
12
13
use tower_http::cors::CorsLayer;
13
14
use tracing::{info, warn};
14
15
15
16
mod auth;
17
+
mod backfill;
16
18
mod database;
17
19
mod feed_algorithm;
18
20
mod jetstream_consumer;
···
185
187
).into_response();
186
188
}
187
189
};
190
+
191
+
// Check if user has any follows, if not, backfill them
192
+
let db_for_backfill = Arc::clone(&state.db);
193
+
let requester_did_clone = requester_did.clone();
194
+
tokio::spawn(async move {
195
+
// Check if we have any follows for this user
196
+
let has_follows = sqlx::query("SELECT COUNT(*) as count FROM follows WHERE follower_did = ?")
197
+
.bind(&requester_did_clone)
198
+
.fetch_one(&db_for_backfill.pool)
199
+
.await
200
+
.ok()
201
+
.and_then(|row| row.try_get::<i64, _>("count").ok())
202
+
.unwrap_or(0);
203
+
204
+
if has_follows == 0 {
205
+
info!("No follows found for {}, triggering backfill", requester_did_clone);
206
+
if let Err(e) = backfill::backfill_follows(db_for_backfill, &requester_did_clone).await {
207
+
warn!("Backfill failed for {}: {}", requester_did_clone, e);
208
+
}
209
+
}
210
+
});
188
211
189
212
let feed_algorithm = FollowingNoRepostsFeed::new(Arc::clone(&state.db));
190
213