j4ck.xyz / tweets2bsky
A simple tool which lets you scrape twitter accounts and crosspost them to bluesky accounts! Comes with a CLI and a webapp for managing profiles! Works with images/videos/link embeds/threads.
fork atom

fix: avoid duplicate posts from recursive backfill

Jack G 60a3c3e3 a072cde6

+24 -4
unified split
+24 -4
src/index.ts
··· 1199 1199 bskyIdentifier: string, 1200 1200 tweets: Tweet[], 1201 1201 dryRun = false, 1202 + sharedProcessedMap?: ProcessedTweetsMap, 1202 1203 ): Promise<void> { 1203 1204 // Filter tweets to ensure they're actually from this user 1204 1205 const filteredTweets = tweets.filter((t) => { ··· 1212 1213 return true; 1213 1214 }); 1214 1215 1215 - const processedTweets = loadProcessedTweets(bskyIdentifier); 1216 - 1217 1216 // Maintain a local map that updates in real-time for intra-batch replies 1218 - const localProcessedMap: ProcessedTweetsMap = { ...processedTweets }; 1217 + const localProcessedMap: ProcessedTweetsMap = 1218 + sharedProcessedMap ?? { ...loadProcessedTweets(bskyIdentifier) }; 1219 1219 1220 1220 const toProcess = filteredTweets.filter((t) => !localProcessedMap[t.id_str || t.id || '']); 1221 1221 ··· 1235 1235 1236 1236 if (localProcessedMap[tweetId]) continue; 1237 1237 1238 + // Fallback to DB in case a nested backfill already saved this tweet. 1239 + const dbRecord = dbService.getTweet(tweetId, bskyIdentifier); 1240 + if (dbRecord) { 1241 + localProcessedMap[tweetId] = { 1242 + uri: dbRecord.bsky_uri, 1243 + cid: dbRecord.bsky_cid, 1244 + root: 1245 + dbRecord.bsky_root_uri && dbRecord.bsky_root_cid 1246 + ? { uri: dbRecord.bsky_root_uri, cid: dbRecord.bsky_root_cid } 1247 + : undefined, 1248 + tail: 1249 + dbRecord.bsky_tail_uri && dbRecord.bsky_tail_cid 1250 + ? { uri: dbRecord.bsky_tail_uri, cid: dbRecord.bsky_tail_cid } 1251 + : undefined, 1252 + migrated: dbRecord.status === 'migrated', 1253 + skipped: dbRecord.status === 'skipped', 1254 + }; 1255 + continue; 1256 + } 1257 + 1238 1258 const isRetweet = tweet.isRetweet || tweet.retweeted_status_id_str || tweet.text?.startsWith('RT @'); 1239 1259 1240 1260 if (isRetweet) { ··· 1284 1304 if (parentAuthor?.toLowerCase() === twitterUsername.toLowerCase()) { 1285 1305 console.log(`[${twitterUsername}] 🔄 Parent is ours (@${parentAuthor}). Backfilling parent first...`); 1286 1306 // Recursively process the parent 1287 - await processTweets(agent, twitterUsername, bskyIdentifier, [parentTweet], dryRun); 1307 + await processTweets(agent, twitterUsername, bskyIdentifier, [parentTweet], dryRun, localProcessedMap); 1288 1308 1289 1309 // Check if it was saved 1290 1310 const savedParent = dbService.getTweet(replyStatusId, bskyIdentifier);