indexx.dev
forked from
j4ck.xyz/tweets2bsky
A simple tool which lets you scrape twitter accounts and crosspost them to bluesky accounts! Comes with a CLI and a webapp for managing profiles! Works with images/videos/link embeds/threads.
feat: recursive backfill for broken threads
- Automatically fetch and process missing parent tweets if they belong to the same user.\n- Resolves issues with threaded replies being skipped during partial backfills.
+51
+51
src/index.ts
+51
src/index.ts
···
963
963
if (replyStatusId && localProcessedMap[replyStatusId]) {
964
964
console.log(`[${twitterUsername}] 🧵 Threading reply to post in ${bskyIdentifier}: ${replyStatusId}`);
965
965
replyParentInfo = localProcessedMap[replyStatusId] ?? null;
966
+
} else if (replyStatusId) {
967
+
// Parent missing from local batch/DB. Attempt to fetch it if it's a self-thread.
968
+
// We assume it's a self-thread if we don't have it, but we'll verify author after fetch.
969
+
console.log(`[${twitterUsername}] 🕵️ Parent ${replyStatusId} missing. Checking if backfillable...`);
970
+
971
+
let parentBackfilled = false;
972
+
try {
973
+
const scraper = await getTwitterScraper();
974
+
if (scraper) {
975
+
const parentRaw = await scraper.getTweet(replyStatusId);
976
+
if (parentRaw) {
977
+
const parentTweet = mapScraperTweetToLocalTweet(parentRaw);
978
+
const parentAuthor = parentTweet.user?.screen_name;
979
+
980
+
if (parentAuthor?.toLowerCase() === twitterUsername.toLowerCase()) {
981
+
console.log(`[${twitterUsername}] 🔄 Parent is ours (@${parentAuthor}). Backfilling parent first...`);
982
+
// Recursively process the parent
983
+
await processTweets(agent, twitterUsername, bskyIdentifier, [parentTweet], dryRun);
984
+
985
+
// Check if it was saved
986
+
const savedParent = dbService.getTweet(replyStatusId, bskyIdentifier);
987
+
if (savedParent && savedParent.status === 'migrated') {
988
+
// Update local map
989
+
localProcessedMap[replyStatusId] = {
990
+
uri: savedParent.bsky_uri,
991
+
cid: savedParent.bsky_cid,
992
+
root: (savedParent.bsky_root_uri && savedParent.bsky_root_cid) ? { uri: savedParent.bsky_root_uri, cid: savedParent.bsky_root_cid } : undefined,
993
+
tail: (savedParent.bsky_tail_uri && savedParent.bsky_tail_cid) ? { uri: savedParent.bsky_tail_uri, cid: savedParent.bsky_tail_cid } : undefined,
994
+
migrated: true
995
+
};
996
+
replyParentInfo = localProcessedMap[replyStatusId] ?? null;
997
+
parentBackfilled = true;
998
+
console.log(`[${twitterUsername}] ✅ Parent backfilled. Resuming thread.`);
999
+
}
1000
+
} else {
1001
+
console.log(`[${twitterUsername}] ⏩ Parent is by @${parentAuthor}. Skipping external reply.`);
1002
+
}
1003
+
}
1004
+
}
1005
+
} catch (e) {
1006
+
console.warn(`[${twitterUsername}] ⚠️ Failed to fetch/backfill parent ${replyStatusId}:`, e);
1007
+
}
1008
+
1009
+
if (!parentBackfilled) {
1010
+
console.log(`[${twitterUsername}] ⏩ Skipping external/unknown reply (Parent not found or external).`);
1011
+
if (!dryRun) {
1012
+
saveProcessedTweet(twitterUsername, bskyIdentifier, tweetId, { skipped: true, text: tweetText });
1013
+
localProcessedMap[tweetId] = { skipped: true, text: tweetText };
1014
+
}
1015
+
continue;
1016
+
}
966
1017
} else {
967
1018
console.log(`[${twitterUsername}] ⏩ Skipping external/unknown reply.`);
968
1019
if (!dryRun) {