t1c.dev
forked from
rocksky.app/rocksky
A decentralized music tracking and discovery platform built on AT Protocol 馃幍
tsiry-sandratraina.com
refactor: restructure deduplication logic to use a do-while loop for batch processing
1import chalk from "chalk";
2import { ctx } from "context";
3import { eq } from "drizzle-orm";
4import { createAgent } from "lib/agent";
5import tables from "schema";
6
7const args = process.argv.slice(2);
8
9if (args.length === 0) {
10 console.error("Please provide user author identifier (handle or DID).");
11 console.log(`Usage: ${chalk.cyan("npm run feed -- <handle|did>")}`);
12 process.exit(1);
13}
14
15let did = args[0];
16
17if (!did.startsWith("did:plc:")) {
18 did = await ctx.baseIdResolver.handle.resolve(did);
19}
20
21const agent = await createAgent(ctx.oauthClient, did);
22let cursor: string | undefined;
23const BATCH_SIZE = 100;
24let i = 1;
25do {
26 const records = await agent.com.atproto.repo.listRecords({
27 repo: agent.assertDid,
28 collection: "app.rocksky.scrobble",
29 limit: BATCH_SIZE,
30 });
31
32 for (const record of records.data.records) {
33 const result = await ctx.db
34 .select()
35 .from(tables.scrobbles)
36 .where(eq(tables.scrobbles.uri, record.uri))
37 .limit(1);
38 if (result.length === 0) {
39 console.log(`${i} Deleting record:`);
40 console.log(record);
41 const rkey = record.uri.split("/").pop();
42 await agent.com.atproto.repo.deleteRecord({
43 repo: agent.assertDid,
44 collection: "app.rocksky.scrobble",
45 rkey,
46 });
47 await new Promise((resolve) => setTimeout(resolve, 1000)); // rate limit
48 } else {
49 console.log(chalk.greenBright(`${i} Keeping record:`));
50 console.log(record);
51 }
52 i += 1;
53 }
54 cursor = records.data.cursor;
55} while (cursor);
56
57console.log(chalk.greenBright("Deduplication complete."));
58
59process.exit(0);