rocksky.app
A decentralized music tracking and discovery platform built on AT Protocol 🎵
rocksky.app
spotify
atproto
lastfm
musicbrainz
scrobbling
listenbrainz
Batch events for WebSocket streaming
Send events as JSON arrays to improve throughput (batch size 50).
Increase PAGE_SIZE to 500 and send queued events in batches.
Update client to handle batched messages and reduce ping interval
to 30s. Adjust progress/logging to report events every 500.
tsiry-sandratraina.com
+16
-5
tap/scripts/test-client.ts
+16
-5
tap/scripts/test-client.ts
···
23
23
console.log("📤 Sending ping...");
24
24
ws.send("ping");
25
25
26
-
// Send ping every 5 seconds
26
+
// Send ping every 30 seconds (less frequent to not interfere with fast streaming)
27
27
setInterval(() => {
28
28
if (ws.readyState === WebSocket.OPEN) {
29
29
const now = Date.now();
···
33
33
);
34
34
ws.send("ping");
35
35
}
36
-
}, 5000);
36
+
}, 30000);
37
37
};
38
38
39
39
ws.onmessage = async (event) => {
···
43
43
44
44
try {
45
45
const data = JSON.parse(event.data);
46
-
if (data.type === "connected") {
46
+
47
+
// Handle batched messages (array of events)
48
+
if (Array.isArray(data)) {
49
+
messageCount += data.length;
50
+
if (messageCount % 500 === 0 || messageCount <= 50) {
51
+
console.log(
52
+
`📨 [${elapsed}s] Batch received: ${data.length} events (total: ${messageCount})`,
53
+
);
54
+
}
55
+
}
56
+
// Handle single messages
57
+
else if (data.type === "connected") {
47
58
console.log(`📨 [${elapsed}s] Connection confirmed: ${data.message}`);
48
59
} else if (data.type === "heartbeat") {
49
60
console.log(`💓 [${elapsed}s] Heartbeat received`);
···
59
70
console.log(`📨 [${elapsed}s] Message #${messageCount}: ${event.data}`);
60
71
}
61
72
62
-
if (messageCount % 100 === 0) {
73
+
if (messageCount % 500 === 0) {
63
74
const rate = (messageCount / parseFloat(elapsed)).toFixed(2);
64
75
console.log(
65
-
`📊 Progress: ${messageCount} messages received in ${elapsed}s (${rate} msg/s)`,
76
+
`📊 Progress: ${messageCount} events received in ${elapsed}s (${rate} events/s)`,
66
77
);
67
78
}
68
79
+41
-34
tap/src/main.ts
+41
-34
tap/src/main.ts
···
2
2
import logger from "./logger.ts";
3
3
import schema from "./schema/mod.ts";
4
4
import { asc, inArray } from "drizzle-orm";
5
-
import { omit } from "@es-toolkit/es-toolkit/compat";
6
5
import type { SelectEvent } from "./schema/event.ts";
7
6
import { assureAdminAuth, parseTapEvent } from "@atproto/tap";
8
7
import { addToBatch, flushBatch } from "./batch.ts";
9
8
10
-
const PAGE_SIZE = 100;
11
-
const YIELD_EVERY_N_PAGES = 5;
12
-
const YIELD_DELAY_MS = 100;
9
+
const PAGE_SIZE = 500;
10
+
const BATCH_SEND_SIZE = 50;
13
11
const ADMIN_PASSWORD = Deno.env.get("TAP_ADMIN_PASSWORD")!;
14
12
15
13
interface ClientState {
···
43
41
return false;
44
42
}
45
43
44
+
function formatEvent(evt: SelectEvent): string {
45
+
const { createdAt: _createdAt, record, ...rest } = evt;
46
+
if (record) {
47
+
return JSON.stringify({ ...rest, record: JSON.parse(record) });
48
+
}
49
+
return JSON.stringify(rest);
50
+
}
51
+
46
52
export function broadcastEvent(evt: SelectEvent) {
47
-
const message = JSON.stringify({
48
-
...omit(evt, "createdAt", "record"),
49
-
...(evt.record && {
50
-
record: JSON.parse(evt.record),
51
-
}),
52
-
});
53
+
const message = formatEvent(evt);
53
54
54
55
for (const [socket, state] of connectedClients.entries()) {
55
56
if (socket.readyState === WebSocket.OPEN) {
···
201
202
logger.info`📄 Fetching page ${page}... (${totalEvents} events sent so far)`;
202
203
}
203
204
205
+
// Batch send events for better performance
206
+
const batchMessages: string[] = [];
204
207
for (let i = 0; i < events.length; i++) {
205
208
const evt = events[i];
206
209
···
209
212
return;
210
213
}
211
214
212
-
const success = safeSend(
213
-
socket,
214
-
JSON.stringify({
215
-
...omit(evt, "createdAt", "record"),
216
-
...(evt.record && {
217
-
record: JSON.parse(evt.record),
218
-
}),
219
-
}),
220
-
totalEvents,
221
-
);
215
+
batchMessages.push(formatEvent(evt));
216
+
217
+
// Send batch when full or at end of page
218
+
if (
219
+
batchMessages.length >= BATCH_SEND_SIZE ||
220
+
i === events.length - 1
221
+
) {
222
+
const batchMessage = `[${batchMessages.join(",")}]`;
223
+
const success = safeSend(socket, batchMessage, totalEvents);
222
224
223
-
if (success) {
224
-
totalEvents++;
225
-
} else {
226
-
logger.error`❌ Failed to send event at index ${totalEvents}, stopping pagination`;
227
-
return;
225
+
if (success) {
226
+
totalEvents += batchMessages.length;
227
+
batchMessages.length = 0; // Clear batch
228
+
} else {
229
+
logger.error`❌ Failed to send batch at ${totalEvents}, stopping pagination`;
230
+
return;
231
+
}
228
232
}
229
233
}
230
234
···
247
251
if (queuedCount > 0) {
248
252
logger.info`📦 Sending ${queuedCount} queued events...`;
249
253
254
+
// Batch send queued events
255
+
const queueMessages: string[] = [];
250
256
for (const evt of clientState.queue) {
251
257
if (socket.readyState !== WebSocket.OPEN) break;
252
258
253
-
safeSend(
254
-
socket,
255
-
JSON.stringify({
256
-
...omit(evt, "createdAt", "record"),
257
-
...(evt.record && {
258
-
record: JSON.parse(evt.record),
259
-
}),
260
-
}),
261
-
);
259
+
queueMessages.push(formatEvent(evt));
260
+
261
+
if (queueMessages.length >= BATCH_SEND_SIZE) {
262
+
safeSend(socket, `[${queueMessages.join(",")}]`);
263
+
queueMessages.length = 0;
264
+
}
265
+
}
266
+
267
+
if (queueMessages.length > 0) {
268
+
safeSend(socket, `[${queueMessages.join(",")}]`);
262
269
}
263
270
264
271
clientState.queue = [];