skywatch.blue
A tool for parsing traffic on the jetstream and applying a moderation workstream based on regexp based rules
feat: Add environment variable validation
Adds a function to validate environment variables. This will catch any
issues early in the startup process.
+181
-106
src/main.ts
+181
-106
src/main.ts
···
1
-
import fs from 'node:fs';
1
+
import fs from "node:fs";
2
2
3
3
import type {
4
4
CommitCreateEvent,
5
5
CommitUpdateEvent,
6
-
IdentityEvent } from '@skyware/jetstream';
7
-
import {
8
-
Jetstream,
9
-
} from '@skyware/jetstream';
10
-
6
+
IdentityEvent,
7
+
} from "@skyware/jetstream";
8
+
import { Jetstream } from "@skyware/jetstream";
11
9
12
-
import { checkHandle } from './checkHandles.js';
13
-
import { checkPosts } from './checkPosts.js';
14
-
import { checkDescription, checkDisplayName } from './checkProfiles.js';
15
-
import { checkStarterPack, checkNewStarterPack } from './checkStarterPack.js';
10
+
import { checkHandle } from "./checkHandles.js";
11
+
import { checkPosts } from "./checkPosts.js";
12
+
import { checkDescription, checkDisplayName } from "./checkProfiles.js";
13
+
import { checkStarterPack, checkNewStarterPack } from "./checkStarterPack.js";
16
14
import {
17
15
CURSOR_UPDATE_INTERVAL,
18
16
FIREHOSE_URL,
19
17
METRICS_PORT,
20
18
WANTED_COLLECTION,
21
-
} from './config.js';
22
-
import logger from './logger.js';
23
-
import { startMetricsServer } from './metrics.js';
24
-
import type { Post, LinkFeature } from './types.js';
19
+
} from "./config.js";
20
+
import { validateEnvironment } from "./validateEnv.js";
21
+
import logger from "./logger.js";
22
+
import { startMetricsServer } from "./metrics.js";
23
+
import type { Post, LinkFeature } from "./types.js";
24
+
25
+
validateEnvironment();
25
26
26
27
let cursor = 0;
27
28
let cursorUpdateInterval: NodeJS.Timeout;
···
31
32
}
32
33
33
34
try {
34
-
logger.info('Trying to read cursor from cursor.txt...');
35
-
cursor = Number(fs.readFileSync('cursor.txt', 'utf8'));
36
-
logger.info(`Cursor found: ${cursor.toString()} (${epochUsToDateTime(cursor)})`);
35
+
logger.info("Trying to read cursor from cursor.txt...");
36
+
cursor = Number(fs.readFileSync("cursor.txt", "utf8"));
37
+
logger.info(
38
+
`Cursor found: ${cursor.toString()} (${epochUsToDateTime(cursor)})`,
39
+
);
37
40
} catch (error) {
38
-
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
41
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") {
39
42
cursor = Math.floor(Date.now() * 1000);
40
43
logger.info(
41
44
`Cursor not found in cursor.txt, setting cursor to: ${cursor.toString()} (${epochUsToDateTime(cursor)})`,
42
45
);
43
-
fs.writeFileSync('cursor.txt', cursor.toString(), 'utf8');
46
+
fs.writeFileSync("cursor.txt", cursor.toString(), "utf8");
44
47
} else {
45
48
logger.error(error);
46
49
process.exit(1);
···
53
56
cursor,
54
57
});
55
58
56
-
jetstream.on('open', () => {
59
+
jetstream.on("open", () => {
57
60
if (jetstream.cursor) {
58
61
logger.info(
59
62
`Connected to Jetstream at ${FIREHOSE_URL} with cursor ${jetstream.cursor.toString()} (${epochUsToDateTime(jetstream.cursor)})`,
···
68
71
logger.info(
69
72
`Cursor updated to: ${jetstream.cursor.toString()} (${epochUsToDateTime(jetstream.cursor)})`,
70
73
);
71
-
fs.writeFile('cursor.txt', jetstream.cursor.toString(), (err) => {
74
+
fs.writeFile("cursor.txt", jetstream.cursor.toString(), (err) => {
72
75
if (err) logger.error(err);
73
76
});
74
77
}
75
78
}, CURSOR_UPDATE_INTERVAL);
76
79
});
77
80
78
-
jetstream.on('close', () => {
81
+
jetstream.on("close", () => {
79
82
clearInterval(cursorUpdateInterval);
80
-
logger.info('Jetstream connection closed.');
83
+
logger.info("Jetstream connection closed.");
81
84
});
82
85
83
-
jetstream.on('error', (error) => {
86
+
jetstream.on("error", (error) => {
84
87
logger.error(`Jetstream error: ${error.message}`);
85
88
});
86
89
87
90
// Check for post updates
88
91
89
92
jetstream.onCreate(
90
-
'app.bsky.feed.post',
91
-
(event: CommitCreateEvent<'app.bsky.feed.post'>) => {
93
+
"app.bsky.feed.post",
94
+
(event: CommitCreateEvent<"app.bsky.feed.post">) => {
92
95
try {
93
96
const atURI = `at://${event.did}/app.bsky.feed.post/${event.commit.rkey}`;
94
-
const hasFacets = Object.hasOwn(event.commit.record, 'facets');
95
-
const hasText = Object.hasOwn(event.commit.record, 'text');
97
+
const hasFacets = Object.hasOwn(event.commit.record, "facets");
98
+
const hasText = Object.hasOwn(event.commit.record, "text");
96
99
97
100
const tasks: Promise<void>[] = [];
98
101
···
100
103
if (hasFacets && event.commit.record.facets) {
101
104
const hasLinkType = event.commit.record.facets.some((facet) =>
102
105
facet.features.some(
103
-
(feature) => feature.$type === 'app.bsky.richtext.facet#link',
106
+
(feature) => feature.$type === "app.bsky.richtext.facet#link",
104
107
),
105
108
);
106
109
107
110
if (hasLinkType) {
108
-
const urls = event.commit.record.facets.flatMap((facet) =>
109
-
facet.features.filter(
110
-
(feature) => feature.$type === 'app.bsky.richtext.facet#link',
111
-
),
112
-
)
111
+
const urls = event.commit.record.facets
112
+
.flatMap((facet) =>
113
+
facet.features.filter(
114
+
(feature) => feature.$type === "app.bsky.richtext.facet#link",
115
+
),
116
+
)
113
117
.map((feature: LinkFeature) => feature.uri);
114
118
115
119
urls.forEach((url) => {
···
123
127
cid: event.commit.cid,
124
128
},
125
129
];
126
-
tasks.push(checkPosts(posts).catch((error: unknown) => {
127
-
logger.error(`Error checking post links for ${event.did}:`, error);
128
-
}));
130
+
tasks.push(
131
+
checkPosts(posts).catch((error: unknown) => {
132
+
logger.error(
133
+
`Error checking post links for ${event.did}:`,
134
+
error,
135
+
);
136
+
}),
137
+
);
129
138
});
130
139
}
131
140
} else if (hasText && event.commit.record.text) {
···
139
148
cid: event.commit.cid,
140
149
},
141
150
];
142
-
tasks.push(checkPosts(posts).catch((error: unknown) => {
143
-
logger.error(`Error checking post text for ${event.did}:`, error);
144
-
}));
151
+
tasks.push(
152
+
checkPosts(posts).catch((error: unknown) => {
153
+
logger.error(`Error checking post text for ${event.did}:`, error);
154
+
}),
155
+
);
145
156
}
146
157
147
158
// Wait for all tasks to complete
···
156
167
157
168
// Check for profile updates
158
169
jetstream.onUpdate(
159
-
'app.bsky.actor.profile',
160
-
(event: CommitUpdateEvent<'app.bsky.actor.profile'>) => {
170
+
"app.bsky.actor.profile",
171
+
(event: CommitUpdateEvent<"app.bsky.actor.profile">) => {
161
172
try {
162
173
const tasks: Promise<void>[] = [];
163
174
164
175
if (event.commit.record.displayName || event.commit.record.description) {
165
-
const displayName = event.commit.record.displayName ?? '';
166
-
const description = event.commit.record.description ?? '';
167
-
176
+
const displayName = event.commit.record.displayName ?? "";
177
+
const description = event.commit.record.description ?? "";
178
+
168
179
tasks.push(
169
-
checkDescription(event.did, event.time_us, displayName, description)
170
-
.catch((error: unknown) => {
171
-
logger.error(`Error checking profile description for ${event.did}:`, error);
172
-
})
180
+
checkDescription(
181
+
event.did,
182
+
event.time_us,
183
+
displayName,
184
+
description,
185
+
).catch((error: unknown) => {
186
+
logger.error(
187
+
`Error checking profile description for ${event.did}:`,
188
+
error,
189
+
);
190
+
}),
173
191
);
174
-
192
+
175
193
tasks.push(
176
-
checkDisplayName(event.did, event.time_us, displayName, description)
177
-
.catch((error: unknown) => {
178
-
logger.error(`Error checking profile display name for ${event.did}:`, error);
179
-
})
194
+
checkDisplayName(
195
+
event.did,
196
+
event.time_us,
197
+
displayName,
198
+
description,
199
+
).catch((error: unknown) => {
200
+
logger.error(
201
+
`Error checking profile display name for ${event.did}:`,
202
+
error,
203
+
);
204
+
}),
180
205
);
181
206
}
182
207
183
208
if (event.commit.record.joinedViaStarterPack) {
184
209
tasks.push(
185
-
checkStarterPack(event.did, event.time_us, event.commit.record.joinedViaStarterPack.uri)
186
-
.catch((error: unknown) => {
187
-
logger.error(`Error checking starter pack for ${event.did}:`, error);
188
-
})
210
+
checkStarterPack(
211
+
event.did,
212
+
event.time_us,
213
+
event.commit.record.joinedViaStarterPack.uri,
214
+
).catch((error: unknown) => {
215
+
logger.error(
216
+
`Error checking starter pack for ${event.did}:`,
217
+
error,
218
+
);
219
+
}),
189
220
);
190
221
}
191
222
···
194
225
void Promise.allSettled(tasks);
195
226
}
196
227
} catch (error: unknown) {
197
-
logger.error(`Error processing profile update event for ${event.did}:`, error);
228
+
logger.error(
229
+
`Error processing profile update event for ${event.did}:`,
230
+
error,
231
+
);
198
232
}
199
233
},
200
234
);
···
202
236
// Check for profile updates
203
237
204
238
jetstream.onCreate(
205
-
'app.bsky.actor.profile',
206
-
(event: CommitCreateEvent<'app.bsky.actor.profile'>) => {
239
+
"app.bsky.actor.profile",
240
+
(event: CommitCreateEvent<"app.bsky.actor.profile">) => {
207
241
try {
208
242
const tasks: Promise<void>[] = [];
209
243
210
244
if (event.commit.record.displayName || event.commit.record.description) {
211
-
const displayName = event.commit.record.displayName ?? '';
212
-
const description = event.commit.record.description ?? '';
213
-
245
+
const displayName = event.commit.record.displayName ?? "";
246
+
const description = event.commit.record.description ?? "";
247
+
214
248
tasks.push(
215
-
checkDescription(event.did, event.time_us, displayName, description)
216
-
.catch((error: unknown) => {
217
-
logger.error(`Error checking profile description for ${event.did}:`, error);
218
-
})
249
+
checkDescription(
250
+
event.did,
251
+
event.time_us,
252
+
displayName,
253
+
description,
254
+
).catch((error: unknown) => {
255
+
logger.error(
256
+
`Error checking profile description for ${event.did}:`,
257
+
error,
258
+
);
259
+
}),
219
260
);
220
-
261
+
221
262
tasks.push(
222
-
checkDisplayName(event.did, event.time_us, displayName, description)
223
-
.catch((error: unknown) => {
224
-
logger.error(`Error checking profile display name for ${event.did}:`, error);
225
-
})
263
+
checkDisplayName(
264
+
event.did,
265
+
event.time_us,
266
+
displayName,
267
+
description,
268
+
).catch((error: unknown) => {
269
+
logger.error(
270
+
`Error checking profile display name for ${event.did}:`,
271
+
error,
272
+
);
273
+
}),
226
274
);
227
275
228
276
if (event.commit.record.joinedViaStarterPack) {
229
277
tasks.push(
230
-
checkStarterPack(event.did, event.time_us, event.commit.record.joinedViaStarterPack.uri)
231
-
.catch((error: unknown) => {
232
-
logger.error(`Error checking starter pack for ${event.did}:`, error);
233
-
})
278
+
checkStarterPack(
279
+
event.did,
280
+
event.time_us,
281
+
event.commit.record.joinedViaStarterPack.uri,
282
+
).catch((error: unknown) => {
283
+
logger.error(
284
+
`Error checking starter pack for ${event.did}:`,
285
+
error,
286
+
);
287
+
}),
234
288
);
235
289
}
236
290
···
240
294
}
241
295
}
242
296
} catch (error: unknown) {
243
-
logger.error(`Error processing profile creation event for ${event.did}:`, error);
297
+
logger.error(
298
+
`Error processing profile creation event for ${event.did}:`,
299
+
error,
300
+
);
244
301
}
245
302
},
246
303
);
247
304
248
305
jetstream.onCreate(
249
-
'app.bsky.graph.starterpack',
250
-
(event: CommitCreateEvent<'app.bsky.graph.starterpack'>) => {
306
+
"app.bsky.graph.starterpack",
307
+
(event: CommitCreateEvent<"app.bsky.graph.starterpack">) => {
251
308
try {
252
309
const atURI = `at://${event.did}/app.bsky.feed.post/${event.commit.rkey}`;
253
310
const { name, description } = event.commit.record;
···
260
317
name,
261
318
description,
262
319
).catch((error: unknown) => {
263
-
logger.error(`Error checking new starter pack for ${event.did}:`, error);
320
+
logger.error(
321
+
`Error checking new starter pack for ${event.did}:`,
322
+
error,
323
+
);
264
324
});
265
325
} catch (error: unknown) {
266
-
logger.error(`Error processing starter pack creation event for ${event.did}:`, error);
326
+
logger.error(
327
+
`Error processing starter pack creation event for ${event.did}:`,
328
+
error,
329
+
);
267
330
}
268
331
},
269
332
);
270
333
271
334
jetstream.onUpdate(
272
-
'app.bsky.graph.starterpack',
273
-
(event: CommitUpdateEvent<'app.bsky.graph.starterpack'>) => {
335
+
"app.bsky.graph.starterpack",
336
+
(event: CommitUpdateEvent<"app.bsky.graph.starterpack">) => {
274
337
try {
275
338
const atURI = `at://${event.did}/app.bsky.feed.post/${event.commit.rkey}`;
276
339
const { name, description } = event.commit.record;
···
283
346
name,
284
347
description,
285
348
).catch((error: unknown) => {
286
-
logger.error(`Error checking updated starter pack for ${event.did}:`, error);
349
+
logger.error(
350
+
`Error checking updated starter pack for ${event.did}:`,
351
+
error,
352
+
);
287
353
});
288
354
} catch (error: unknown) {
289
-
logger.error(`Error processing starter pack update event for ${event.did}:`, error);
355
+
logger.error(
356
+
`Error processing starter pack update event for ${event.did}:`,
357
+
error,
358
+
);
290
359
}
291
360
},
292
361
);
293
362
294
363
// Check for handle updates
295
-
jetstream.on('identity', (event: IdentityEvent) => {
364
+
jetstream.on("identity", (event: IdentityEvent) => {
296
365
try {
297
366
if (event.identity.handle) {
298
-
void checkHandle(event.identity.did, event.identity.handle, event.time_us)
299
-
.catch((error: unknown) => {
300
-
logger.error(`Error checking handle for ${event.identity.did}:`, error);
301
-
});
367
+
void checkHandle(
368
+
event.identity.did,
369
+
event.identity.handle,
370
+
event.time_us,
371
+
).catch((error: unknown) => {
372
+
logger.error(`Error checking handle for ${event.identity.did}:`, error);
373
+
});
302
374
}
303
375
} catch (error: unknown) {
304
-
logger.error(`Error processing identity event for ${event.identity.did}:`, error);
376
+
logger.error(
377
+
`Error processing identity event for ${event.identity.did}:`,
378
+
error,
379
+
);
305
380
}
306
381
});
307
382
···
311
386
metricsServer = startMetricsServer(METRICS_PORT);
312
387
logger.info(`Metrics server started on port ${METRICS_PORT.toString()}`);
313
388
} catch (error: unknown) {
314
-
logger.error('Failed to start metrics server:', error);
389
+
logger.error("Failed to start metrics server:", error);
315
390
process.exit(1);
316
391
}
317
392
···
326
401
// Start jetstream with error handling
327
402
try {
328
403
jetstream.start();
329
-
logger.info('Jetstream started successfully');
404
+
logger.info("Jetstream started successfully");
330
405
} catch (error: unknown) {
331
-
logger.error('Failed to start jetstream:', error);
406
+
logger.error("Failed to start jetstream:", error);
332
407
process.exit(1);
333
408
}
334
409
335
410
function shutdown() {
336
411
try {
337
-
logger.info('Shutting down gracefully...');
412
+
logger.info("Shutting down gracefully...");
338
413
if (jetstream.cursor) {
339
-
fs.writeFileSync('cursor.txt', jetstream.cursor.toString(), 'utf8');
414
+
fs.writeFileSync("cursor.txt", jetstream.cursor.toString(), "utf8");
340
415
}
341
416
jetstream.close();
342
417
if (metricsServer) {
343
418
metricsServer.close(() => {
344
-
logger.info('Metrics server closed');
419
+
logger.info("Metrics server closed");
345
420
});
346
421
}
347
-
logger.info('Shutdown completed successfully');
422
+
logger.info("Shutdown completed successfully");
348
423
} catch (error: unknown) {
349
-
logger.error('Error shutting down gracefully:', error);
424
+
logger.error("Error shutting down gracefully:", error);
350
425
process.exit(1);
351
426
}
352
427
}
353
428
354
429
// Global error handlers
355
-
process.on('unhandledRejection', (reason, promise) => {
356
-
logger.error('Unhandled Promise Rejection at:', promise, 'reason:', reason);
430
+
process.on("unhandledRejection", (reason, promise) => {
431
+
logger.error("Unhandled Promise Rejection at:", promise, "reason:", reason);
357
432
// Don't exit the process for unhandled rejections, just log them
358
433
});
359
434
360
-
process.on('uncaughtException', (error) => {
361
-
logger.error('Uncaught Exception:', error);
435
+
process.on("uncaughtException", (error) => {
436
+
logger.error("Uncaught Exception:", error);
362
437
shutdown();
363
438
});
364
439
365
-
process.on('SIGINT', shutdown);
366
-
process.on('SIGTERM', shutdown);
440
+
process.on("SIGINT", shutdown);
441
+
process.on("SIGTERM", shutdown);