edavis.dev
this repo has no description
feat(bsky-activity): tidy up and use newer jetstream JSON shapes
+12
-15
bsky-activity.py
+12
-15
bsky-activity.py
···
2
2
3
3
import asyncio
4
4
from datetime import datetime, timezone
5
-
from io import BytesIO
6
5
import json
7
6
import os
8
7
import sqlite3
9
8
import sys
10
9
11
-
from atproto import CAR
12
10
import redis
13
-
import dag_cbor
14
11
import websockets
15
12
16
13
app_bsky_allowlist = set([
···
39
36
40
37
async with websockets.connect(relay_url, ping_timeout=60) as firehose:
41
38
while True:
42
-
payload = BytesIO(await firehose.recv())
43
-
44
-
yield json.load(payload)
39
+
yield json.loads(await firehose.recv())
45
40
46
41
async def main():
47
42
redis_cnx = redis.Redis()
···
66
61
sys.stdout.flush()
67
62
68
63
op_count = 0
69
-
async for payload in bsky_activity():
70
-
if payload['opType'] != 'c':
64
+
async for event in bsky_activity():
65
+
if event['type'] != 'com':
66
+
continue
67
+
68
+
payload = event['commit']
69
+
if payload['type'] != 'c':
71
70
continue
72
71
73
72
collection = payload['collection']
74
73
if collection not in app_bsky_allowlist:
75
74
continue
76
75
77
-
repo_did = payload['did']
76
+
repo_did = event['did']
78
77
repo_update_time = datetime.now(timezone.utc)
79
78
db_cnx.execute(
80
79
'insert into users values (:did, :ts) on conflict (did) do update set ts = :ts',
···
93
92
94
93
op_count += 1
95
94
if op_count % 500 == 0:
96
-
now = datetime.now(timezone.utc)
97
-
payload_seq = payload['seq']
98
-
payload_lag = now - repo_update_time
99
-
100
-
sys.stdout.write(f'seq: {payload_seq}, lag: {payload_lag.total_seconds()}\n')
101
-
redis_pipe.set('dev.edavis.muninsky.seq', payload_seq)
95
+
current_time_ms = datetime.now(timezone.utc).timestamp()
96
+
event_time_ms = event['time_us'] / 1_000_000
97
+
current_lag = current_time_ms - event_time_ms
98
+
sys.stdout.write(f'lag: {current_lag:.2f}\n')
102
99
redis_pipe.execute()
103
100
db_cnx.commit()
104
101
sys.stdout.flush()