edavis.dev
this repo has no description
bsky-activity.py: switch to jetstream
+10
-35
bsky-activity.py
+10
-35
bsky-activity.py
···
3
3
import asyncio
4
4
from datetime import datetime, timezone
5
5
from io import BytesIO
6
+
import json
6
7
import os
7
8
import sqlite3
8
9
import sys
···
29
30
])
30
31
31
32
async def bsky_activity():
32
-
redis_cnx = redis.Redis()
33
-
relay_url = 'wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos'
34
-
firehose_seq = redis_cnx.get('dev.edavis.muninsky.seq')
35
-
if firehose_seq:
36
-
relay_url += f'?cursor={firehose_seq.decode()}'
33
+
relay_url = 'ws://localhost:6008/subscribe'
37
34
38
35
sys.stdout.write(f'opening websocket connection to {relay_url}\n')
39
36
sys.stdout.flush()
40
37
41
38
async with websockets.connect(relay_url, ping_timeout=60) as firehose:
42
39
while True:
43
-
frame = BytesIO(await firehose.recv())
44
-
header = dag_cbor.decode(frame, allow_concat=True)
45
-
if header['op'] != 1 or header['t'] != '#commit':
46
-
continue
40
+
payload = BytesIO(await firehose.recv())
47
41
48
-
payload = dag_cbor.decode(frame)
49
-
if payload['tooBig']:
50
-
# TODO(ejd): figure out how to get blocks out-of-band
51
-
continue
52
-
53
-
# TODO(ejd): figure out how to validate blocks
54
-
blocks = payload.pop('blocks')
55
-
car_parsed = CAR.from_bytes(blocks)
56
-
57
-
message = payload.copy()
58
-
del message['ops']
59
-
message['commit'] = message['commit'].encode('base32')
60
-
61
-
for commit_op in payload['ops']:
62
-
op = commit_op.copy()
63
-
if op['cid'] is not None:
64
-
op['cid'] = op['cid'].encode('base32')
65
-
op['record'] = car_parsed.blocks.get(op['cid'])
66
-
67
-
yield message, op
42
+
yield json.load(payload)
68
43
69
44
async def main():
70
45
redis_cnx = redis.Redis()
···
89
64
sys.stdout.flush()
90
65
91
66
op_count = 0
92
-
async for commit, op in bsky_activity():
93
-
if op['action'] != 'create':
67
+
async for payload in bsky_activity():
68
+
if payload['opType'] != 'c':
94
69
continue
95
70
96
-
collection, _ = op['path'].split('/')
71
+
collection = payload['collection']
97
72
if collection not in app_bsky_allowlist:
98
73
continue
99
74
100
-
repo_did = commit['repo']
101
-
repo_update_time = datetime.strptime(commit['time'], '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=timezone.utc)
75
+
repo_did = payload['did']
76
+
repo_update_time = datetime.now(timezone.utc)
102
77
db_cnx.execute(
103
78
'insert into users values (:did, :ts) on conflict (did) do update set ts = :ts',
104
79
{'did': repo_did, 'ts': repo_update_time.timestamp()}
···
111
86
op_count += 1
112
87
if op_count % 500 == 0:
113
88
now = datetime.now(timezone.utc)
114
-
payload_seq = commit['seq']
89
+
payload_seq = payload['seq']
115
90
payload_lag = now - repo_update_time
116
91
117
92
sys.stdout.write(f'seq: {payload_seq}, lag: {payload_lag.total_seconds()}\n')