graham.systems
forked from
samuel.fm/statusphere-react
the statusphere demo reworked into a vite/react app in a monorepo
add jetstream
samuel.fm
+272
-29
-1
lexicons/xyz/statusphere/getStatuses.json
-1
lexicons/xyz/statusphere/getStatuses.json
+4
-3
packages/appview/src/context.ts
+4
-3
packages/appview/src/context.ts
···
2
import { Firehose } from '@atproto/sync'
3
import pino from 'pino'
4
5
-
import { Database } from './db'
6
-
import { BidirectionalResolver } from './id-resolver'
7
8
// Application state passed to the router and elsewhere
9
export type AppContext = {
10
db: Database
11
-
ingester: Firehose
12
logger: pino.Logger
13
oauthClient: OAuthClient
14
resolver: BidirectionalResolver
···
2
import { Firehose } from '@atproto/sync'
3
import pino from 'pino'
4
5
+
import { Database } from '#/db'
6
+
import { BidirectionalResolver } from '#/id-resolver'
7
+
import { Jetstream } from '#/ingestors'
8
9
// Application state passed to the router and elsewhere
10
export type AppContext = {
11
db: Database
12
+
ingester: Firehose | Jetstream<any>
13
logger: pino.Logger
14
oauthClient: OAuthClient
15
resolver: BidirectionalResolver
+15
packages/appview/src/db.ts
+15
packages/appview/src/db.ts
···
53
},
54
}
55
56
+
migrations['003'] = {
57
+
async up(db: Kysely<unknown>) {},
58
+
async down(_db: Kysely<unknown>) {},
59
+
}
60
+
61
migrations['002'] = {
62
async up(db: Kysely<unknown>) {
63
await db.schema
64
.createTable('cursor')
65
.addColumn('id', 'integer', (col) => col.primaryKey())
66
.addColumn('seq', 'integer', (col) => col.notNull())
67
+
.execute()
68
+
69
+
// Insert initial cursor values:
70
+
// id=1 is for firehose, id=2 is for jetstream
71
+
await db
72
+
.insertInto('cursor' as never)
73
+
.values([
74
+
{ id: 1, seq: 0 },
75
+
{ id: 2, seq: 0 },
76
+
])
77
.execute()
78
},
79
async down(db: Kysely<unknown>) {
+4
-3
packages/appview/src/index.ts
+4
-3
packages/appview/src/index.ts
···
14
import { createDb, migrateToLatest } from '#/db'
15
import * as error from '#/error'
16
import { createBidirectionalResolver, createIdResolver } from '#/id-resolver'
17
-
import { createIngester } from '#/ingester'
18
import { createServer } from '#/lexicons'
19
import { env } from '#/lib/env'
20
···
36
// Create the atproto utilities
37
const oauthClient = await createClient(db)
38
const baseIdResolver = createIdResolver()
39
-
const ingester = await createIngester(db, baseIdResolver)
40
const resolver = createBidirectionalResolver(baseIdResolver)
41
const ctx = {
42
db,
···
103
})
104
}
105
} else {
106
-
server.xrpc.router.set('trust proxy', true)
107
}
108
109
// Use the port from env (should be 3001 for the API server)
···
14
import { createDb, migrateToLatest } from '#/db'
15
import * as error from '#/error'
16
import { createBidirectionalResolver, createIdResolver } from '#/id-resolver'
17
+
import { createFirehoseIngester, createJetstreamIngester } from '#/ingestors'
18
import { createServer } from '#/lexicons'
19
import { env } from '#/lib/env'
20
···
36
// Create the atproto utilities
37
const oauthClient = await createClient(db)
38
const baseIdResolver = createIdResolver()
39
+
const ingester = await createJetstreamIngester(db)
40
+
// Alternative: const ingester = await createFirehoseIngester(db, baseIdResolver)
41
const resolver = createBidirectionalResolver(baseIdResolver)
42
const ctx = {
43
db,
···
104
})
105
}
106
} else {
107
+
app.set('trust proxy', true)
108
}
109
110
// Use the port from env (should be 3001 for the API server)
+4
-1
packages/appview/src/ingester.ts
packages/appview/src/ingestors/firehose.ts
+4
-1
packages/appview/src/ingester.ts
packages/appview/src/ingestors/firehose.ts
+2
packages/appview/src/ingestors/index.ts
+2
packages/appview/src/ingestors/index.ts
+213
packages/appview/src/ingestors/jetstream.ts
+213
packages/appview/src/ingestors/jetstream.ts
···
···
1
+
import { XyzStatusphereStatus } from '@statusphere/lexicon'
2
+
import pino from 'pino'
3
+
import WebSocket from 'ws'
4
+
5
+
import type { Database } from '#/db'
6
+
7
+
export async function createJetstreamIngester(db: Database) {
8
+
const logger = pino({ name: 'jetstream ingestion' })
9
+
10
+
const cursor = await db
11
+
.selectFrom('cursor')
12
+
.where('id', '=', 2)
13
+
.select('seq')
14
+
.executeTakeFirst()
15
+
16
+
logger.info(`start cursor: ${cursor?.seq}`)
17
+
18
+
// For throttling cursor writes
19
+
let lastCursorWrite = 0
20
+
21
+
return new Jetstream<XyzStatusphereStatus.Record>({
22
+
logger,
23
+
cursor: cursor?.seq || undefined,
24
+
setCursor: async (seq) => {
25
+
const now = Date.now()
26
+
27
+
if (now - lastCursorWrite >= 30000) {
28
+
lastCursorWrite = now
29
+
logger.info(`writing cursor: ${seq}`)
30
+
await db
31
+
.updateTable('cursor')
32
+
.set({ seq })
33
+
.where('id', '=', 2)
34
+
.execute()
35
+
}
36
+
},
37
+
handleEvent: async (evt) => {
38
+
// ignore account and identity events
39
+
if (
40
+
evt.kind !== 'commit' ||
41
+
evt.commit.collection !== 'xyz.statusphere.status'
42
+
)
43
+
return
44
+
45
+
const now = new Date()
46
+
const uri = `at://${evt.did}/${evt.commit.collection}/${evt.commit.rkey}`
47
+
48
+
if (
49
+
(evt.commit.operation === 'create' ||
50
+
evt.commit.operation === 'update') &&
51
+
XyzStatusphereStatus.isRecord(evt.commit.record)
52
+
) {
53
+
const validatedRecord = XyzStatusphereStatus.validateRecord(
54
+
evt.commit.record,
55
+
)
56
+
if (!validatedRecord.success) return
57
+
58
+
// Store the status in our SQLite
59
+
await db
60
+
.insertInto('status')
61
+
.values({
62
+
uri,
63
+
authorDid: evt.did,
64
+
status: validatedRecord.value.status,
65
+
createdAt: validatedRecord.value.createdAt,
66
+
indexedAt: now.toISOString(),
67
+
})
68
+
.onConflict((oc) =>
69
+
oc.column('uri').doUpdateSet({
70
+
status: validatedRecord.value.status,
71
+
indexedAt: now.toISOString(),
72
+
}),
73
+
)
74
+
.execute()
75
+
} else if (evt.commit.operation === 'delete') {
76
+
// Remove the status from our SQLite
77
+
await db.deleteFrom('status').where('uri', '=', uri).execute()
78
+
}
79
+
},
80
+
onError: (err) => {
81
+
logger.error({ err }, 'error during jetstream ingestion')
82
+
},
83
+
wantedCollections: ['xyz.statusphere.status'],
84
+
})
85
+
}
86
+
87
+
export class Jetstream<T> {
88
+
private logger: pino.Logger
89
+
private handleEvent: (evt: JetstreamEvent<T>) => Promise<void>
90
+
private onError: (err: unknown) => void
91
+
private setCursor?: (seq: number) => Promise<void>
92
+
private cursor?: number
93
+
private ws?: WebSocket
94
+
private isStarted = false
95
+
private wantedCollections: string[]
96
+
97
+
constructor({
98
+
logger,
99
+
cursor,
100
+
setCursor,
101
+
handleEvent,
102
+
onError,
103
+
wantedCollections,
104
+
}: {
105
+
logger: pino.Logger
106
+
cursor?: number
107
+
setCursor?: (seq: number) => Promise<void>
108
+
handleEvent: (evt: any) => Promise<void>
109
+
onError: (err: any) => void
110
+
wantedCollections: string[]
111
+
}) {
112
+
this.logger = logger
113
+
this.cursor = cursor
114
+
this.setCursor = setCursor
115
+
this.handleEvent = handleEvent
116
+
this.onError = onError
117
+
this.wantedCollections = wantedCollections
118
+
}
119
+
120
+
constructUrlWithQuery = (): string => {
121
+
const params = new URLSearchParams()
122
+
params.append('wantedCollections', this.wantedCollections.join(','))
123
+
if (this.cursor !== undefined) {
124
+
params.append('cursor', this.cursor.toString())
125
+
}
126
+
return `wss://jetstream.mozzius.dev/subscribe?${params.toString()}`
127
+
}
128
+
129
+
start() {
130
+
if (this.isStarted) return
131
+
this.isStarted = true
132
+
this.ws = new WebSocket(this.constructUrlWithQuery())
133
+
134
+
this.ws.on('open', () => {
135
+
this.logger.info('Jetstream connection opened.')
136
+
})
137
+
138
+
this.ws.on('message', async (data) => {
139
+
try {
140
+
const event: JetstreamEvent<T> = JSON.parse(data.toString())
141
+
142
+
// Update cursor if provided
143
+
if (event.time_us !== undefined && this.setCursor) {
144
+
await this.setCursor(event.time_us)
145
+
}
146
+
147
+
await this.handleEvent(event)
148
+
} catch (err) {
149
+
this.onError(err)
150
+
}
151
+
})
152
+
153
+
this.ws.on('error', (err) => {
154
+
this.onError(err)
155
+
})
156
+
157
+
this.ws.on('close', (code, reason) => {
158
+
this.logger.error(`Jetstream closed. Code: ${code}, Reason: ${reason}`)
159
+
this.isStarted = false
160
+
})
161
+
}
162
+
163
+
destroy() {
164
+
if (this.ws) {
165
+
this.ws.close()
166
+
this.isStarted = false
167
+
}
168
+
}
169
+
}
170
+
171
+
type JetstreamEvent<T> = {
172
+
did: string
173
+
time_us: number
174
+
} & (CommitEvent<T> | AccountEvent | IdentityEvent)
175
+
176
+
type CommitEvent<T> = {
177
+
kind: 'commit'
178
+
commit:
179
+
| {
180
+
operation: 'create' | 'update'
181
+
record: T
182
+
rev: string
183
+
collection: string
184
+
rkey: string
185
+
cid: string
186
+
}
187
+
| {
188
+
operation: 'delete'
189
+
rev: string
190
+
collection: string
191
+
rkey: string
192
+
}
193
+
}
194
+
195
+
type IdentityEvent = {
196
+
kind: 'identity'
197
+
identity: {
198
+
did: string
199
+
handle: string
200
+
seq: number
201
+
time: string
202
+
}
203
+
}
204
+
205
+
type AccountEvent = {
206
+
kind: 'account'
207
+
account: {
208
+
active: boolean
209
+
did: string
210
+
seq: number
211
+
time: string
212
+
}
213
+
}
-3
packages/appview/src/lexicons/lexicons.ts
-3
packages/appview/src/lexicons/lexicons.ts
-1
packages/appview/src/lexicons/types/xyz/statusphere/getStatuses.ts
-1
packages/appview/src/lexicons/types/xyz/statusphere/getStatuses.ts
+1
-1
packages/client/src/components/StatusForm.tsx
+1
-1
packages/client/src/components/StatusForm.tsx
+11
-2
packages/client/src/components/StatusList.tsx
+11
-2
packages/client/src/components/StatusList.tsx
···
2
import { useQuery } from '@tanstack/react-query'
3
4
import api from '#/services/api'
5
6
const StatusList = () => {
7
// Use React Query to fetch and cache statuses
···
23
24
// Destructure data
25
const statuses = data?.statuses || []
26
27
if (isPending && !data) {
28
return (
29
-
<div className="py-4 text-center text-gray-500 dark:text-gray-400">
30
-
Loading statuses...
31
</div>
32
)
33
}
···
2
import { useQuery } from '@tanstack/react-query'
3
4
import api from '#/services/api'
5
+
import { STATUS_OPTIONS } from './StatusForm'
6
7
const StatusList = () => {
8
// Use React Query to fetch and cache statuses
···
24
25
// Destructure data
26
const statuses = data?.statuses || []
27
+
28
+
// Get a random emoji from the STATUS_OPTIONS array
29
+
const randomEmoji = STATUS_OPTIONS[Math.floor(Math.random() * STATUS_OPTIONS.length)]
30
31
if (isPending && !data) {
32
return (
33
+
<div className="py-8 text-center">
34
+
<div className="text-5xl mb-2 animate-pulse inline-block">
35
+
{randomEmoji}
36
+
</div>
37
+
<div className="text-gray-500 dark:text-gray-400">
38
+
Loading statuses...
39
+
</div>
40
</div>
41
)
42
}
+7
-10
packages/client/src/pages/HomePage.tsx
+7
-10
packages/client/src/pages/HomePage.tsx
···
1
import Header from '#/components/Header'
2
-
import StatusForm from '#/components/StatusForm'
3
import StatusList from '#/components/StatusList'
4
import { useAuth } from '#/hooks/useAuth'
5
6
const HomePage = () => {
7
const { user, loading, error } = useAuth()
8
9
if (loading) {
10
return (
11
-
<div className="flex justify-center items-center py-16">
12
-
<div className="text-center p-6">
13
-
<h2 className="text-2xl font-semibold mb-2 text-gray-800 dark:text-gray-200">
14
-
Loading Statusphere...
15
-
</h2>
16
-
<p className="text-gray-600 dark:text-gray-400">
17
-
Setting up your experience
18
-
</p>
19
-
</div>
20
</div>
21
)
22
}
···
1
import Header from '#/components/Header'
2
+
import StatusForm, { STATUS_OPTIONS } from '#/components/StatusForm'
3
import StatusList from '#/components/StatusList'
4
import { useAuth } from '#/hooks/useAuth'
5
6
const HomePage = () => {
7
const { user, loading, error } = useAuth()
8
9
+
// Get a random emoji from the STATUS_OPTIONS array
10
+
const randomEmoji =
11
+
STATUS_OPTIONS[Math.floor(Math.random() * STATUS_OPTIONS.length)]
12
+
13
if (loading) {
14
return (
15
+
<div className="flex justify-center items-center h-[80vh]">
16
+
<div className="text-9xl animate-pulse">{randomEmoji}</div>
17
</div>
18
)
19
}
+11
packages/client/vite.config.ts
+11
packages/client/vite.config.ts
···
21
'^/(xrpc|oauth|client-metadata\.json)/.*': {
22
target: 'http://localhost:3001',
23
changeOrigin: true,
24
+
configure: (proxy, _options) => {
25
+
proxy.on('error', (err, _req, _res) => {
26
+
console.log('PROXY ERROR', err);
27
+
});
28
+
proxy.on('proxyReq', (proxyReq, req, _res) => {
29
+
console.log('PROXY REQUEST', req.method, req.url);
30
+
});
31
+
proxy.on('proxyRes', (proxyRes, req, _res) => {
32
+
console.log('PROXY RESPONSE', req.method, req.url, proxyRes.statusCode);
33
+
});
34
+
},
35
},
36
},
37
},
-3
packages/lexicon/src/lexicons.ts
-3
packages/lexicon/src/lexicons.ts
-1
packages/lexicon/src/types/xyz/statusphere/getStatuses.ts
-1
packages/lexicon/src/types/xyz/statusphere/getStatuses.ts