evbogue.com
a demonstration replicated social networking web app built with anproto
wiredove.net/
social
ed25519
protocols
Improve initial sync feed handling
+315
-95
+240
-67
adder.js
+240
-67
adder.js
···
1
1
import { render } from './render.js'
2
2
import { apds } from 'apds'
3
3
4
+
const getController = () => {
5
+
if (!window.__feedController) {
6
+
window.__feedController = {
7
+
feeds: new Map(),
8
+
getFeed(src) {
9
+
const state = this.feeds.get(src)
10
+
if (state && state.container && !document.body.contains(state.container)) {
11
+
this.feeds.delete(src)
12
+
return null
13
+
}
14
+
return state || null
15
+
}
16
+
}
17
+
}
18
+
return window.__feedController
19
+
}
20
+
21
+
const normalizeTimestamp = (ts) => {
22
+
const value = Number.parseInt(ts, 10)
23
+
return Number.isNaN(value) ? 0 : value
24
+
}
25
+
4
26
const addPosts = async (posts, div) => {
5
27
for (const post of posts) {
6
28
const ts = post.ts || (post.opened ? Number.parseInt(post.opened.substring(0, 13), 10) : 0)
···
27
49
return 0
28
50
}
29
51
30
-
const isAscending = (log) => {
31
-
if (!log || log.length < 2) { return false }
32
-
let left = 0
33
-
let right = 0
52
+
const sortDesc = (a, b) => b.ts - a.ts
53
+
54
+
const buildEntries = (log) => {
55
+
if (!log) { return [] }
56
+
const entries = []
57
+
const seen = new Set()
34
58
for (const post of log) {
35
-
left = getTimestamp(post)
36
-
if (left) { break }
59
+
if (!post || !post.hash) { continue }
60
+
if (seen.has(post.hash)) { continue }
61
+
seen.add(post.hash)
62
+
const ts = getTimestamp(post)
63
+
entries.push({ hash: post.hash, ts })
64
+
}
65
+
entries.sort(sortDesc)
66
+
return entries
67
+
}
68
+
69
+
const insertEntry = (state, entry) => {
70
+
if (!entry || !entry.hash || !entry.ts) { return -1 }
71
+
if (state.seen.has(entry.hash)) { return -1 }
72
+
const list = state.entries
73
+
const prevLen = list.length
74
+
let lo = 0
75
+
let hi = list.length
76
+
while (lo < hi) {
77
+
const mid = Math.floor((lo + hi) / 2)
78
+
if (list[mid].ts >= entry.ts) {
79
+
lo = mid + 1
80
+
} else {
81
+
hi = mid
82
+
}
37
83
}
38
-
for (let i = log.length - 1; i >= 0; i--) {
39
-
right = getTimestamp(log[i])
40
-
if (right) { break }
84
+
list.splice(lo, 0, entry)
85
+
state.seen.add(entry.hash)
86
+
if (lo <= state.cursor) {
87
+
if (state.cursor === prevLen && lo === prevLen) { return lo }
88
+
state.cursor += 1
89
+
}
90
+
return lo
91
+
}
92
+
93
+
const isAtTop = () => {
94
+
const scrollEl = document.scrollingElement || document.documentElement || document.body
95
+
const scrollTop = scrollEl.scrollTop || window.scrollY || 0
96
+
return scrollTop <= 10
97
+
}
98
+
99
+
const ensureBanner = (state) => {
100
+
if (state.banner && state.banner.parentNode === state.container) { return state.banner }
101
+
const banner = document.createElement('div')
102
+
banner.className = 'new-posts-banner'
103
+
banner.style.display = 'none'
104
+
const button = document.createElement('button')
105
+
button.type = 'button'
106
+
button.className = 'new-posts-button'
107
+
button.addEventListener('click', async () => {
108
+
await flushPending(state)
109
+
})
110
+
banner.appendChild(button)
111
+
state.container.insertBefore(banner, state.container.firstChild)
112
+
state.banner = banner
113
+
state.bannerButton = button
114
+
return banner
115
+
}
116
+
117
+
const updateBanner = (state) => {
118
+
if (!state.banner || !state.bannerButton) { return }
119
+
const count = state.pending.length
120
+
if (!count) {
121
+
state.banner.style.display = 'none'
122
+
return
123
+
}
124
+
state.bannerButton.textContent = `Show ${count} new post${count === 1 ? '' : 's'}`
125
+
state.banner.style.display = 'block'
126
+
}
127
+
128
+
const renderEntry = async (state, entry) => {
129
+
const div = render.insertByTimestamp(state.container, entry.hash, entry.ts)
130
+
if (!div) { return }
131
+
if (entry.blob) {
132
+
await render.blob(entry.blob)
133
+
} else {
134
+
const sig = await apds.get(entry.hash)
135
+
if (sig) { await render.blob(sig) }
136
+
}
137
+
state.rendered.add(entry.hash)
138
+
}
139
+
140
+
const flushPending = async (state) => {
141
+
if (!state.pending.length) { return }
142
+
const pending = state.pending.slice().sort(sortDesc)
143
+
state.pending = []
144
+
updateBanner(state)
145
+
for (const entry of pending) {
146
+
await renderEntry(state, entry)
147
+
state.latestVisibleTs = Math.max(state.latestVisibleTs || 0, entry.ts)
148
+
if (!state.oldestVisibleTs) { state.oldestVisibleTs = entry.ts }
149
+
}
150
+
}
151
+
152
+
const enqueuePost = async (state, entry) => {
153
+
if (!entry || !entry.hash || !entry.ts) { return }
154
+
insertEntry(state, entry)
155
+
if (!state.latestVisibleTs) {
156
+
await renderEntry(state, entry)
157
+
state.latestVisibleTs = entry.ts
158
+
state.oldestVisibleTs = entry.ts
159
+
return
160
+
}
161
+
if (entry.ts < state.oldestVisibleTs && state.rendered.size < state.pageSize) {
162
+
await renderEntry(state, entry)
163
+
state.oldestVisibleTs = entry.ts
164
+
return
165
+
}
166
+
const inWindow = state.oldestVisibleTs && entry.ts >= state.oldestVisibleTs && entry.ts <= state.latestVisibleTs
167
+
if (entry.ts > state.latestVisibleTs) {
168
+
if (isAtTop()) {
169
+
await renderEntry(state, entry)
170
+
state.latestVisibleTs = entry.ts
171
+
if (!state.oldestVisibleTs) { state.oldestVisibleTs = entry.ts }
172
+
} else {
173
+
state.pending.push(entry)
174
+
updateBanner(state)
175
+
}
176
+
return
177
+
}
178
+
if (inWindow) {
179
+
await renderEntry(state, entry)
180
+
state.latestVisibleTs = Math.max(state.latestVisibleTs, entry.ts)
181
+
state.oldestVisibleTs = Math.min(state.oldestVisibleTs || entry.ts, entry.ts)
41
182
}
42
-
return left && right ? left < right : false
183
+
}
184
+
185
+
window.__feedEnqueue = async (src, entry) => {
186
+
const controller = getController()
187
+
const state = controller.getFeed(src)
188
+
if (!state) { return false }
189
+
await enqueuePost(state, entry)
190
+
return true
43
191
}
44
192
45
193
export const adder = (log, src, div) => {
46
-
if (log && log[0]) {
47
-
let index = 0
48
-
const ascending = isAscending(log)
49
-
let loading = false
50
-
let armed = false
51
-
const sentinelId = 'scroll-sentinel'
194
+
if (!div) { return }
195
+
const pageSize = 25
196
+
const entries = buildEntries(log || [])
197
+
let loading = false
198
+
let armed = false
199
+
const sentinelId = 'scroll-sentinel'
200
+
201
+
let posts = []
202
+
const state = {
203
+
src,
204
+
container: div,
205
+
entries,
206
+
cursor: 0,
207
+
seen: new Set(entries.map(entry => entry.hash)),
208
+
rendered: new Set(),
209
+
pending: [],
210
+
pageSize,
211
+
latestVisibleTs: 0,
212
+
oldestVisibleTs: 0,
213
+
banner: null,
214
+
bannerButton: null
215
+
}
216
+
getController().feeds.set(src, state)
217
+
ensureBanner(state)
52
218
53
-
let posts = []
54
-
const takeSlice = () => {
55
-
if (ascending) {
56
-
const end = log.length - index
57
-
const start = Math.max(0, end - 25)
58
-
posts = log.slice(start, end).reverse()
59
-
} else {
60
-
posts = log.slice(index, index + 25)
219
+
const takeSlice = () => {
220
+
posts = []
221
+
if (state.cursor >= entries.length) { return posts }
222
+
let idx = state.cursor
223
+
while (idx < entries.length && posts.length < pageSize) {
224
+
const entry = entries[idx]
225
+
if (!state.rendered.has(entry.hash)) {
226
+
posts.push(entry)
61
227
}
62
-
index = index + 25
63
-
return posts
228
+
idx += 1
64
229
}
230
+
state.cursor = idx
231
+
return posts
232
+
}
65
233
66
-
const ensureSentinel = () => {
67
-
let sentinel = document.getElementById(sentinelId)
68
-
if (!sentinel) {
69
-
sentinel = document.createElement('div')
70
-
sentinel.id = sentinelId
71
-
sentinel.style.height = '1px'
234
+
const ensureSentinel = () => {
235
+
let sentinel = document.getElementById(sentinelId)
236
+
if (!sentinel) {
237
+
sentinel = document.createElement('div')
238
+
sentinel.id = sentinelId
239
+
sentinel.style.height = '1px'
240
+
}
241
+
if (sentinel.parentNode && sentinel.parentNode !== div) {
242
+
sentinel.parentNode.removeChild(sentinel)
243
+
}
244
+
div.appendChild(sentinel)
245
+
return sentinel
246
+
}
247
+
248
+
const loadNext = async () => {
249
+
if (loading) { return }
250
+
if (window.location.hash.substring(1) !== src) { return }
251
+
loading = true
252
+
try {
253
+
const next = takeSlice()
254
+
if (!next.length) { return false }
255
+
await addPosts(next, div)
256
+
for (const entry of next) {
257
+
state.rendered.add(entry.hash)
72
258
}
73
-
if (sentinel.parentNode && sentinel.parentNode !== div) {
74
-
sentinel.parentNode.removeChild(sentinel)
259
+
if (!state.latestVisibleTs && next[0]) {
260
+
state.latestVisibleTs = normalizeTimestamp(next[0].ts)
75
261
}
76
-
div.appendChild(sentinel)
77
-
return sentinel
78
-
}
79
-
80
-
const loadNext = async () => {
81
-
if (loading) { return }
82
-
if (window.location.hash.substring(1) !== src) { return }
83
-
loading = true
84
-
try {
85
-
const next = takeSlice()
86
-
if (!next.length) { return false }
87
-
await addPosts(next, div)
88
-
ensureSentinel()
89
-
return true
90
-
} finally {
91
-
loading = false
262
+
if (next[next.length - 1]) {
263
+
state.oldestVisibleTs = normalizeTimestamp(next[next.length - 1].ts)
92
264
}
265
+
ensureSentinel()
266
+
return true
267
+
} finally {
268
+
loading = false
93
269
}
94
-
95
-
void loadNext()
96
-
const armScroll = () => {
97
-
armed = true
98
-
}
99
-
window.addEventListener('scroll', armScroll, { passive: true, once: true })
100
-
const sentinel = ensureSentinel()
101
-
const observer = new IntersectionObserver(async (entries) => {
102
-
const entry = entries[0]
103
-
if (!entry || !entry.isIntersecting) { return }
104
-
if (!armed) { return }
105
-
const hasMore = await loadNext()
106
-
if (hasMore === false) {
107
-
observer.disconnect()
108
-
}
109
-
}, { root: null, rootMargin: '0px 0px', threshold: 0 })
270
+
}
110
271
111
-
observer.observe(sentinel)
272
+
void loadNext()
273
+
const armScroll = () => {
274
+
armed = true
112
275
}
276
+
window.addEventListener('scroll', armScroll, { passive: true, once: true })
277
+
const sentinel = ensureSentinel()
278
+
const observer = new IntersectionObserver(async (entries) => {
279
+
const entry = entries[0]
280
+
if (!entry || !entry.isIntersecting) { return }
281
+
if (!armed) { return }
282
+
await loadNext()
283
+
}, { root: null, rootMargin: '0px 0px', threshold: 0 })
284
+
285
+
observer.observe(sentinel)
113
286
}
+14
-3
composer.js
+14
-3
composer.js
···
120
120
const scroller = document.getElementById('scroller')
121
121
const opened = await apds.open(signed)
122
122
const ts = opened ? opened.substring(0, 13) : Date.now().toString()
123
-
const placeholder = render.insertByTimestamp(scroller, hash, ts)
124
-
if (placeholder) {
125
-
await render.blob(signed)
123
+
if (window.__feedEnqueue) {
124
+
const src = window.location.hash.substring(1)
125
+
const queued = await window.__feedEnqueue(src, { hash, ts: Number.parseInt(ts, 10), blob: signed })
126
+
if (!queued) {
127
+
const placeholder = render.insertByTimestamp(scroller, hash, ts)
128
+
if (placeholder) {
129
+
await render.blob(signed)
130
+
}
131
+
}
132
+
} else {
133
+
const placeholder = render.insertByTimestamp(scroller, hash, ts)
134
+
if (placeholder) {
135
+
await render.blob(signed)
136
+
}
126
137
}
127
138
overlay.remove()
128
139
}
+2
connect.js
+2
connect.js
···
1
1
import { apds } from 'apds'
2
2
import { makeRoom } from './gossip.js'
3
3
import { makeWs} from './websocket.js'
4
+
import { send } from './send.js'
4
5
5
6
await apds.start('wiredovedbversion1')
6
7
···
9
10
//await makeWs('wss://apds.anproto.com/')
10
11
makeWs('wss://pub.wiredove.net/')
11
12
makeRoom('wiredovev1')
13
+
send('evSFOKnXaF9ZWSsff8bVfXP6+XnGZUj8XNp6bca590k=')
12
14
}
+53
-19
network_queue.js
+53
-19
network_queue.js
···
1
1
const SEND_DELAY_MS = 100
2
+
const HASH_RETRY_MS = 800
2
3
const queue = []
3
4
const pending = new Map()
4
5
let drainTimer = null
5
6
let draining = false
7
+
let nextHashTarget = 'ws'
6
8
7
9
const senders = {
8
10
ws: null,
···
16
18
return null
17
19
}
18
20
19
-
const normalizeTargets = (targets) => {
20
-
if (!targets || targets === 'both') { return { ws: true, gossip: true } }
21
-
if (targets === 'ws') { return { ws: true, gossip: false } }
22
-
if (targets === 'gossip') { return { ws: false, gossip: true } }
23
-
return { ws: true, gossip: true }
24
-
}
21
+
const isHash = (msg) => typeof msg === 'string' && msg.length === 44
22
+
23
+
const flipTarget = (target) => (target === 'ws' ? 'gossip' : 'ws')
25
24
26
25
export const registerNetworkSenders = (config = {}) => {
27
26
if (config.sendWs) { senders.ws = config.sendWs }
···
46
45
queue.splice(index, 1)
47
46
}
48
47
48
+
const pickHashTarget = (item) => {
49
+
const wsReady = isTargetReady('ws')
50
+
const gossipReady = isTargetReady('gossip')
51
+
if (!item.sent.ws && !item.sent.gossip) {
52
+
const preferred = nextHashTarget
53
+
const preferredReady = preferred === 'ws' ? wsReady : gossipReady
54
+
if (preferredReady) { return preferred }
55
+
const fallback = flipTarget(preferred)
56
+
const fallbackReady = fallback === 'ws' ? wsReady : gossipReady
57
+
if (fallbackReady) { return fallback }
58
+
return null
59
+
}
60
+
if (item.sent.ws && item.sent.gossip) { return null }
61
+
const firstTarget = item.firstTarget
62
+
if (!firstTarget) { return null }
63
+
const otherTarget = flipTarget(firstTarget)
64
+
const otherReady = otherTarget === 'ws' ? wsReady : gossipReady
65
+
if (!item.sent[otherTarget] && otherReady && Date.now() - item.sentAt[firstTarget] >= HASH_RETRY_MS) {
66
+
return otherTarget
67
+
}
68
+
return null
69
+
}
70
+
49
71
const drainQueue = () => {
50
72
drainTimer = null
51
73
if (draining) {
···
56
78
try {
57
79
for (let i = 0; i < queue.length; i += 1) {
58
80
const item = queue[i]
59
-
const wsReady = item.targets.ws && !item.sent.ws && isTargetReady('ws')
60
-
const gossipReady = item.targets.gossip && !item.sent.gossip && isTargetReady('gossip')
81
+
if (item.kind === 'hash') {
82
+
const target = pickHashTarget(item)
83
+
if (!target) { continue }
84
+
sendToTarget(target, item.msg)
85
+
item.sent[target] = true
86
+
item.sentAt[target] = Date.now()
87
+
if (!item.firstTarget) {
88
+
item.firstTarget = target
89
+
nextHashTarget = flipTarget(target)
90
+
}
91
+
if (item.sent.ws && item.sent.gossip) {
92
+
cleanupItem(item, i)
93
+
}
94
+
break
95
+
}
96
+
const wsReady = !item.sent.ws && isTargetReady('ws')
97
+
const gossipReady = !item.sent.gossip && isTargetReady('gossip')
61
98
if (!wsReady && !gossipReady) { continue }
62
99
if (wsReady) {
63
100
sendToTarget('ws', item.msg)
···
67
104
sendToTarget('gossip', item.msg)
68
105
item.sent.gossip = true
69
106
}
70
-
const wsDone = !item.targets.ws || item.sent.ws
71
-
const gossipDone = !item.targets.gossip || item.sent.gossip
72
-
if (wsDone && gossipDone) {
73
-
cleanupItem(item, i)
74
-
}
107
+
const wsDone = item.sent.ws
108
+
const gossipDone = item.sent.gossip
109
+
if (wsDone && gossipDone) { cleanupItem(item, i) }
75
110
break
76
111
}
77
112
} finally {
···
82
117
}
83
118
}
84
119
85
-
export const queueSend = (msg, targets = 'both') => {
120
+
export const queueSend = (msg) => {
86
121
const key = getKey(msg)
87
-
const targetFlags = normalizeTargets(targets)
88
122
if (key && pending.has(key)) {
89
123
const item = pending.get(key)
90
-
item.targets.ws = item.targets.ws || targetFlags.ws
91
-
item.targets.gossip = item.targets.gossip || targetFlags.gossip
92
124
if (!drainTimer) { drainTimer = setTimeout(drainQueue, 0) }
93
125
return
94
126
}
95
127
const item = {
96
128
msg,
97
129
key,
98
-
targets: targetFlags,
99
-
sent: { ws: false, gossip: false }
130
+
kind: isHash(msg) ? 'hash' : 'blob',
131
+
sent: { ws: false, gossip: false },
132
+
sentAt: { ws: 0, gossip: 0 },
133
+
firstTarget: null
100
134
}
101
135
queue.push(item)
102
136
if (key) { pending.set(key, item) }
+1
-1
send.js
+1
-1
send.js
+5
-5
settings.js
+5
-5
settings.js
···
77
77
if (log) {
78
78
const ar = []
79
79
for (const msg of log) {
80
-
queueSend(msg.sig, 'ws')
80
+
queueSend(msg.sig)
81
81
if (msg.text) {
82
-
queueSend(msg.text, 'ws')
82
+
queueSend(msg.text)
83
83
const yaml = await apds.parseYaml(msg.text)
84
84
if (yaml.image && !ar.includes(yaml.image)) {
85
85
const get = await apds.get(yaml.image)
86
86
if (get) {
87
-
queueSend(get, 'ws')
87
+
queueSend(get)
88
88
ar.push(yaml.image)
89
89
}
90
90
}
···
95
95
const src = image.match(/!\[.*?\]\((.*?)\)/)[1]
96
96
const imgBlob = await apds.get(src)
97
97
if (imgBlob && !ar.includes(src)) {
98
-
queueSend(imgBlob, 'ws')
98
+
queueSend(imgBlob)
99
99
ar.push(src)
100
100
}
101
101
}
···
104
104
}
105
105
if (!msg.text) {
106
106
const get = await apds.get(msg.opened.substring(13))
107
-
if (get) { queueSend(get, 'ws') }
107
+
if (get) { queueSend(get) }
108
108
}
109
109
}
110
110
}