tangled.org
+9
-1
appview/db/jetstream.go
+9
-1
appview/db/jetstream.go
···
1
package db
2
3
func (d *DB) SaveLastTimeUs(lastTimeUs int64) error {
4
+
_, err := d.db.Exec(`insert into _jetstream (last_time_us) values (?)`, lastTimeUs)
5
return err
6
+
}
7
+
8
+
func (d *DB) UpdateLastTimeUs(lastTimeUs int64) error {
9
+
_, err := d.db.Exec(`update _jetstream set last_time_us = ? where rowid = 1`, lastTimeUs)
10
+
if err != nil {
11
+
return err
12
+
}
13
+
return nil
14
}
15
16
func (d *DB) GetLastTimeUs() (int64, error) {
+1
-1
appview/state/middleware.go
+1
-1
appview/state/middleware.go
+1
-1
appview/state/settings.go
+1
-1
appview/state/settings.go
+4
-2
appview/state/state.go
+4
-2
appview/state/state.go
···
8
"encoding/json"
9
"fmt"
10
"log"
11
"net/http"
12
"strings"
13
"time"
···
59
60
resolver := appview.NewResolver()
61
62
-
jc, err := jetstream.NewJetstreamClient("appview", []string{tangled.GraphFollowNSID}, nil, db, false)
63
if err != nil {
64
return nil, fmt.Errorf("failed to create jetstream client: %w", err)
65
}
···
83
if err != nil {
84
return fmt.Errorf("failed to add follow to db: %w", err)
85
}
86
-
return db.SaveLastTimeUs(e.TimeUS)
87
}
88
89
return nil
···
125
126
resolved, err := s.resolver.ResolveIdent(ctx, handle)
127
if err != nil {
128
s.pages.Notice(w, "login-msg", fmt.Sprintf("\"%s\" is an invalid handle.", handle))
129
return
130
}
···
8
"encoding/json"
9
"fmt"
10
"log"
11
+
"log/slog"
12
"net/http"
13
"strings"
14
"time"
···
60
61
resolver := appview.NewResolver()
62
63
+
jc, err := jetstream.NewJetstreamClient("appview", []string{tangled.GraphFollowNSID}, nil, slog.Default(), db, false)
64
if err != nil {
65
return nil, fmt.Errorf("failed to create jetstream client: %w", err)
66
}
···
84
if err != nil {
85
return fmt.Errorf("failed to add follow to db: %w", err)
86
}
87
+
return db.UpdateLastTimeUs(e.TimeUS)
88
}
89
90
return nil
···
126
127
resolved, err := s.resolver.ResolveIdent(ctx, handle)
128
if err != nil {
129
+
log.Println("failed to resolve handle:", err)
130
s.pages.Notice(w, "login-msg", fmt.Sprintf("\"%s\" is an invalid handle.", handle))
131
return
132
}
+1
-1
cmd/knotserver/main.go
+1
-1
cmd/knotserver/main.go
+44
-29
jetstream/jetstream.go
+44
-29
jetstream/jetstream.go
···
3
import (
4
"context"
5
"fmt"
6
"sync"
7
"time"
8
···
15
type DB interface {
16
GetLastTimeUs() (int64, error)
17
SaveLastTimeUs(int64) error
18
}
19
20
type JetstreamClient struct {
21
cfg *client.ClientConfig
22
client *client.Client
23
ident string
24
25
-
db DB
26
-
reconnectCh chan struct{}
27
-
waitForDid bool
28
-
mu sync.RWMutex
29
}
30
31
func (j *JetstreamClient) AddDid(did string) {
···
35
j.mu.Lock()
36
j.cfg.WantedDids = append(j.cfg.WantedDids, did)
37
j.mu.Unlock()
38
-
j.reconnectCh <- struct{}{}
39
}
40
41
func (j *JetstreamClient) UpdateDids(dids []string) {
42
j.mu.Lock()
43
for _, did := range dids {
44
if did != "" {
45
}
46
-
j.cfg.WantedDids = append(j.cfg.WantedDids, did)
47
}
48
j.mu.Unlock()
49
-
j.reconnectCh <- struct{}{}
50
}
51
52
-
func NewJetstreamClient(ident string, collections []string, cfg *client.ClientConfig, db DB, waitForDid bool) (*JetstreamClient, error) {
53
if cfg == nil {
54
cfg = client.DefaultClientConfig()
55
cfg.WebsocketURL = "wss://jetstream1.us-west.bsky.network/subscribe"
···
60
cfg: cfg,
61
ident: ident,
62
db: db,
63
64
// This will make the goroutine in StartJetstream wait until
65
// cfg.WantedDids has been populated, typically using UpdateDids.
66
-
waitForDid: waitForDid,
67
-
reconnectCh: make(chan struct{}, 1),
68
}, nil
69
}
70
71
// StartJetstream starts the jetstream client and processes events using the provided processFunc.
72
// The caller is responsible for saving the last time_us to the database (just use your db.SaveLastTimeUs).
73
func (j *JetstreamClient) StartJetstream(ctx context.Context, processFunc func(context.Context, *models.Event) error) error {
74
-
logger := log.FromContext(ctx)
75
76
sched := sequential.NewScheduler(j.ident, logger, processFunc)
77
···
82
j.client = client
83
84
go func() {
85
-
lastTimeUs := j.getLastTimeUs(ctx)
86
if j.waitForDid {
87
for len(j.cfg.WantedDids) == 0 {
88
time.Sleep(time.Second)
89
}
90
}
91
logger.Info("done waiting for did")
92
-
j.connectAndRead(ctx, &lastTimeUs)
93
}()
94
95
return nil
96
}
97
98
-
func (j *JetstreamClient) connectAndRead(ctx context.Context, cursor *int64) {
99
l := log.FromContext(ctx)
100
for {
101
select {
102
-
case <-j.reconnectCh:
103
-
l.Info("(re)connecting jetstream client")
104
-
j.client.Scheduler.Shutdown()
105
-
if err := j.client.ConnectAndRead(ctx, cursor); err != nil {
106
-
l.Error("error reading jetstream", "error", err)
107
-
}
108
-
default:
109
-
if err := j.client.ConnectAndRead(ctx, cursor); err != nil {
110
-
l.Error("error reading jetstream", "error", err)
111
-
}
112
}
113
}
114
}
115
116
-
func (j *JetstreamClient) getLastTimeUs(ctx context.Context) int64 {
117
l := log.FromContext(ctx)
118
lastTimeUs, err := j.db.GetLastTimeUs()
119
if err != nil {
···
121
lastTimeUs = time.Now().UnixMicro()
122
err = j.db.SaveLastTimeUs(lastTimeUs)
123
if err != nil {
124
-
l.Error("failed to save last time us")
125
}
126
}
127
···
129
if time.Now().UnixMicro()-lastTimeUs > 7*24*60*60*1000*1000 {
130
lastTimeUs = time.Now().UnixMicro()
131
l.Warn("last time us is older than a week. discarding that and starting from now")
132
-
err = j.db.SaveLastTimeUs(lastTimeUs)
133
if err != nil {
134
-
l.Error("failed to save last time us")
135
}
136
}
137
138
l.Info("found last time_us", "time_us", lastTimeUs)
139
-
return lastTimeUs
140
}
···
3
import (
4
"context"
5
"fmt"
6
+
"log/slog"
7
"sync"
8
"time"
9
···
16
type DB interface {
17
GetLastTimeUs() (int64, error)
18
SaveLastTimeUs(int64) error
19
+
UpdateLastTimeUs(int64) error
20
}
21
22
type JetstreamClient struct {
23
cfg *client.ClientConfig
24
client *client.Client
25
ident string
26
+
l *slog.Logger
27
28
+
db DB
29
+
waitForDid bool
30
+
mu sync.RWMutex
31
+
32
+
cancel context.CancelFunc
33
+
cancelMu sync.Mutex
34
}
35
36
func (j *JetstreamClient) AddDid(did string) {
···
40
j.mu.Lock()
41
j.cfg.WantedDids = append(j.cfg.WantedDids, did)
42
j.mu.Unlock()
43
}
44
45
func (j *JetstreamClient) UpdateDids(dids []string) {
46
j.mu.Lock()
47
for _, did := range dids {
48
if did != "" {
49
+
j.cfg.WantedDids = append(j.cfg.WantedDids, did)
50
}
51
}
52
j.mu.Unlock()
53
+
54
+
j.cancelMu.Lock()
55
+
if j.cancel != nil {
56
+
j.cancel()
57
+
}
58
+
j.cancelMu.Unlock()
59
}
60
61
+
func NewJetstreamClient(ident string, collections []string, cfg *client.ClientConfig, logger *slog.Logger, db DB, waitForDid bool) (*JetstreamClient, error) {
62
if cfg == nil {
63
cfg = client.DefaultClientConfig()
64
cfg.WebsocketURL = "wss://jetstream1.us-west.bsky.network/subscribe"
···
69
cfg: cfg,
70
ident: ident,
71
db: db,
72
+
l: logger,
73
74
// This will make the goroutine in StartJetstream wait until
75
// cfg.WantedDids has been populated, typically using UpdateDids.
76
+
waitForDid: waitForDid,
77
}, nil
78
}
79
80
// StartJetstream starts the jetstream client and processes events using the provided processFunc.
81
// The caller is responsible for saving the last time_us to the database (just use your db.SaveLastTimeUs).
82
func (j *JetstreamClient) StartJetstream(ctx context.Context, processFunc func(context.Context, *models.Event) error) error {
83
+
logger := j.l
84
85
sched := sequential.NewScheduler(j.ident, logger, processFunc)
86
···
91
j.client = client
92
93
go func() {
94
if j.waitForDid {
95
for len(j.cfg.WantedDids) == 0 {
96
time.Sleep(time.Second)
97
}
98
}
99
logger.Info("done waiting for did")
100
+
j.connectAndRead(ctx)
101
}()
102
103
return nil
104
}
105
106
+
func (j *JetstreamClient) connectAndRead(ctx context.Context) {
107
l := log.FromContext(ctx)
108
for {
109
+
cursor := j.getLastTimeUs(ctx)
110
+
111
+
connCtx, cancel := context.WithCancel(ctx)
112
+
j.cancelMu.Lock()
113
+
j.cancel = cancel
114
+
j.cancelMu.Unlock()
115
+
116
+
if err := j.client.ConnectAndRead(connCtx, cursor); err != nil {
117
+
l.Error("error reading jetstream", "error", err)
118
+
}
119
+
120
select {
121
+
case <-ctx.Done():
122
+
l.Info("context done, stopping jetstream")
123
+
return
124
+
case <-connCtx.Done():
125
+
l.Info("connection context done, reconnecting")
126
+
continue
127
}
128
}
129
}
130
131
+
func (j *JetstreamClient) getLastTimeUs(ctx context.Context) *int64 {
132
l := log.FromContext(ctx)
133
lastTimeUs, err := j.db.GetLastTimeUs()
134
if err != nil {
···
136
lastTimeUs = time.Now().UnixMicro()
137
err = j.db.SaveLastTimeUs(lastTimeUs)
138
if err != nil {
139
+
l.Error("failed to save last time us", "error", err)
140
}
141
}
142
···
144
if time.Now().UnixMicro()-lastTimeUs > 7*24*60*60*1000*1000 {
145
lastTimeUs = time.Now().UnixMicro()
146
l.Warn("last time us is older than a week. discarding that and starting from now")
147
+
err = j.db.UpdateLastTimeUs(lastTimeUs)
148
if err != nil {
149
+
l.Error("failed to save last time us", "error", err)
150
}
151
}
152
153
l.Info("found last time_us", "time_us", lastTimeUs)
154
+
return &lastTimeUs
155
}
+9
-1
knotserver/db/jetstream.go
+9
-1
knotserver/db/jetstream.go
···
1
package db
2
3
func (d *DB) SaveLastTimeUs(lastTimeUs int64) error {
4
+
_, err := d.db.Exec(`insert into _jetstream (last_time_us) values (?)`, lastTimeUs)
5
return err
6
+
}
7
+
8
+
func (d *DB) UpdateLastTimeUs(lastTimeUs int64) error {
9
+
_, err := d.db.Exec(`update _jetstream set last_time_us = ? where rowid = 1`, lastTimeUs)
10
+
if err != nil {
11
+
return err
12
+
}
13
+
return nil
14
}
15
16
func (d *DB) GetLastTimeUs() (int64, error) {
+7
-8
knotserver/jetstream.go
+7
-8
knotserver/jetstream.go
···
29
return nil
30
}
31
32
-
func (h *Handle) processKnotMember(ctx context.Context, did string, record tangled.KnotMember) error {
33
l := log.FromContext(ctx)
34
35
if record.Domain != h.c.Server.Hostname {
···
43
return fmt.Errorf("failed to enforce permissions: %w", err)
44
}
45
46
-
l.Info("adding member")
47
if err := h.e.AddMember(ThisServer, record.Member); err != nil {
48
l.Error("failed to add member", "error", err)
49
return fmt.Errorf("failed to add member: %w", err)
···
59
return fmt.Errorf("failed to fetch and add keys: %w", err)
60
}
61
62
h.jc.UpdateDids([]string{did})
63
return nil
64
}
···
129
if err := json.Unmarshal(raw, &record); err != nil {
130
return fmt.Errorf("failed to unmarshal record: %w", err)
131
}
132
-
if err := h.processKnotMember(ctx, did, record); err != nil {
133
return fmt.Errorf("failed to process knot member: %w", err)
134
}
135
-
}
136
-
137
-
lastTimeUs := event.TimeUS
138
-
if err := h.db.SaveLastTimeUs(lastTimeUs); err != nil {
139
-
return fmt.Errorf("failed to save last time us: %w", err)
140
}
141
142
return nil
···
29
return nil
30
}
31
32
+
func (h *Handle) processKnotMember(ctx context.Context, did string, record tangled.KnotMember, eventTime int64) error {
33
l := log.FromContext(ctx)
34
35
if record.Domain != h.c.Server.Hostname {
···
43
return fmt.Errorf("failed to enforce permissions: %w", err)
44
}
45
46
if err := h.e.AddMember(ThisServer, record.Member); err != nil {
47
l.Error("failed to add member", "error", err)
48
return fmt.Errorf("failed to add member: %w", err)
···
58
return fmt.Errorf("failed to fetch and add keys: %w", err)
59
}
60
61
+
lastTimeUs := eventTime + 1
62
+
fmt.Println("lastTimeUs", lastTimeUs)
63
+
if err := h.db.UpdateLastTimeUs(lastTimeUs); err != nil {
64
+
return fmt.Errorf("failed to save last time us: %w", err)
65
+
}
66
h.jc.UpdateDids([]string{did})
67
return nil
68
}
···
133
if err := json.Unmarshal(raw, &record); err != nil {
134
return fmt.Errorf("failed to unmarshal record: %w", err)
135
}
136
+
if err := h.processKnotMember(ctx, did, record, event.TimeUS); err != nil {
137
return fmt.Errorf("failed to process knot member: %w", err)
138
}
139
}
140
141
return nil