+14

cmd/spindle/main.go

··· 1 1 package main 2 2 3 3 + import ( 4 4 + "context" 5 5 + "os" 6 6 + 7 7 + "tangled.sh/tangled.sh/core/log" 8 8 + "tangled.sh/tangled.sh/core/spindle" 9 9 + ) 10 10 + 3 11 func main() { 12 12 + ctx := log.NewContext(context.Background(), "spindle") 13 13 + err := spindle.Run(ctx) 14 14 + if err != nil { 15 15 + log.FromContext(ctx).Error("error running spindle", "error", err) 16 16 + os.Exit(-1) 17 17 + } 4 18 }

+28

spindle/config/config.go

··· 1 1 + package config 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + 6 6 + "github.com/sethvargo/go-envconfig" 7 7 + ) 8 8 + 9 9 + type Server struct { 10 10 + ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:6555"` 11 11 + DBPath string `env:"DB_PATH, default=spindle.db"` 12 12 + Hostname string `env:"HOSTNAME, required"` 13 13 + JetstreamEndpoint string `env:"JETSTREAM_ENDPOINT, default=wss://jetstream1.us-west.bsky.network/subscribe"` 14 14 + } 15 15 + 16 16 + type Config struct { 17 17 + Server Server `env:",prefix=SPINDLE_SERVER_"` 18 18 + } 19 19 + 20 20 + func Load(ctx context.Context) (*Config, error) { 21 21 + var cfg Config 22 22 + err := envconfig.Process(ctx, &cfg) 23 23 + if err != nil { 24 24 + return nil, err 25 25 + } 26 26 + 27 27 + return &cfg, nil 28 28 + }

+56

spindle/db/db.go

··· 1 1 + package db 2 2 + 3 3 + import "database/sql" 4 4 + 5 5 + type DB struct { 6 6 + *sql.DB 7 7 + } 8 8 + 9 9 + func Make(dbPath string) (*DB, error) { 10 10 + db, err := sql.Open("sqlite3", dbPath) 11 11 + if err != nil { 12 12 + return nil, err 13 13 + } 14 14 + 15 15 + _, err = db.Exec(` 16 16 + pragma journal_mode = WAL; 17 17 + pragma synchronous = normal; 18 18 + pragma foreign_keys = on; 19 19 + pragma temp_store = memory; 20 20 + pragma mmap_size = 30000000000; 21 21 + pragma page_size = 32768; 22 22 + pragma auto_vacuum = incremental; 23 23 + pragma busy_timeout = 5000; 24 24 + 25 25 + create table if not exists known_dids ( 26 26 + did text primary key 27 27 + ); 28 28 + 29 29 + create table if not exists pipelines ( 30 30 + rkey text not null, 31 31 + pipeline text not null, -- json 32 32 + primary key rkey 33 33 + ); 34 34 + `) 35 35 + if err != nil { 36 36 + return nil, err 37 37 + } 38 38 + 39 39 + return &DB{db}, nil 40 40 + } 41 41 + 42 42 + func (d *DB) SaveLastTimeUs(lastTimeUs int64) error { 43 43 + _, err := d.Exec(` 44 44 + insert into _jetstream (id, last_time_us) 45 45 + values (1, ?) 46 46 + on conflict(id) do update set last_time_us = excluded.last_time_us 47 47 + `, lastTimeUs) 48 48 + return err 49 49 + } 50 50 + 51 51 + func (d *DB) GetLastTimeUs() (int64, error) { 52 52 + var lastTimeUs int64 53 53 + row := d.QueryRow(`select last_time_us from _jetstream where id = 1;`) 54 54 + err := row.Scan(&lastTimeUs) 55 55 + return lastTimeUs, err 56 56 + }

+83

spindle/db/pipelines.go

··· 1 1 + package db 2 2 + 3 3 + import ( 4 4 + "fmt" 5 5 + ) 6 6 + 7 7 + type Pipeline struct { 8 8 + Rkey string 9 9 + PipelineJson string 10 10 + } 11 11 + 12 12 + func (d *DB) InsertPipeline(pipeline Pipeline) error { 13 13 + _, err := d.Exec( 14 14 + `insert into pipelines (rkey, nsid, event) values (?, ?, ?)`, 15 15 + pipeline.Rkey, 16 16 + pipeline.PipelineJson, 17 17 + ) 18 18 + 19 19 + return err 20 20 + } 21 21 + 22 22 + func (d *DB) GetPipeline(rkey, cursor string) (Pipeline, error) { 23 23 + whereClause := "where rkey = ?" 24 24 + args := []any{rkey} 25 25 + 26 26 + if cursor != "" { 27 27 + whereClause += " and rkey > ?" 28 28 + args = append(args, cursor) 29 29 + } 30 30 + 31 31 + query := fmt.Sprintf(` 32 32 + select rkey, pipeline 33 33 + from pipelines 34 34 + %s 35 35 + limit 1 36 36 + `, whereClause) 37 37 + 38 38 + row := d.QueryRow(query, args...) 39 39 + 40 40 + var p Pipeline 41 41 + err := row.Scan(&p.Rkey, &p.PipelineJson) 42 42 + if err != nil { 43 43 + return Pipeline{}, err 44 44 + } 45 45 + 46 46 + return p, nil 47 47 + } 48 48 + 49 49 + func (d *DB) GetPipelines(cursor string) ([]Pipeline, error) { 50 50 + whereClause := "" 51 51 + args := []any{} 52 52 + if cursor != "" { 53 53 + whereClause = "where rkey > ?" 54 54 + args = append(args, cursor) 55 55 + } 56 56 + 57 57 + query := fmt.Sprintf(` 58 58 + select rkey, nsid, pipeline 59 59 + from pipelines 60 60 + %s 61 61 + order by rkey asc 62 62 + limit 100 63 63 + `, whereClause) 64 64 + 65 65 + rows, err := d.Query(query, args...) 66 66 + if err != nil { 67 67 + return nil, err 68 68 + } 69 69 + defer rows.Close() 70 70 + 71 71 + var evts []Pipeline 72 72 + for rows.Next() { 73 73 + var ev Pipeline 74 74 + rows.Scan(&ev.Rkey, &ev.PipelineJson) 75 75 + evts = append(evts, ev) 76 76 + } 77 77 + 78 78 + if err := rows.Err(); err != nil { 79 79 + return nil, err 80 80 + } 81 81 + 82 82 + return evts, nil 83 83 + }

+71

spindle/server.go

··· 1 1 + package spindle 2 2 + 3 3 + import ( 4 4 + "fmt" 5 5 + "log/slog" 6 6 + "net/http" 7 7 + 8 8 + "golang.org/x/net/context" 9 9 + "tangled.sh/tangled.sh/core/api/tangled" 10 10 + "tangled.sh/tangled.sh/core/jetstream" 11 11 + "tangled.sh/tangled.sh/core/knotserver/notifier" 12 12 + "tangled.sh/tangled.sh/core/log" 13 13 + "tangled.sh/tangled.sh/core/rbac" 14 14 + "tangled.sh/tangled.sh/core/spindle/config" 15 15 + "tangled.sh/tangled.sh/core/spindle/db" 16 16 + ) 17 17 + 18 18 + type Spindle struct { 19 19 + jc *jetstream.JetstreamClient 20 20 + db *db.DB 21 21 + e *rbac.Enforcer 22 22 + l *slog.Logger 23 23 + n *notifier.Notifier 24 24 + } 25 25 + 26 26 + func Run(ctx context.Context) error { 27 27 + cfg, err := config.Load(ctx) 28 28 + if err != nil { 29 29 + return fmt.Errorf("failed to load config: %w", err) 30 30 + } 31 31 + 32 32 + d, err := db.Make(cfg.Server.DBPath) 33 33 + if err != nil { 34 34 + return fmt.Errorf("failed to setup db: %w", err) 35 35 + } 36 36 + 37 37 + e, err := rbac.NewEnforcer(cfg.Server.DBPath) 38 38 + if err != nil { 39 39 + return fmt.Errorf("failed to setup rbac enforcer: %w", err) 40 40 + } 41 41 + 42 42 + logger := log.FromContext(ctx) 43 43 + 44 44 + collections := []string{tangled.SpindleMemberNSID} 45 45 + jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, logger, d, true, false) 46 46 + if err != nil { 47 47 + return fmt.Errorf("failed to setup jetstream client: %w", err) 48 48 + } 49 49 + 50 50 + n := notifier.New() 51 51 + 52 52 + spindle := Spindle{ 53 53 + jc: jc, 54 54 + e: e, 55 55 + db: d, 56 56 + l: logger, 57 57 + n: &n, 58 58 + } 59 59 + 60 60 + logger.Info("starting spindle server", "address", cfg.Server.ListenAddr) 61 61 + logger.Error("server error", "error", http.ListenAndServe(cfg.Server.ListenAddr, spindle.Router())) 62 62 + 63 63 + return nil 64 64 + } 65 65 + 66 66 + func (s *Spindle) Router() http.Handler { 67 67 + mux := &http.ServeMux{} 68 68 + 69 69 + mux.HandleFunc("/events", s.Events) 70 70 + return mux 71 71 + }

+93

spindle/stream.go

··· 1 1 + package spindle 2 2 + 3 3 + import ( 4 4 + "net/http" 5 5 + "time" 6 6 + 7 7 + "github.com/gorilla/websocket" 8 8 + "golang.org/x/net/context" 9 9 + ) 10 10 + 11 11 + var upgrader = websocket.Upgrader{ 12 12 + ReadBufferSize: 1024, 13 13 + WriteBufferSize: 1024, 14 14 + } 15 15 + 16 16 + func (s *Spindle) Events(w http.ResponseWriter, r *http.Request) { 17 17 + l := s.l.With("handler", "Events") 18 18 + l.Info("received new connection") 19 19 + 20 20 + conn, err := upgrader.Upgrade(w, r, nil) 21 21 + if err != nil { 22 22 + l.Error("websocket upgrade failed", "err", err) 23 23 + w.WriteHeader(http.StatusInternalServerError) 24 24 + return 25 25 + } 26 26 + defer conn.Close() 27 27 + l.Info("upgraded http to wss") 28 28 + 29 29 + ch := s.n.Subscribe() 30 30 + defer s.n.Unsubscribe(ch) 31 31 + 32 32 + ctx, cancel := context.WithCancel(r.Context()) 33 33 + defer cancel() 34 34 + go func() { 35 35 + for { 36 36 + if _, _, err := conn.NextReader(); err != nil { 37 37 + l.Error("failed to read", "err", err) 38 38 + cancel() 39 39 + return 40 40 + } 41 41 + } 42 42 + }() 43 43 + 44 44 + cursor := "" 45 45 + 46 46 + // complete backfill first before going to live data 47 47 + l.Info("going through backfill", "cursor", cursor) 48 48 + if err := s.streamPipelines(conn, &cursor); err != nil { 49 49 + l.Error("failed to backfill", "err", err) 50 50 + return 51 51 + } 52 52 + 53 53 + for { 54 54 + // wait for new data or timeout 55 55 + select { 56 56 + case <-ctx.Done(): 57 57 + l.Info("stopping stream: client closed connection") 58 58 + return 59 59 + case <-ch: 60 60 + // we have been notified of new data 61 61 + l.Info("going through live data", "cursor", cursor) 62 62 + if err := s.streamPipelines(conn, &cursor); err != nil { 63 63 + l.Error("failed to stream", "err", err) 64 64 + return 65 65 + } 66 66 + case <-time.After(30 * time.Second): 67 67 + // send a keep-alive 68 68 + l.Info("sent keepalive") 69 69 + if err = conn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second)); err != nil { 70 70 + l.Error("failed to write control", "err", err) 71 71 + } 72 72 + } 73 73 + } 74 74 + } 75 75 + 76 76 + func (s *Spindle) streamPipelines(conn *websocket.Conn, cursor *string) error { 77 77 + ops, err := s.db.GetPipelines(*cursor) 78 78 + if err != nil { 79 79 + s.l.Debug("err", "err", err) 80 80 + return err 81 81 + } 82 82 + s.l.Debug("ops", "ops", ops) 83 83 + 84 84 + for _, op := range ops { 85 85 + if err := conn.WriteJSON(op); err != nil { 86 86 + s.l.Debug("err", "err", err) 87 87 + return err 88 88 + } 89 89 + *cursor = op.Rkey 90 90 + } 91 91 + 92 92 + return nil 93 93 + }