+5 -57

pkg/api/api.go

··· 35 35 "stream.place/streamplace/pkg/director" 36 36 apierrors "stream.place/streamplace/pkg/errors" 37 37 "stream.place/streamplace/pkg/linking" 38 38 + "stream.place/streamplace/pkg/localdb" 38 39 "stream.place/streamplace/pkg/log" 39 40 "stream.place/streamplace/pkg/media" 40 41 "stream.place/streamplace/pkg/mist/mistconfig" ··· 56 57 CLI *config.CLI 57 58 Model model.Model 58 59 StatefulDB *statedb.StatefulDB 60 60 + LocalDB localdb.LocalDB 59 61 Updater *Updater 60 62 Signer *eip712.EIP712Signer 61 63 Mimes map[string]string ··· 93 95 mu sync.RWMutex 94 96 } 95 97 96 96 - func MakeStreamplaceAPI(cli *config.CLI, mod model.Model, statefulDB *statedb.StatefulDB, noter notifications.FirebaseNotifier, mm *media.MediaManager, ms media.MediaSigner, bus *bus.Bus, atsync *atproto.ATProtoSynchronizer, d *director.Director, op *oatproxy.OATProxy) (*StreamplaceAPI, error) { 98 98 + func MakeStreamplaceAPI(cli *config.CLI, mod model.Model, statefulDB *statedb.StatefulDB, noter notifications.FirebaseNotifier, mm *media.MediaManager, ms media.MediaSigner, bus *bus.Bus, atsync *atproto.ATProtoSynchronizer, d *director.Director, op *oatproxy.OATProxy, ldb localdb.LocalDB) (*StreamplaceAPI, error) { 97 99 updater, err := PrepareUpdater(cli) 98 100 if err != nil { 99 101 return nil, err ··· 117 119 sessionsLock: sync.RWMutex{}, 118 120 rtmpSessions: make(map[string]*media.RTMPSession), 119 121 rtmpSessionsLock: sync.Mutex{}, 122 122 + LocalDB: ldb, 120 123 } 121 124 a.Mimes, err = updater.GetMimes() 122 125 if err != nil { ··· 152 155 Recorder: metrics.NewRecorder(metrics.Config{}), 153 156 }) 154 157 var xrpc http.Handler 155 155 - xrpc, err := spxrpc.NewServer(ctx, a.CLI, a.Model, a.StatefulDB, a.op, mdlw, a.ATSync, a.Bus) 158 158 + xrpc, err := spxrpc.NewServer(ctx, a.CLI, a.Model, a.StatefulDB, a.op, mdlw, a.ATSync, a.Bus, a.LocalDB) 156 159 if err != nil { 157 160 return nil, err 158 161 } ··· 203 206 addHandle(apiRouter, "GET", "/api/chat/:repoDID", a.HandleChat(ctx)) 204 207 addHandle(apiRouter, "GET", "/api/websocket/:repoDID", a.HandleWebsocket(ctx)) 205 208 addHandle(apiRouter, "GET", "/api/livestream/:repoDID", a.HandleLivestream(ctx)) 206 206 - addHandle(apiRouter, "GET", "/api/segment/recent", a.HandleRecentSegments(ctx)) 207 207 - addHandle(apiRouter, "GET", "/api/segment/recent/:repoDID", a.HandleUserRecentSegments(ctx)) 208 209 addHandle(apiRouter, "GET", "/api/bluesky/resolve/:handle", a.HandleBlueskyResolve(ctx)) 209 210 addHandle(apiRouter, "GET", "/api/view-count/:user", a.HandleViewCount(ctx)) 210 211 addHandle(apiRouter, "GET", "/api/clip/:user/:file", a.HandleClip(ctx)) ··· 558 559 return 559 560 } 560 561 w.WriteHeader(201) 561 561 - } 562 562 - } 563 563 - 564 564 - func (a *StreamplaceAPI) HandleRecentSegments(ctx context.Context) httprouter.Handle { 565 565 - return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { 566 566 - segs, err := a.Model.MostRecentSegments() 567 567 - if err != nil { 568 568 - apierrors.WriteHTTPInternalServerError(w, "could not get segments", err) 569 569 - return 570 570 - } 571 571 - bs, err := json.Marshal(segs) 572 572 - if err != nil { 573 573 - apierrors.WriteHTTPInternalServerError(w, "could not marshal segments", err) 574 574 - return 575 575 - } 576 576 - w.Header().Add("Content-Type", "application/json") 577 577 - if _, err := w.Write(bs); err != nil { 578 578 - log.Error(ctx, "error writing response", "error", err) 579 579 - } 580 580 - } 581 581 - } 582 582 - 583 583 - func (a *StreamplaceAPI) HandleUserRecentSegments(ctx context.Context) httprouter.Handle { 584 584 - return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { 585 585 - user := params.ByName("repoDID") 586 586 - if user == "" { 587 587 - apierrors.WriteHTTPBadRequest(w, "user required", nil) 588 588 - return 589 589 - } 590 590 - user, err := a.NormalizeUser(ctx, user) 591 591 - if err != nil { 592 592 - apierrors.WriteHTTPNotFound(w, "user not found", err) 593 593 - return 594 594 - } 595 595 - seg, err := a.Model.LatestSegmentForUser(user) 596 596 - if err != nil { 597 597 - apierrors.WriteHTTPInternalServerError(w, "could not get segments", err) 598 598 - return 599 599 - } 600 600 - streamplaceSeg, err := seg.ToStreamplaceSegment() 601 601 - if err != nil { 602 602 - apierrors.WriteHTTPInternalServerError(w, "could not convert segment to streamplace segment", err) 603 603 - return 604 604 - } 605 605 - bs, err := json.Marshal(streamplaceSeg) 606 606 - if err != nil { 607 607 - apierrors.WriteHTTPInternalServerError(w, "could not marshal segments", err) 608 608 - return 609 609 - } 610 610 - w.Header().Add("Content-Type", "application/json") 611 611 - if _, err := w.Write(bs); err != nil { 612 612 - log.Error(ctx, "error writing response", "error", err) 613 613 - } 614 562 } 615 563 } 616 564

+2 -2

pkg/api/api_internal.go

··· 298 298 errors.WriteHTTPBadRequest(w, "id required", nil) 299 299 return 300 300 } 301 301 - segment, err := a.Model.GetSegment(id) 301 301 + segment, err := a.LocalDB.GetSegment(id) 302 302 if err != nil { 303 303 errors.WriteHTTPBadRequest(w, err.Error(), err) 304 304 return ··· 553 553 } 554 554 after := time.Now().Add(-time.Duration(secs) * time.Second) 555 555 w.Header().Set("Content-Type", "video/mp4") 556 556 - err = media.ClipUser(ctx, a.Model, a.CLI, user, w, nil, &after) 556 556 + err = media.ClipUser(ctx, a.LocalDB, a.CLI, user, w, nil, &after) 557 557 if err != nil { 558 558 errors.WriteHTTPInternalServerError(w, "unable to clip user", err) 559 559 return

+1 -1

pkg/api/playback.go

··· 272 272 errors.WriteHTTPNotFound(w, "user not found", err) 273 273 return 274 274 } 275 275 - thumb, err := a.Model.LatestThumbnailForUser(user) 275 275 + thumb, err := a.LocalDB.LatestThumbnailForUser(user) 276 276 if err != nil { 277 277 errors.WriteHTTPInternalServerError(w, "could not query thumbnail", err) 278 278 return

+1 -1

pkg/api/websocket.go

··· 181 181 }() 182 182 183 183 go func() { 184 184 - seg, err := a.Model.LatestSegmentForUser(repoDID) 184 184 + seg, err := a.LocalDB.LatestSegmentForUser(repoDID) 185 185 if err != nil { 186 186 log.Error(ctx, "could not get replies", "error", err) 187 187 return

+11 -5

pkg/cmd/streamplace.go

··· 29 29 "stream.place/streamplace/pkg/director" 30 30 "stream.place/streamplace/pkg/gstinit" 31 31 "stream.place/streamplace/pkg/iroh/generated/iroh_streamplace" 32 32 + "stream.place/streamplace/pkg/localdb" 32 33 "stream.place/streamplace/pkg/log" 33 34 "stream.place/streamplace/pkg/media" 34 35 "stream.place/streamplace/pkg/notifications" ··· 237 238 return fmt.Errorf("error creating streamplace dir at %s:%w", cli.DataDir, err) 238 239 } 239 240 241 241 + ldb, err := localdb.MakeDB(cli.LocalDBURL) 242 242 + if err != nil { 243 243 + return err 244 244 + } 245 245 + 240 246 mod, err := model.MakeDB(cli.DataFilePath([]string{"index"})) 241 247 if err != nil { 242 248 return err ··· 291 297 return fmt.Errorf("failed to migrate: %w", err) 292 298 } 293 299 294 294 - mm, err := media.MakeMediaManager(ctx, &cli, signer, mod, b, atsync) 300 300 + mm, err := media.MakeMediaManager(ctx, &cli, signer, mod, b, atsync, ldb) 295 301 if err != nil { 296 302 return err 297 303 } ··· 380 386 ClientMetadata: clientMetadata, 381 387 Public: cli.PublicOAuth, 382 388 }) 383 383 - d := director.NewDirector(mm, mod, &cli, b, op, state, replicator) 384 384 - a, err := api.MakeStreamplaceAPI(&cli, mod, state, noter, mm, ms, b, atsync, d, op) 389 389 + d := director.NewDirector(mm, mod, &cli, b, op, state, replicator, ldb) 390 390 + a, err := api.MakeStreamplaceAPI(&cli, mod, state, noter, mm, ms, b, atsync, d, op, ldb) 385 391 if err != nil { 386 392 return err 387 393 } ··· 446 452 }) 447 453 448 454 group.Go(func() error { 449 449 - return storage.StartSegmentCleaner(ctx, mod, &cli) 455 455 + return storage.StartSegmentCleaner(ctx, ldb, &cli) 450 456 }) 451 457 452 458 group.Go(func() error { 453 453 - return mod.StartSegmentCleaner(ctx) 459 459 + return ldb.StartSegmentCleaner(ctx) 454 460 }) 455 461 456 462 group.Go(func() error {

+3

pkg/config/config.go

··· 56 56 Build *BuildFlags 57 57 DataDir string 58 58 DBURL string 59 59 + LocalDBURL string 59 60 EthAccountAddr string 60 61 EthKeystorePath string 61 62 EthPassword string ··· 242 243 cli.StringSliceFlag(fs, &cli.AdminDIDs, "admin-dids", []string{}, "comma-separated list of DIDs that are authorized to modify branding and other admin operations") 243 244 cli.StringSliceFlag(fs, &cli.Syndicate, "syndicate", []string{}, "list of DIDs that we should rebroadcast ('*' for everybody)") 244 245 fs.BoolVar(&cli.PlayerTelemetry, "player-telemetry", true, "enable player telemetry") 246 246 + fs.StringVar(&cli.LocalDBURL, "local-db-url", "sqlite://$SP_DATA_DIR/localdb.sqlite", "URL of the local database to use for storing local data") 247 247 + cli.dataDirFlags = append(cli.dataDirFlags, &cli.LocalDBURL) 245 248 246 249 fs.Bool("external-signing", true, "DEPRECATED, does nothing.") 247 250 fs.Bool("insecure", false, "DEPRECATED, does nothing.")

+5 -1

pkg/director/director.go

··· 9 9 "golang.org/x/sync/errgroup" 10 10 "stream.place/streamplace/pkg/bus" 11 11 "stream.place/streamplace/pkg/config" 12 12 + "stream.place/streamplace/pkg/localdb" 12 13 "stream.place/streamplace/pkg/log" 13 14 "stream.place/streamplace/pkg/media" 14 15 "stream.place/streamplace/pkg/model" ··· 32 33 op *oatproxy.OATProxy 33 34 statefulDB *statedb.StatefulDB 34 35 replicator replication.Replicator 36 36 + localDB localdb.LocalDB 35 37 } 36 38 37 37 - func NewDirector(mm *media.MediaManager, mod model.Model, cli *config.CLI, bus *bus.Bus, op *oatproxy.OATProxy, statefulDB *statedb.StatefulDB, replicator replication.Replicator) *Director { 39 39 + func NewDirector(mm *media.MediaManager, mod model.Model, cli *config.CLI, bus *bus.Bus, op *oatproxy.OATProxy, statefulDB *statedb.StatefulDB, replicator replication.Replicator, ldb localdb.LocalDB) *Director { 38 40 return &Director{ 39 41 mm: mm, 40 42 mod: mod, ··· 45 47 op: op, 46 48 statefulDB: statefulDB, 47 49 replicator: replicator, 50 50 + localDB: ldb, 48 51 } 49 52 } 50 53 ··· 79 82 // Initialize notification channels (buffered size 1 for coalescing) 80 83 statusUpdateChan: make(chan struct{}, 1), 81 84 originUpdateChan: make(chan struct{}, 1), 85 85 + localDB: d.localDB, 82 86 } 83 87 d.streamSessions[not.Segment.RepoDID] = ss 84 88 g.Go(func() error {

+6 -4

pkg/director/stream_session.go

··· 20 20 "stream.place/streamplace/pkg/bus" 21 21 "stream.place/streamplace/pkg/config" 22 22 "stream.place/streamplace/pkg/livepeer" 23 23 + "stream.place/streamplace/pkg/localdb" 23 24 "stream.place/streamplace/pkg/log" 24 25 "stream.place/streamplace/pkg/media" 25 26 "stream.place/streamplace/pkg/model" ··· 44 45 lastStatus time.Time 45 46 lastStatusCID *string 46 47 lastOriginTime time.Time 48 48 + localDB localdb.LocalDB 47 49 48 50 // Channels for background workers 49 51 statusUpdateChan chan struct{} // Signal to update status ··· 178 180 aqt := aqtime.FromTime(notif.Segment.StartTime) 179 181 ctx = log.WithLogValues(ctx, "segID", notif.Segment.ID, "repoDID", notif.Segment.RepoDID, "timestamp", aqt.FileSafeString()) 180 182 notif.Segment.MediaData.Size = len(notif.Data) 181 181 - err := ss.mod.CreateSegment(notif.Segment) 183 183 + err := ss.localDB.CreateSegment(notif.Segment) 182 184 if err != nil { 183 185 return fmt.Errorf("could not add segment to database: %w", err) 184 186 } ··· 292 294 return nil 293 295 } 294 296 defer lock.Unlock() 295 295 - oldThumb, err := ss.mod.LatestThumbnailForUser(not.Segment.RepoDID) 297 297 + oldThumb, err := ss.localDB.LatestThumbnailForUser(not.Segment.RepoDID) 296 298 if err != nil { 297 299 return err 298 300 } ··· 311 313 if err != nil { 312 314 return err 313 315 } 314 314 - thumb := &model.Thumbnail{ 316 316 + thumb := &localdb.Thumbnail{ 315 317 Format: "jpeg", 316 318 SegmentID: not.Segment.ID, 317 319 } 318 318 - err = ss.mod.CreateThumbnail(thumb) 320 320 + err = ss.localDB.CreateThumbnail(thumb) 319 321 if err != nil { 320 322 return err 321 323 }

+81

pkg/localdb/localdb.go

··· 1 1 + package localdb 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "fmt" 6 6 + "strings" 7 7 + "time" 8 8 + 9 9 + "gorm.io/driver/sqlite" 10 10 + "gorm.io/gorm" 11 11 + "gorm.io/plugin/prometheus" 12 12 + "stream.place/streamplace/pkg/config" 13 13 + "stream.place/streamplace/pkg/log" 14 14 + ) 15 15 + 16 16 + type LocalDB interface { 17 17 + CreateSegment(segment *Segment) error 18 18 + MostRecentSegments() ([]Segment, error) 19 19 + LatestSegmentForUser(user string) (*Segment, error) 20 20 + LatestSegmentsForUser(user string, limit int, before *time.Time, after *time.Time) ([]Segment, error) 21 21 + FilterLiveRepoDIDs(repoDIDs []string) ([]string, error) 22 22 + CreateThumbnail(thumb *Thumbnail) error 23 23 + LatestThumbnailForUser(user string) (*Thumbnail, error) 24 24 + GetSegment(id string) (*Segment, error) 25 25 + GetExpiredSegments(ctx context.Context) ([]Segment, error) 26 26 + DeleteSegment(ctx context.Context, id string) error 27 27 + StartSegmentCleaner(ctx context.Context) error 28 28 + SegmentCleaner(ctx context.Context) error 29 29 + } 30 30 + 31 31 + type LocalDatabase struct { 32 32 + DB *gorm.DB 33 33 + } 34 34 + 35 35 + func MakeDB(dbURL string) (LocalDB, error) { 36 36 + log.Log(context.Background(), "starting database", "dbURL", dbURL) 37 37 + if strings.HasPrefix(dbURL, "sqlite://") { 38 38 + dbURL = dbURL[len("sqlite://"):] 39 39 + } else if dbURL != ":memory:" { 40 40 + return nil, fmt.Errorf("unsupported database URL (most start with sqlite://): %s", dbURL) 41 41 + } 42 42 + dial := sqlite.Open(dbURL) 43 43 + 44 44 + db, err := gorm.Open(dial, &gorm.Config{ 45 45 + SkipDefaultTransaction: true, 46 46 + TranslateError: true, 47 47 + Logger: config.GormLogger, 48 48 + }) 49 49 + if err != nil { 50 50 + return nil, fmt.Errorf("error starting database: %w", err) 51 51 + } 52 52 + err = db.Exec("PRAGMA journal_mode=WAL;").Error 53 53 + if err != nil { 54 54 + return nil, fmt.Errorf("error setting journal mode: %w", err) 55 55 + } 56 56 + 57 57 + err = db.Use(prometheus.New(prometheus.Config{ 58 58 + DBName: "localdb", 59 59 + RefreshInterval: 10, 60 60 + StartServer: false, 61 61 + })) 62 62 + if err != nil { 63 63 + return nil, fmt.Errorf("error using prometheus plugin: %w", err) 64 64 + } 65 65 + 66 66 + sqlDB, err := db.DB() 67 67 + if err != nil { 68 68 + return nil, fmt.Errorf("error getting database: %w", err) 69 69 + } 70 70 + sqlDB.SetMaxOpenConns(1) 71 71 + for _, model := range []any{ 72 72 + Segment{}, 73 73 + Thumbnail{}, 74 74 + } { 75 75 + err = db.AutoMigrate(model) 76 76 + if err != nil { 77 77 + return nil, err 78 78 + } 79 79 + } 80 80 + return &LocalDatabase{DB: db}, nil 81 81 + }

+410

pkg/localdb/segment.go

··· 1 1 + package localdb 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "database/sql/driver" 6 6 + "encoding/json" 7 7 + "errors" 8 8 + "fmt" 9 9 + "time" 10 10 + 11 11 + "gorm.io/gorm" 12 12 + "stream.place/streamplace/pkg/aqtime" 13 13 + "stream.place/streamplace/pkg/log" 14 14 + "stream.place/streamplace/pkg/streamplace" 15 15 + ) 16 16 + 17 17 + type SegmentMediadataVideo struct { 18 18 + Width int `json:"width"` 19 19 + Height int `json:"height"` 20 20 + FPSNum int `json:"fpsNum"` 21 21 + FPSDen int `json:"fpsDen"` 22 22 + BFrames bool `json:"bframes"` 23 23 + } 24 24 + 25 25 + type SegmentMediadataAudio struct { 26 26 + Rate int `json:"rate"` 27 27 + Channels int `json:"channels"` 28 28 + } 29 29 + 30 30 + type SegmentMediaData struct { 31 31 + Video []*SegmentMediadataVideo `json:"video"` 32 32 + Audio []*SegmentMediadataAudio `json:"audio"` 33 33 + Duration int64 `json:"duration"` 34 34 + Size int `json:"size"` 35 35 + } 36 36 + 37 37 + // Scan scan value into Jsonb, implements sql.Scanner interface 38 38 + func (j *SegmentMediaData) Scan(value any) error { 39 39 + bytes, ok := value.([]byte) 40 40 + if !ok { 41 41 + return errors.New(fmt.Sprint("Failed to unmarshal JSONB value:", value)) 42 42 + } 43 43 + 44 44 + result := SegmentMediaData{} 45 45 + err := json.Unmarshal(bytes, &result) 46 46 + *j = SegmentMediaData(result) 47 47 + return err 48 48 + } 49 49 + 50 50 + // Value return json value, implement driver.Valuer interface 51 51 + func (j SegmentMediaData) Value() (driver.Value, error) { 52 52 + return json.Marshal(j) 53 53 + } 54 54 + 55 55 + // ContentRights represents content rights and attribution information 56 56 + type ContentRights struct { 57 57 + CopyrightNotice *string `json:"copyrightNotice,omitempty"` 58 58 + CopyrightYear *int64 `json:"copyrightYear,omitempty"` 59 59 + Creator *string `json:"creator,omitempty"` 60 60 + CreditLine *string `json:"creditLine,omitempty"` 61 61 + License *string `json:"license,omitempty"` 62 62 + } 63 63 + 64 64 + // Scan scan value into ContentRights, implements sql.Scanner interface 65 65 + func (c *ContentRights) Scan(value any) error { 66 66 + if value == nil { 67 67 + *c = ContentRights{} 68 68 + return nil 69 69 + } 70 70 + bytes, ok := value.([]byte) 71 71 + if !ok { 72 72 + return errors.New(fmt.Sprint("Failed to unmarshal ContentRights value:", value)) 73 73 + } 74 74 + 75 75 + result := ContentRights{} 76 76 + err := json.Unmarshal(bytes, &result) 77 77 + *c = ContentRights(result) 78 78 + return err 79 79 + } 80 80 + 81 81 + // Value return json value, implement driver.Valuer interface 82 82 + func (c ContentRights) Value() (driver.Value, error) { 83 83 + return json.Marshal(c) 84 84 + } 85 85 + 86 86 + // DistributionPolicy represents distribution policy information 87 87 + type DistributionPolicy struct { 88 88 + DeleteAfterSeconds *int64 `json:"deleteAfterSeconds,omitempty"` 89 89 + } 90 90 + 91 91 + // Scan scan value into DistributionPolicy, implements sql.Scanner interface 92 92 + func (d *DistributionPolicy) Scan(value any) error { 93 93 + if value == nil { 94 94 + *d = DistributionPolicy{} 95 95 + return nil 96 96 + } 97 97 + bytes, ok := value.([]byte) 98 98 + if !ok { 99 99 + return errors.New(fmt.Sprint("Failed to unmarshal DistributionPolicy value:", value)) 100 100 + } 101 101 + 102 102 + result := DistributionPolicy{} 103 103 + err := json.Unmarshal(bytes, &result) 104 104 + *d = DistributionPolicy(result) 105 105 + return err 106 106 + } 107 107 + 108 108 + // Value return json value, implement driver.Valuer interface 109 109 + func (d DistributionPolicy) Value() (driver.Value, error) { 110 110 + return json.Marshal(d) 111 111 + } 112 112 + 113 113 + // ContentWarningsSlice is a custom type for storing content warnings as JSON in the database 114 114 + type ContentWarningsSlice []string 115 115 + 116 116 + // Scan scan value into ContentWarningsSlice, implements sql.Scanner interface 117 117 + func (c *ContentWarningsSlice) Scan(value any) error { 118 118 + if value == nil { 119 119 + *c = ContentWarningsSlice{} 120 120 + return nil 121 121 + } 122 122 + bytes, ok := value.([]byte) 123 123 + if !ok { 124 124 + return errors.New(fmt.Sprint("Failed to unmarshal ContentWarningsSlice value:", value)) 125 125 + } 126 126 + 127 127 + result := ContentWarningsSlice{} 128 128 + err := json.Unmarshal(bytes, &result) 129 129 + *c = ContentWarningsSlice(result) 130 130 + return err 131 131 + } 132 132 + 133 133 + // Value return json value, implement driver.Valuer interface 134 134 + func (c ContentWarningsSlice) Value() (driver.Value, error) { 135 135 + return json.Marshal(c) 136 136 + } 137 137 + 138 138 + type Segment struct { 139 139 + ID string `json:"id" gorm:"primaryKey"` 140 140 + SigningKeyDID string `json:"signingKeyDID" gorm:"column:signing_key_did"` 141 141 + StartTime time.Time `json:"startTime" gorm:"index:latest_segments,priority:2;index:start_time"` 142 142 + RepoDID string `json:"repoDID" gorm:"index:latest_segments,priority:1;column:repo_did"` 143 143 + Title string `json:"title"` 144 144 + Size int `json:"size" gorm:"column:size"` 145 145 + MediaData *SegmentMediaData `json:"mediaData,omitempty"` 146 146 + ContentWarnings ContentWarningsSlice `json:"contentWarnings,omitempty"` 147 147 + ContentRights *ContentRights `json:"contentRights,omitempty"` 148 148 + DistributionPolicy *DistributionPolicy `json:"distributionPolicy,omitempty"` 149 149 + DeleteAfter *time.Time `json:"deleteAfter,omitempty" gorm:"column:delete_after;index:delete_after"` 150 150 + } 151 151 + 152 152 + func (s *Segment) ToStreamplaceSegment() (*streamplace.Segment, error) { 153 153 + aqt := aqtime.FromTime(s.StartTime) 154 154 + if s.MediaData == nil { 155 155 + return nil, fmt.Errorf("media data is nil") 156 156 + } 157 157 + if len(s.MediaData.Video) == 0 || s.MediaData.Video[0] == nil { 158 158 + return nil, fmt.Errorf("video data is nil") 159 159 + } 160 160 + if len(s.MediaData.Audio) == 0 || s.MediaData.Audio[0] == nil { 161 161 + return nil, fmt.Errorf("audio data is nil") 162 162 + } 163 163 + duration := s.MediaData.Duration 164 164 + sizei64 := int64(s.Size) 165 165 + 166 166 + // Convert model metadata to streamplace metadata 167 167 + var contentRights *streamplace.MetadataContentRights 168 168 + if s.ContentRights != nil { 169 169 + contentRights = &streamplace.MetadataContentRights{ 170 170 + CopyrightNotice: s.ContentRights.CopyrightNotice, 171 171 + CopyrightYear: s.ContentRights.CopyrightYear, 172 172 + Creator: s.ContentRights.Creator, 173 173 + CreditLine: s.ContentRights.CreditLine, 174 174 + License: s.ContentRights.License, 175 175 + } 176 176 + } 177 177 + 178 178 + var contentWarnings *streamplace.MetadataContentWarnings 179 179 + if len(s.ContentWarnings) > 0 { 180 180 + contentWarnings = &streamplace.MetadataContentWarnings{ 181 181 + Warnings: []string(s.ContentWarnings), 182 182 + } 183 183 + } 184 184 + 185 185 + var distributionPolicy *streamplace.MetadataDistributionPolicy 186 186 + if s.DistributionPolicy != nil && s.DistributionPolicy.DeleteAfterSeconds != nil { 187 187 + distributionPolicy = &streamplace.MetadataDistributionPolicy{ 188 188 + DeleteAfter: s.DistributionPolicy.DeleteAfterSeconds, 189 189 + } 190 190 + } 191 191 + 192 192 + return &streamplace.Segment{ 193 193 + LexiconTypeID: "place.stream.segment", 194 194 + Creator: s.RepoDID, 195 195 + Id: s.ID, 196 196 + SigningKey: s.SigningKeyDID, 197 197 + StartTime: string(aqt), 198 198 + Duration: &duration, 199 199 + Size: &sizei64, 200 200 + ContentRights: contentRights, 201 201 + ContentWarnings: contentWarnings, 202 202 + DistributionPolicy: distributionPolicy, 203 203 + Video: []*streamplace.Segment_Video{ 204 204 + { 205 205 + Codec: "h264", 206 206 + Width: int64(s.MediaData.Video[0].Width), 207 207 + Height: int64(s.MediaData.Video[0].Height), 208 208 + Framerate: &streamplace.Segment_Framerate{ 209 209 + Num: int64(s.MediaData.Video[0].FPSNum), 210 210 + Den: int64(s.MediaData.Video[0].FPSDen), 211 211 + }, 212 212 + Bframes: &s.MediaData.Video[0].BFrames, 213 213 + }, 214 214 + }, 215 215 + Audio: []*streamplace.Segment_Audio{ 216 216 + { 217 217 + Codec: "opus", 218 218 + Rate: int64(s.MediaData.Audio[0].Rate), 219 219 + Channels: int64(s.MediaData.Audio[0].Channels), 220 220 + }, 221 221 + }, 222 222 + }, nil 223 223 + } 224 224 + 225 225 + func (m *LocalDatabase) CreateSegment(seg *Segment) error { 226 226 + err := m.DB.Model(Segment{}).Create(seg).Error 227 227 + if err != nil { 228 228 + return err 229 229 + } 230 230 + return nil 231 231 + } 232 232 + 233 233 + // should return the most recent segment for each user, ordered by most recent first 234 234 + // only includes segments from the last 30 seconds 235 235 + func (m *LocalDatabase) MostRecentSegments() ([]Segment, error) { 236 236 + var segments []Segment 237 237 + thirtySecondsAgo := time.Now().Add(-30 * time.Second) 238 238 + 239 239 + err := m.DB.Table("segments"). 240 240 + Select("segments.*"). 241 241 + Where("start_time > ?", thirtySecondsAgo.UTC()). 242 242 + Order("start_time DESC"). 243 243 + Find(&segments).Error 244 244 + if err != nil { 245 245 + return nil, err 246 246 + } 247 247 + if segments == nil { 248 248 + return []Segment{}, nil 249 249 + } 250 250 + 251 251 + segmentMap := make(map[string]Segment) 252 252 + for _, seg := range segments { 253 253 + prev, ok := segmentMap[seg.RepoDID] 254 254 + if !ok { 255 255 + segmentMap[seg.RepoDID] = seg 256 256 + } else { 257 257 + if seg.StartTime.After(prev.StartTime) { 258 258 + segmentMap[seg.RepoDID] = seg 259 259 + } 260 260 + } 261 261 + } 262 262 + 263 263 + filteredSegments := []Segment{} 264 264 + for _, seg := range segmentMap { 265 265 + filteredSegments = append(filteredSegments, seg) 266 266 + } 267 267 + 268 268 + return filteredSegments, nil 269 269 + } 270 270 + 271 271 + func (m *LocalDatabase) LatestSegmentForUser(user string) (*Segment, error) { 272 272 + var seg Segment 273 273 + err := m.DB.Model(Segment{}).Where("repo_did = ?", user).Order("start_time DESC").First(&seg).Error 274 274 + if err != nil { 275 275 + return nil, err 276 276 + } 277 277 + return &seg, nil 278 278 + } 279 279 + 280 280 + func (m *LocalDatabase) FilterLiveRepoDIDs(repoDIDs []string) ([]string, error) { 281 281 + if len(repoDIDs) == 0 { 282 282 + return []string{}, nil 283 283 + } 284 284 + 285 285 + thirtySecondsAgo := time.Now().Add(-30 * time.Second) 286 286 + 287 287 + var liveDIDs []string 288 288 + 289 289 + err := m.DB.Table("segments"). 290 290 + Select("DISTINCT repo_did"). 291 291 + Where("repo_did IN ? AND start_time > ?", repoDIDs, thirtySecondsAgo.UTC()). 292 292 + Pluck("repo_did", &liveDIDs).Error 293 293 + 294 294 + if err != nil { 295 295 + return nil, err 296 296 + } 297 297 + 298 298 + return liveDIDs, nil 299 299 + } 300 300 + 301 301 + func (m *LocalDatabase) LatestSegmentsForUser(user string, limit int, before *time.Time, after *time.Time) ([]Segment, error) { 302 302 + var segs []Segment 303 303 + if before == nil { 304 304 + later := time.Now().Add(1000 * time.Hour) 305 305 + before = &later 306 306 + } 307 307 + if after == nil { 308 308 + earlier := time.Time{} 309 309 + after = &earlier 310 310 + } 311 311 + err := m.DB.Model(Segment{}).Where("repo_did = ? AND start_time < ? AND start_time > ?", user, before.UTC(), after.UTC()).Order("start_time DESC").Limit(limit).Find(&segs).Error 312 312 + if err != nil { 313 313 + return nil, err 314 314 + } 315 315 + return segs, nil 316 316 + } 317 317 + 318 318 + func (m *LocalDatabase) GetSegment(id string) (*Segment, error) { 319 319 + var seg Segment 320 320 + 321 321 + err := m.DB.Model(&Segment{}). 322 322 + Preload("Repo"). 323 323 + Where("id = ?", id). 324 324 + First(&seg).Error 325 325 + 326 326 + if errors.Is(err, gorm.ErrRecordNotFound) { 327 327 + return nil, nil 328 328 + } 329 329 + if err != nil { 330 330 + return nil, err 331 331 + } 332 332 + 333 333 + return &seg, nil 334 334 + } 335 335 + 336 336 + func (m *LocalDatabase) GetExpiredSegments(ctx context.Context) ([]Segment, error) { 337 337 + 338 338 + var expiredSegments []Segment 339 339 + now := time.Now() 340 340 + err := m.DB. 341 341 + Where("delete_after IS NOT NULL AND delete_after < ?", now.UTC()). 342 342 + Find(&expiredSegments).Error 343 343 + if err != nil { 344 344 + return nil, err 345 345 + } 346 346 + 347 347 + return expiredSegments, nil 348 348 + } 349 349 + 350 350 + func (m *LocalDatabase) DeleteSegment(ctx context.Context, id string) error { 351 351 + return m.DB.Delete(&Segment{}, "id = ?", id).Error 352 352 + } 353 353 + 354 354 + func (m *LocalDatabase) StartSegmentCleaner(ctx context.Context) error { 355 355 + err := m.SegmentCleaner(ctx) 356 356 + if err != nil { 357 357 + return err 358 358 + } 359 359 + ticker := time.NewTicker(1 * time.Minute) 360 360 + defer ticker.Stop() 361 361 + 362 362 + for { 363 363 + select { 364 364 + case <-ctx.Done(): 365 365 + return nil 366 366 + case <-ticker.C: 367 367 + err := m.SegmentCleaner(ctx) 368 368 + if err != nil { 369 369 + log.Error(ctx, "Failed to clean segments", "error", err) 370 370 + } 371 371 + } 372 372 + } 373 373 + } 374 374 + 375 375 + func (m *LocalDatabase) SegmentCleaner(ctx context.Context) error { 376 376 + // Calculate the cutoff time (10 minutes ago) 377 377 + cutoffTime := aqtime.FromTime(time.Now().Add(-10 * time.Minute)).Time() 378 378 + 379 379 + // Find all unique repo_did values 380 380 + var repoDIDs []string 381 381 + if err := m.DB.Model(&Segment{}).Distinct("repo_did").Pluck("repo_did", &repoDIDs).Error; err != nil { 382 382 + log.Error(ctx, "Failed to get unique repo_dids for segment cleaning", "error", err) 383 383 + return err 384 384 + } 385 385 + 386 386 + // For each user, keep their last 10 segments and delete older ones 387 387 + for _, repoDID := range repoDIDs { 388 388 + // Get IDs of the last 10 segments for this user 389 389 + var keepSegmentIDs []string 390 390 + if err := m.DB.Model(&Segment{}). 391 391 + Where("repo_did = ?", repoDID). 392 392 + Order("start_time DESC"). 393 393 + Limit(10). 394 394 + Pluck("id", &keepSegmentIDs).Error; err != nil { 395 395 + log.Error(ctx, "Failed to get segment IDs to keep", "repo_did", repoDID, "error", err) 396 396 + return err 397 397 + } 398 398 + 399 399 + // Delete old segments except the ones we want to keep 400 400 + result := m.DB.Where("repo_did = ? AND start_time < ? AND id NOT IN ?", 401 401 + repoDID, cutoffTime, keepSegmentIDs).Delete(&Segment{}) 402 402 + 403 403 + if result.Error != nil { 404 404 + log.Error(ctx, "Failed to clean old segments", "repo_did", repoDID, "error", result.Error) 405 405 + } else if result.RowsAffected > 0 { 406 406 + log.Log(ctx, "Cleaned old segments", "repo_did", repoDID, "count", result.RowsAffected) 407 407 + } 408 408 + } 409 409 + return nil 410 410 + }

+59

pkg/localdb/segment_test.go

··· 1 1 + package localdb 2 2 + 3 3 + import ( 4 4 + "fmt" 5 5 + "sync" 6 6 + "testing" 7 7 + "time" 8 8 + 9 9 + "github.com/stretchr/testify/require" 10 10 + "stream.place/streamplace/pkg/config" 11 11 + ) 12 12 + 13 13 + func TestSegmentPerf(t *testing.T) { 14 14 + config.DisableSQLLogging() 15 15 + // dburl := filepath.Join(t.TempDir(), "test.db") 16 16 + db, err := MakeDB(":memory:") 17 17 + require.NoError(t, err) 18 18 + // Create a ldb instance 19 19 + ldb := db.(*LocalDatabase) 20 20 + t.Cleanup(func() { 21 21 + // os.Remove(dburl) 22 22 + }) 23 23 + 24 24 + defer config.EnableSQLLogging() 25 25 + // Create 250000 segments with timestamps 1 hour ago, each one second apart 26 26 + wg := sync.WaitGroup{} 27 27 + segCount := 250000 28 28 + wg.Add(segCount) 29 29 + baseTime := time.Now() 30 30 + for i := 0; i < segCount; i++ { 31 31 + segment := &Segment{ 32 32 + ID: fmt.Sprintf("segment-%d", i), 33 33 + RepoDID: "did:plc:test123", 34 34 + StartTime: baseTime.Add(-time.Duration(i) * time.Second).UTC(), 35 35 + } 36 36 + go func() { 37 37 + defer wg.Done() 38 38 + err = ldb.DB.Create(segment).Error 39 39 + require.NoError(t, err) 40 40 + }() 41 41 + } 42 42 + wg.Wait() 43 43 + 44 44 + startTime := time.Now() 45 45 + wg = sync.WaitGroup{} 46 46 + runs := 1000 47 47 + wg.Add(runs) 48 48 + for i := 0; i < runs; i++ { 49 49 + go func() { 50 50 + defer wg.Done() 51 51 + _, err := ldb.MostRecentSegments() 52 52 + require.NoError(t, err) 53 53 + // require.Len(t, segments, 1) 54 54 + }() 55 55 + } 56 56 + wg.Wait() 57 57 + fmt.Printf("Time taken: %s\n", time.Since(startTime)) 58 58 + require.Less(t, time.Since(startTime), 10*time.Second) 59 59 + }

+60

pkg/localdb/thumbnail.go

··· 1 1 + package localdb 2 2 + 3 3 + import ( 4 4 + "fmt" 5 5 + 6 6 + "github.com/google/uuid" 7 7 + ) 8 8 + 9 9 + type Thumbnail struct { 10 10 + ID string `json:"id" gorm:"primaryKey"` 11 11 + Format string `json:"format"` 12 12 + SegmentID string `json:"segmentId" gorm:"index"` 13 13 + Segment Segment `json:"segment,omitempty" gorm:"foreignKey:SegmentID;references:id"` 14 14 + } 15 15 + 16 16 + func (m *LocalDatabase) CreateThumbnail(thumb *Thumbnail) error { 17 17 + uu, err := uuid.NewV7() 18 18 + if err != nil { 19 19 + return err 20 20 + } 21 21 + if thumb.SegmentID == "" { 22 22 + return fmt.Errorf("segmentID is required") 23 23 + } 24 24 + thumb.ID = uu.String() 25 25 + err = m.DB.Model(Thumbnail{}).Create(thumb).Error 26 26 + if err != nil { 27 27 + return err 28 28 + } 29 29 + return nil 30 30 + } 31 31 + 32 32 + // return the most recent thumbnail for a user 33 33 + func (m *LocalDatabase) LatestThumbnailForUser(user string) (*Thumbnail, error) { 34 34 + var thumbnail Thumbnail 35 35 + 36 36 + res := m.DB.Table("thumbnails AS t"). 37 37 + Select("t.*"). 38 38 + Joins("JOIN segments AS s ON t.segment_id = s.id"). 39 39 + Where("s.repo_did = ?", user). 40 40 + Order("s.start_time DESC"). 41 41 + Limit(1). 42 42 + Scan(&thumbnail) 43 43 + 44 44 + if res.RowsAffected == 0 { 45 45 + return nil, nil 46 46 + } 47 47 + if res.Error != nil { 48 48 + return nil, res.Error 49 49 + } 50 50 + 51 51 + var seg Segment 52 52 + err := m.DB.First(&seg, "id = ?", thumbnail.SegmentID).Error 53 53 + if err != nil { 54 54 + return nil, fmt.Errorf("could not find segment for thumbnail SegmentID=%s", thumbnail.SegmentID) 55 55 + } 56 56 + 57 57 + thumbnail.Segment = seg 58 58 + 59 59 + return &thumbnail, nil 60 60 + }

+3 -3

pkg/media/clip_user.go

··· 10 10 11 11 "stream.place/streamplace/pkg/aqtime" 12 12 "stream.place/streamplace/pkg/config" 13 13 - "stream.place/streamplace/pkg/model" 13 13 + "stream.place/streamplace/pkg/localdb" 14 14 ) 15 15 16 16 - func ClipUser(ctx context.Context, mod model.Model, cli *config.CLI, user string, writer io.Writer, before *time.Time, after *time.Time) error { 17 17 - segments, err := mod.LatestSegmentsForUser(user, -1, before, after) 16 16 + func ClipUser(ctx context.Context, localDB localdb.LocalDB, cli *config.CLI, user string, writer io.Writer, before *time.Time, after *time.Time) error { 17 17 + segments, err := localDB.LatestSegmentsForUser(user, -1, before, after) 18 18 if err != nil { 19 19 return fmt.Errorf("unable to get segments: %w", err) 20 20 }

+11 -8

pkg/media/media.go

··· 21 21 c2patypes "stream.place/streamplace/pkg/c2patypes" 22 22 "stream.place/streamplace/pkg/config" 23 23 "stream.place/streamplace/pkg/gstinit" 24 24 + "stream.place/streamplace/pkg/localdb" 24 25 "stream.place/streamplace/pkg/model" 25 26 "stream.place/streamplace/pkg/streamplace" 26 27 ··· 51 52 atsync *atproto.ATProtoSynchronizer 52 53 webrtcAPI *webrtc.API 53 54 webrtcConfig webrtc.Configuration 55 55 + localDB localdb.LocalDB 54 56 } 55 57 56 58 type NewSegmentNotification struct { 57 57 - Segment *model.Segment 59 59 + Segment *localdb.Segment 58 60 Data []byte 59 61 Metadata *SegmentMetadata 60 62 Local bool ··· 65 67 return SelfTest(ctx) 66 68 } 67 69 68 68 - func MakeMediaManager(ctx context.Context, cli *config.CLI, signer crypto.Signer, mod model.Model, bus *bus.Bus, atsync *atproto.ATProtoSynchronizer) (*MediaManager, error) { 70 70 + func MakeMediaManager(ctx context.Context, cli *config.CLI, signer crypto.Signer, mod model.Model, bus *bus.Bus, atsync *atproto.ATProtoSynchronizer, ldb localdb.LocalDB) (*MediaManager, error) { 69 71 gstinit.InitGST() 70 72 err := SelfTest(ctx) 71 73 if err != nil { ··· 127 129 atsync: atsync, 128 130 webrtcAPI: api, 129 131 webrtcConfig: config, 132 132 + localDB: ldb, 130 133 }, nil 131 134 } 132 135 ··· 190 193 Title string 191 194 Creator string 192 195 ContentWarnings []string 193 193 - ContentRights *model.ContentRights 194 194 - DistributionPolicy *model.DistributionPolicy 196 196 + ContentRights *localdb.ContentRights 197 197 + DistributionPolicy *localdb.DistributionPolicy 195 198 MetadataConfiguration *streamplace.MetadataConfiguration 196 199 Livestream *streamplace.Livestream 197 200 } ··· 312 315 } 313 316 314 317 // extractContentRights extracts content rights from the C2PA manifest 315 315 - func extractContentRights(mani *c2patypes.Manifest) *model.ContentRights { 318 318 + func extractContentRights(mani *c2patypes.Manifest) *localdb.ContentRights { 316 319 ass := findAssertion(mani, StreamplaceMetadata) 317 320 if ass == nil { 318 321 return nil ··· 323 326 return nil 324 327 } 325 328 326 326 - rights := &model.ContentRights{} 329 329 + rights := &localdb.ContentRights{} 327 330 328 331 // Extract copyright notice 329 332 if notice, ok := data["dc:rights"]; ok { ··· 375 378 } 376 379 377 380 // extractDistributionPolicy extracts distribution policy from the C2PA manifest 378 378 - func extractDistributionPolicy(mani *c2patypes.Manifest, segmentStart aqtime.AQTime) *model.DistributionPolicy { 381 381 + func extractDistributionPolicy(mani *c2patypes.Manifest, segmentStart aqtime.AQTime) *localdb.DistributionPolicy { 379 382 metadataConfig := extractMetadataConfiguration(mani) 380 383 if metadataConfig == nil { 381 384 return nil ··· 392 395 // deleteAfter contains an offset in seconds from creation time 393 396 deleteAfterSeconds := *metadataConfig.DistributionPolicy.DeleteAfter 394 397 395 395 - return &model.DistributionPolicy{ 398 398 + return &localdb.DistributionPolicy{ 396 399 DeleteAfterSeconds: &deleteAfterSeconds, 397 400 } 398 401 }

+9 -9

pkg/media/media_data_parser.go

··· 13 13 "github.com/go-gst/go-gst/gst" 14 14 "github.com/go-gst/go-gst/gst/app" 15 15 "go.opentelemetry.io/otel" 16 16 + "stream.place/streamplace/pkg/localdb" 16 17 "stream.place/streamplace/pkg/log" 17 17 - "stream.place/streamplace/pkg/model" 18 18 ) 19 19 20 20 func padProbeEmpty(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { 21 21 return gst.PadProbeOK 22 22 } 23 23 24 24 - func ParseSegmentMediaData(ctx context.Context, mp4bs []byte) (*model.SegmentMediaData, error) { 24 24 + func ParseSegmentMediaData(ctx context.Context, mp4bs []byte) (*localdb.SegmentMediaData, error) { 25 25 ctx, span := otel.Tracer("signer").Start(ctx, "ParseSegmentMediaData") 26 26 defer span.End() 27 27 ctx = log.WithLogValues(ctx, "GStreamerFunc", "ParseSegmentMediaData") ··· 40 40 return nil, fmt.Errorf("error creating SegmentMetadata pipeline: %w", err) 41 41 } 42 42 43 43 - var videoMetadata *model.SegmentMediadataVideo 44 44 - var audioMetadata *model.SegmentMediadataAudio 43 43 + var videoMetadata *localdb.SegmentMediadataVideo 44 44 + var audioMetadata *localdb.SegmentMediadataAudio 45 45 46 46 appsrc, err := pipeline.GetElementByName("appsrc") 47 47 if err != nil { ··· 118 118 name := structure.Name() 119 119 120 120 if name[:5] == "video" { 121 121 - videoMetadata = &model.SegmentMediadataVideo{} 121 121 + videoMetadata = &localdb.SegmentMediadataVideo{} 122 122 // Get some common video properties 123 123 widthVal, _ := structure.GetValue("width") 124 124 heightVal, _ := structure.GetValue("height") ··· 147 147 } 148 148 149 149 if name[:5] == "audio" { 150 150 - audioMetadata = &model.SegmentMediadataAudio{} 150 150 + audioMetadata = &localdb.SegmentMediadataAudio{} 151 151 // Get some common audio properties 152 152 rateVal, _ := structure.GetValue("rate") 153 153 channelsVal, _ := structure.GetValue("channels") ··· 275 275 276 276 videoMetadata.BFrames = hasBFrames 277 277 278 278 - meta := &model.SegmentMediaData{ 279 279 - Video: []*model.SegmentMediadataVideo{videoMetadata}, 280 280 - Audio: []*model.SegmentMediadataAudio{audioMetadata}, 278 278 + meta := &localdb.SegmentMediaData{ 279 279 + Video: []*localdb.SegmentMediadataVideo{videoMetadata}, 280 280 + Audio: []*localdb.SegmentMediadataAudio{audioMetadata}, 281 281 } 282 282 283 283 ok, dur := pipeline.QueryDuration(gst.FormatTime)

+4 -1

pkg/media/media_test.go

··· 11 11 "stream.place/streamplace/pkg/bus" 12 12 "stream.place/streamplace/pkg/config" 13 13 ct "stream.place/streamplace/pkg/config/configtesting" 14 14 + "stream.place/streamplace/pkg/localdb" 14 15 "stream.place/streamplace/pkg/model" 15 16 "stream.place/streamplace/pkg/statedb" 16 17 ) ··· 23 24 24 25 func getStaticTestMediaManager(t *testing.T) (*MediaManager, MediaSigner) { 25 26 mod, err := model.MakeDB(":memory:") 27 27 + require.NoError(t, err) 28 28 + ldb, err := localdb.MakeDB(":memory:") 26 29 require.NoError(t, err) 27 30 // signer, err := c2pa.MakeStaticSigner(eip712test.KeyBytes) 28 31 require.NoError(t, err) ··· 42 45 StatefulDB: statedb, 43 46 Bus: bus.NewBus(), 44 47 } 45 45 - mm, err := MakeMediaManager(context.Background(), cli, nil, mod, bus.NewBus(), atsync) 48 48 + mm, err := MakeMediaManager(context.Background(), cli, nil, mod, bus.NewBus(), atsync, ldb) 46 49 require.NoError(t, err) 47 50 // ms, err := MakeMediaSigner(context.Background(), cli, "test-person", signer) 48 51 // require.NoError(t, err)

+5 -5

pkg/media/validate.go

··· 18 18 "stream.place/streamplace/pkg/constants" 19 19 "stream.place/streamplace/pkg/crypto/signers" 20 20 "stream.place/streamplace/pkg/iroh/generated/iroh_streamplace" 21 21 + "stream.place/streamplace/pkg/localdb" 21 22 "stream.place/streamplace/pkg/log" 22 22 - "stream.place/streamplace/pkg/model" 23 23 ) 24 24 25 25 type ManifestAndCert struct { ··· 47 47 48 48 label := manifest.Label 49 49 if label != nil && mm.model != nil { 50 50 - oldSeg, err := mm.model.GetSegment(*label) 50 50 + oldSeg, err := mm.localDB.GetSegment(*label) 51 51 if err != nil { 52 52 return fmt.Errorf("failed to get old segment: %w", err) 53 53 } ··· 117 117 expiryTime := meta.StartTime.Time().Add(time.Duration(*meta.DistributionPolicy.DeleteAfterSeconds) * time.Second) 118 118 deleteAfter = &expiryTime 119 119 } 120 120 - seg := &model.Segment{ 120 120 + seg := &localdb.Segment{ 121 121 ID: *label, 122 122 SigningKeyDID: signingKeyDID, 123 123 RepoDID: repoDID, ··· 125 125 Title: meta.Title, 126 126 Size: len(buf), 127 127 MediaData: mediaData, 128 128 - ContentWarnings: model.ContentWarningsSlice(meta.ContentWarnings), 128 128 + ContentWarnings: localdb.ContentWarningsSlice(meta.ContentWarnings), 129 129 ContentRights: meta.ContentRights, 130 130 DistributionPolicy: meta.DistributionPolicy, 131 131 DeleteAfter: deleteAfter, ··· 205 205 type ValidationResult struct { 206 206 Pub *atcrypto.PublicKeyK256 207 207 Meta *SegmentMetadata 208 208 - MediaData *model.SegmentMediaData 208 208 + MediaData *localdb.SegmentMediaData 209 209 Manifest *c2patypes.Manifest 210 210 Cert string 211 211 }

-15

pkg/model/model.go

··· 28 28 PlayerReport(playerID string) (map[string]any, error) 29 29 ClearPlayerEvents() error 30 30 31 31 - CreateSegment(segment *Segment) error 32 32 - MostRecentSegments() ([]Segment, error) 33 33 - LatestSegmentForUser(user string) (*Segment, error) 34 34 - LatestSegmentsForUser(user string, limit int, before *time.Time, after *time.Time) ([]Segment, error) 35 35 - FilterLiveRepoDIDs(repoDIDs []string) ([]string, error) 36 36 - CreateThumbnail(thumb *Thumbnail) error 37 37 - LatestThumbnailForUser(user string) (*Thumbnail, error) 38 38 - GetSegment(id string) (*Segment, error) 39 39 - GetExpiredSegments(ctx context.Context) ([]Segment, error) 40 40 - DeleteSegment(ctx context.Context, id string) error 41 41 - StartSegmentCleaner(ctx context.Context) error 42 42 - SegmentCleaner(ctx context.Context) error 43 43 - 44 31 GetIdentity(id string) (*Identity, error) 45 32 UpdateIdentity(ident *Identity) error 46 33 ··· 178 165 sqlDB.SetMaxOpenConns(1) 179 166 for _, model := range []any{ 180 167 PlayerEvent{}, 181 181 - Segment{}, 182 182 - Thumbnail{}, 183 168 Identity{}, 184 169 Repo{}, 185 170 SigningKey{},

-411

pkg/model/segment.go

··· 1 1 package model 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "database/sql/driver" 6 6 - "encoding/json" 7 7 - "errors" 8 8 - "fmt" 9 9 - "time" 10 10 - 11 11 - "gorm.io/gorm" 12 12 - "stream.place/streamplace/pkg/aqtime" 13 13 - "stream.place/streamplace/pkg/log" 14 14 - "stream.place/streamplace/pkg/streamplace" 15 15 - ) 16 16 - 17 17 - type SegmentMediadataVideo struct { 18 18 - Width int `json:"width"` 19 19 - Height int `json:"height"` 20 20 - FPSNum int `json:"fpsNum"` 21 21 - FPSDen int `json:"fpsDen"` 22 22 - BFrames bool `json:"bframes"` 23 23 - } 24 24 - 25 25 - type SegmentMediadataAudio struct { 26 26 - Rate int `json:"rate"` 27 27 - Channels int `json:"channels"` 28 28 - } 29 29 - 30 30 - type SegmentMediaData struct { 31 31 - Video []*SegmentMediadataVideo `json:"video"` 32 32 - Audio []*SegmentMediadataAudio `json:"audio"` 33 33 - Duration int64 `json:"duration"` 34 34 - Size int `json:"size"` 35 35 - } 36 36 - 37 37 - // Scan scan value into Jsonb, implements sql.Scanner interface 38 38 - func (j *SegmentMediaData) Scan(value any) error { 39 39 - bytes, ok := value.([]byte) 40 40 - if !ok { 41 41 - return errors.New(fmt.Sprint("Failed to unmarshal JSONB value:", value)) 42 42 - } 43 43 - 44 44 - result := SegmentMediaData{} 45 45 - err := json.Unmarshal(bytes, &result) 46 46 - *j = SegmentMediaData(result) 47 47 - return err 48 48 - } 49 49 - 50 50 - // Value return json value, implement driver.Valuer interface 51 51 - func (j SegmentMediaData) Value() (driver.Value, error) { 52 52 - return json.Marshal(j) 53 53 - } 54 54 - 55 55 - // ContentRights represents content rights and attribution information 56 56 - type ContentRights struct { 57 57 - CopyrightNotice *string `json:"copyrightNotice,omitempty"` 58 58 - CopyrightYear *int64 `json:"copyrightYear,omitempty"` 59 59 - Creator *string `json:"creator,omitempty"` 60 60 - CreditLine *string `json:"creditLine,omitempty"` 61 61 - License *string `json:"license,omitempty"` 62 62 - } 63 63 - 64 64 - // Scan scan value into ContentRights, implements sql.Scanner interface 65 65 - func (c *ContentRights) Scan(value any) error { 66 66 - if value == nil { 67 67 - *c = ContentRights{} 68 68 - return nil 69 69 - } 70 70 - bytes, ok := value.([]byte) 71 71 - if !ok { 72 72 - return errors.New(fmt.Sprint("Failed to unmarshal ContentRights value:", value)) 73 73 - } 74 74 - 75 75 - result := ContentRights{} 76 76 - err := json.Unmarshal(bytes, &result) 77 77 - *c = ContentRights(result) 78 78 - return err 79 79 - } 80 80 - 81 81 - // Value return json value, implement driver.Valuer interface 82 82 - func (c ContentRights) Value() (driver.Value, error) { 83 83 - return json.Marshal(c) 84 84 - } 85 85 - 86 86 - // DistributionPolicy represents distribution policy information 87 87 - type DistributionPolicy struct { 88 88 - DeleteAfterSeconds *int64 `json:"deleteAfterSeconds,omitempty"` 89 89 - } 90 90 - 91 91 - // Scan scan value into DistributionPolicy, implements sql.Scanner interface 92 92 - func (d *DistributionPolicy) Scan(value any) error { 93 93 - if value == nil { 94 94 - *d = DistributionPolicy{} 95 95 - return nil 96 96 - } 97 97 - bytes, ok := value.([]byte) 98 98 - if !ok { 99 99 - return errors.New(fmt.Sprint("Failed to unmarshal DistributionPolicy value:", value)) 100 100 - } 101 101 - 102 102 - result := DistributionPolicy{} 103 103 - err := json.Unmarshal(bytes, &result) 104 104 - *d = DistributionPolicy(result) 105 105 - return err 106 106 - } 107 107 - 108 108 - // Value return json value, implement driver.Valuer interface 109 109 - func (d DistributionPolicy) Value() (driver.Value, error) { 110 110 - return json.Marshal(d) 111 111 - } 112 112 - 113 113 - // ContentWarningsSlice is a custom type for storing content warnings as JSON in the database 114 114 - type ContentWarningsSlice []string 115 115 - 116 116 - // Scan scan value into ContentWarningsSlice, implements sql.Scanner interface 117 117 - func (c *ContentWarningsSlice) Scan(value any) error { 118 118 - if value == nil { 119 119 - *c = ContentWarningsSlice{} 120 120 - return nil 121 121 - } 122 122 - bytes, ok := value.([]byte) 123 123 - if !ok { 124 124 - return errors.New(fmt.Sprint("Failed to unmarshal ContentWarningsSlice value:", value)) 125 125 - } 126 126 - 127 127 - result := ContentWarningsSlice{} 128 128 - err := json.Unmarshal(bytes, &result) 129 129 - *c = ContentWarningsSlice(result) 130 130 - return err 131 131 - } 132 132 - 133 133 - // Value return json value, implement driver.Valuer interface 134 134 - func (c ContentWarningsSlice) Value() (driver.Value, error) { 135 135 - return json.Marshal(c) 136 136 - } 137 137 - 138 138 - type Segment struct { 139 139 - ID string `json:"id" gorm:"primaryKey"` 140 140 - SigningKeyDID string `json:"signingKeyDID" gorm:"column:signing_key_did"` 141 141 - SigningKey *SigningKey `json:"signingKey,omitempty" gorm:"foreignKey:DID;references:SigningKeyDID"` 142 142 - StartTime time.Time `json:"startTime" gorm:"index:latest_segments,priority:2;index:start_time"` 143 143 - RepoDID string `json:"repoDID" gorm:"index:latest_segments,priority:1;column:repo_did"` 144 144 - Repo *Repo `json:"repo,omitempty" gorm:"foreignKey:DID;references:RepoDID"` 145 145 - Title string `json:"title"` 146 146 - Size int `json:"size" gorm:"column:size"` 147 147 - MediaData *SegmentMediaData `json:"mediaData,omitempty"` 148 148 - ContentWarnings ContentWarningsSlice `json:"contentWarnings,omitempty"` 149 149 - ContentRights *ContentRights `json:"contentRights,omitempty"` 150 150 - DistributionPolicy *DistributionPolicy `json:"distributionPolicy,omitempty"` 151 151 - DeleteAfter *time.Time `json:"deleteAfter,omitempty" gorm:"column:delete_after;index:delete_after"` 152 152 - } 153 153 - 154 154 - func (s *Segment) ToStreamplaceSegment() (*streamplace.Segment, error) { 155 155 - aqt := aqtime.FromTime(s.StartTime) 156 156 - if s.MediaData == nil { 157 157 - return nil, fmt.Errorf("media data is nil") 158 158 - } 159 159 - if len(s.MediaData.Video) == 0 || s.MediaData.Video[0] == nil { 160 160 - return nil, fmt.Errorf("video data is nil") 161 161 - } 162 162 - if len(s.MediaData.Audio) == 0 || s.MediaData.Audio[0] == nil { 163 163 - return nil, fmt.Errorf("audio data is nil") 164 164 - } 165 165 - duration := s.MediaData.Duration 166 166 - sizei64 := int64(s.Size) 167 167 - 168 168 - // Convert model metadata to streamplace metadata 169 169 - var contentRights *streamplace.MetadataContentRights 170 170 - if s.ContentRights != nil { 171 171 - contentRights = &streamplace.MetadataContentRights{ 172 172 - CopyrightNotice: s.ContentRights.CopyrightNotice, 173 173 - CopyrightYear: s.ContentRights.CopyrightYear, 174 174 - Creator: s.ContentRights.Creator, 175 175 - CreditLine: s.ContentRights.CreditLine, 176 176 - License: s.ContentRights.License, 177 177 - } 178 178 - } 179 179 - 180 180 - var contentWarnings *streamplace.MetadataContentWarnings 181 181 - if len(s.ContentWarnings) > 0 { 182 182 - contentWarnings = &streamplace.MetadataContentWarnings{ 183 183 - Warnings: []string(s.ContentWarnings), 184 184 - } 185 185 - } 186 186 - 187 187 - var distributionPolicy *streamplace.MetadataDistributionPolicy 188 188 - if s.DistributionPolicy != nil && s.DistributionPolicy.DeleteAfterSeconds != nil { 189 189 - distributionPolicy = &streamplace.MetadataDistributionPolicy{ 190 190 - DeleteAfter: s.DistributionPolicy.DeleteAfterSeconds, 191 191 - } 192 192 - } 193 193 - 194 194 - return &streamplace.Segment{ 195 195 - LexiconTypeID: "place.stream.segment", 196 196 - Creator: s.RepoDID, 197 197 - Id: s.ID, 198 198 - SigningKey: s.SigningKeyDID, 199 199 - StartTime: string(aqt), 200 200 - Duration: &duration, 201 201 - Size: &sizei64, 202 202 - ContentRights: contentRights, 203 203 - ContentWarnings: contentWarnings, 204 204 - DistributionPolicy: distributionPolicy, 205 205 - Video: []*streamplace.Segment_Video{ 206 206 - { 207 207 - Codec: "h264", 208 208 - Width: int64(s.MediaData.Video[0].Width), 209 209 - Height: int64(s.MediaData.Video[0].Height), 210 210 - Framerate: &streamplace.Segment_Framerate{ 211 211 - Num: int64(s.MediaData.Video[0].FPSNum), 212 212 - Den: int64(s.MediaData.Video[0].FPSDen), 213 213 - }, 214 214 - Bframes: &s.MediaData.Video[0].BFrames, 215 215 - }, 216 216 - }, 217 217 - Audio: []*streamplace.Segment_Audio{ 218 218 - { 219 219 - Codec: "opus", 220 220 - Rate: int64(s.MediaData.Audio[0].Rate), 221 221 - Channels: int64(s.MediaData.Audio[0].Channels), 222 222 - }, 223 223 - }, 224 224 - }, nil 225 225 - } 226 226 - 227 227 - func (m *DBModel) CreateSegment(seg *Segment) error { 228 228 - err := m.DB.Model(Segment{}).Create(seg).Error 229 229 - if err != nil { 230 230 - return err 231 231 - } 232 232 - return nil 233 233 - } 234 234 - 235 235 - // should return the most recent segment for each user, ordered by most recent first 236 236 - // only includes segments from the last 30 seconds 237 237 - func (m *DBModel) MostRecentSegments() ([]Segment, error) { 238 238 - var segments []Segment 239 239 - thirtySecondsAgo := time.Now().Add(-30 * time.Second) 240 240 - 241 241 - err := m.DB.Table("segments"). 242 242 - Select("segments.*"). 243 243 - Where("start_time > ?", thirtySecondsAgo.UTC()). 244 244 - Order("start_time DESC"). 245 245 - Find(&segments).Error 246 246 - if err != nil { 247 247 - return nil, err 248 248 - } 249 249 - if segments == nil { 250 250 - return []Segment{}, nil 251 251 - } 252 252 - 253 253 - segmentMap := make(map[string]Segment) 254 254 - for _, seg := range segments { 255 255 - prev, ok := segmentMap[seg.RepoDID] 256 256 - if !ok { 257 257 - segmentMap[seg.RepoDID] = seg 258 258 - } else { 259 259 - if seg.StartTime.After(prev.StartTime) { 260 260 - segmentMap[seg.RepoDID] = seg 261 261 - } 262 262 - } 263 263 - } 264 264 - 265 265 - filteredSegments := []Segment{} 266 266 - for _, seg := range segmentMap { 267 267 - filteredSegments = append(filteredSegments, seg) 268 268 - } 269 269 - 270 270 - return filteredSegments, nil 271 271 - } 272 272 - 273 273 - func (m *DBModel) LatestSegmentForUser(user string) (*Segment, error) { 274 274 - var seg Segment 275 275 - err := m.DB.Model(Segment{}).Where("repo_did = ?", user).Order("start_time DESC").First(&seg).Error 276 276 - if err != nil { 277 277 - return nil, err 278 278 - } 279 279 - return &seg, nil 280 280 - } 281 281 - 282 282 - func (m *DBModel) FilterLiveRepoDIDs(repoDIDs []string) ([]string, error) { 283 283 - if len(repoDIDs) == 0 { 284 284 - return []string{}, nil 285 285 - } 286 286 - 287 287 - thirtySecondsAgo := time.Now().Add(-30 * time.Second) 288 288 - 289 289 - var liveDIDs []string 290 290 - 291 291 - err := m.DB.Table("segments"). 292 292 - Select("DISTINCT repo_did"). 293 293 - Where("repo_did IN ? AND start_time > ?", repoDIDs, thirtySecondsAgo.UTC()). 294 294 - Pluck("repo_did", &liveDIDs).Error 295 295 - 296 296 - if err != nil { 297 297 - return nil, err 298 298 - } 299 299 - 300 300 - return liveDIDs, nil 301 301 - } 302 302 - 303 303 - func (m *DBModel) LatestSegmentsForUser(user string, limit int, before *time.Time, after *time.Time) ([]Segment, error) { 304 304 - var segs []Segment 305 305 - if before == nil { 306 306 - later := time.Now().Add(1000 * time.Hour) 307 307 - before = &later 308 308 - } 309 309 - if after == nil { 310 310 - earlier := time.Time{} 311 311 - after = &earlier 312 312 - } 313 313 - err := m.DB.Model(Segment{}).Where("repo_did = ? AND start_time < ? AND start_time > ?", user, before.UTC(), after.UTC()).Order("start_time DESC").Limit(limit).Find(&segs).Error 314 314 - if err != nil { 315 315 - return nil, err 316 316 - } 317 317 - return segs, nil 318 318 - } 319 319 - 320 320 - func (m *DBModel) GetSegment(id string) (*Segment, error) { 321 321 - var seg Segment 322 322 - 323 323 - err := m.DB.Model(&Segment{}). 324 324 - Preload("Repo"). 325 325 - Where("id = ?", id). 326 326 - First(&seg).Error 327 327 - 328 328 - if errors.Is(err, gorm.ErrRecordNotFound) { 329 329 - return nil, nil 330 330 - } 331 331 - if err != nil { 332 332 - return nil, err 333 333 - } 334 334 - 335 335 - return &seg, nil 336 336 - } 337 337 - 338 338 - func (m *DBModel) GetExpiredSegments(ctx context.Context) ([]Segment, error) { 339 339 - 340 340 - var expiredSegments []Segment 341 341 - now := time.Now() 342 342 - err := m.DB. 343 343 - Where("delete_after IS NOT NULL AND delete_after < ?", now.UTC()). 344 344 - Find(&expiredSegments).Error 345 345 - if err != nil { 346 346 - return nil, err 347 347 - } 348 348 - 349 349 - return expiredSegments, nil 350 350 - } 351 351 - 352 352 - func (m *DBModel) DeleteSegment(ctx context.Context, id string) error { 353 353 - return m.DB.Delete(&Segment{}, "id = ?", id).Error 354 354 - } 355 355 - 356 356 - func (m *DBModel) StartSegmentCleaner(ctx context.Context) error { 357 357 - err := m.SegmentCleaner(ctx) 358 358 - if err != nil { 359 359 - return err 360 360 - } 361 361 - ticker := time.NewTicker(1 * time.Minute) 362 362 - defer ticker.Stop() 363 363 - 364 364 - for { 365 365 - select { 366 366 - case <-ctx.Done(): 367 367 - return nil 368 368 - case <-ticker.C: 369 369 - err := m.SegmentCleaner(ctx) 370 370 - if err != nil { 371 371 - log.Error(ctx, "Failed to clean segments", "error", err) 372 372 - } 373 373 - } 374 374 - } 375 375 - } 376 376 - 377 377 - func (m *DBModel) SegmentCleaner(ctx context.Context) error { 378 378 - // Calculate the cutoff time (10 minutes ago) 379 379 - cutoffTime := aqtime.FromTime(time.Now().Add(-10 * time.Minute)).Time() 380 380 - 381 381 - // Find all unique repo_did values 382 382 - var repoDIDs []string 383 383 - if err := m.DB.Model(&Segment{}).Distinct("repo_did").Pluck("repo_did", &repoDIDs).Error; err != nil { 384 384 - log.Error(ctx, "Failed to get unique repo_dids for segment cleaning", "error", err) 385 385 - return err 386 386 - } 387 387 - 388 388 - // For each user, keep their last 10 segments and delete older ones 389 389 - for _, repoDID := range repoDIDs { 390 390 - // Get IDs of the last 10 segments for this user 391 391 - var keepSegmentIDs []string 392 392 - if err := m.DB.Model(&Segment{}). 393 393 - Where("repo_did = ?", repoDID). 394 394 - Order("start_time DESC"). 395 395 - Limit(10). 396 396 - Pluck("id", &keepSegmentIDs).Error; err != nil { 397 397 - log.Error(ctx, "Failed to get segment IDs to keep", "repo_did", repoDID, "error", err) 398 398 - return err 399 399 - } 400 400 - 401 401 - // Delete old segments except the ones we want to keep 402 402 - result := m.DB.Where("repo_did = ? AND start_time < ? AND id NOT IN ?", 403 403 - repoDID, cutoffTime, keepSegmentIDs).Delete(&Segment{}) 404 404 - 405 405 - if result.Error != nil { 406 406 - log.Error(ctx, "Failed to clean old segments", "repo_did", repoDID, "error", result.Error) 407 407 - } else if result.RowsAffected > 0 { 408 408 - log.Log(ctx, "Cleaned old segments", "repo_did", repoDID, "count", result.RowsAffected) 409 409 - } 410 410 - } 411 411 - return nil 412 412 - }

-65

pkg/model/segment_test.go

··· 1 1 package model 2 2 - 3 3 - import ( 4 4 - "fmt" 5 5 - "sync" 6 6 - "testing" 7 7 - "time" 8 8 - 9 9 - "github.com/stretchr/testify/require" 10 10 - "stream.place/streamplace/pkg/config" 11 11 - ) 12 12 - 13 13 - func TestSegmentPerf(t *testing.T) { 14 14 - config.DisableSQLLogging() 15 15 - // dburl := filepath.Join(t.TempDir(), "test.db") 16 16 - db, err := MakeDB(":memory:") 17 17 - require.NoError(t, err) 18 18 - // Create a model instance 19 19 - model := db.(*DBModel) 20 20 - t.Cleanup(func() { 21 21 - // os.Remove(dburl) 22 22 - }) 23 23 - 24 24 - // Create a repo for testing 25 25 - repo := &Repo{ 26 26 - DID: "did:plc:test123", 27 27 - } 28 28 - err = model.DB.Create(repo).Error 29 29 - require.NoError(t, err) 30 30 - 31 31 - defer config.EnableSQLLogging() 32 32 - // Create 250000 segments with timestamps 1 hour ago, each one second apart 33 33 - wg := sync.WaitGroup{} 34 34 - segCount := 250000 35 35 - wg.Add(segCount) 36 36 - baseTime := time.Now() 37 37 - for i := 0; i < segCount; i++ { 38 38 - segment := &Segment{ 39 39 - ID: fmt.Sprintf("segment-%d", i), 40 40 - RepoDID: repo.DID, 41 41 - StartTime: baseTime.Add(-time.Duration(i) * time.Second).UTC(), 42 42 - } 43 43 - go func() { 44 44 - defer wg.Done() 45 45 - err = model.DB.Create(segment).Error 46 46 - require.NoError(t, err) 47 47 - }() 48 48 - } 49 49 - wg.Wait() 50 50 - 51 51 - startTime := time.Now() 52 52 - wg = sync.WaitGroup{} 53 53 - runs := 1000 54 54 - wg.Add(runs) 55 55 - for i := 0; i < runs; i++ { 56 56 - go func() { 57 57 - defer wg.Done() 58 58 - _, err := model.MostRecentSegments() 59 59 - require.NoError(t, err) 60 60 - // require.Len(t, segments, 1) 61 61 - }() 62 62 - } 63 63 - wg.Wait() 64 64 - fmt.Printf("Time taken: %s\n", time.Since(startTime)) 65 65 - require.Less(t, time.Since(startTime), 10*time.Second) 66 66 - }

-59

pkg/model/thumbnail.go

··· 1 1 package model 2 2 - 3 3 - import ( 4 4 - "fmt" 5 5 - 6 6 - "github.com/google/uuid" 7 7 - ) 8 8 - 9 9 - type Thumbnail struct { 10 10 - ID string `json:"id" gorm:"primaryKey"` 11 11 - Format string `json:"format"` 12 12 - SegmentID string `json:"segmentId" gorm:"index"` 13 13 - Segment Segment `json:"segment,omitempty" gorm:"foreignKey:SegmentID;references:id"` 14 14 - } 15 15 - 16 16 - func (m *DBModel) CreateThumbnail(thumb *Thumbnail) error { 17 17 - uu, err := uuid.NewV7() 18 18 - if err != nil { 19 19 - return err 20 20 - } 21 21 - if thumb.SegmentID == "" { 22 22 - return fmt.Errorf("segmentID is required") 23 23 - } 24 24 - thumb.ID = uu.String() 25 25 - err = m.DB.Model(Thumbnail{}).Create(thumb).Error 26 26 - if err != nil { 27 27 - return err 28 28 - } 29 29 - return nil 30 30 - } 31 31 - 32 32 - // return the most recent thumbnail for a user 33 33 - func (m *DBModel) LatestThumbnailForUser(user string) (*Thumbnail, error) { 34 34 - var thumbnail Thumbnail 35 35 - 36 36 - res := m.DB.Table("thumbnails AS t"). 37 37 - Select("t.*"). 38 38 - Joins("JOIN segments AS s ON t.segment_id = s.id"). 39 39 - Where("s.repo_did = ?", user). 40 40 - Order("s.start_time DESC"). 41 41 - Limit(1). 42 42 - Scan(&thumbnail) 43 43 - 44 44 - if res.RowsAffected == 0 { 45 45 - return nil, nil 46 46 - } 47 47 - if res.Error != nil { 48 48 - return nil, res.Error 49 49 - } 50 50 - 51 51 - var seg Segment 52 52 - err := m.DB.First(&seg, "id = ?", thumbnail.SegmentID).Error 53 53 - if err != nil { 54 54 - return nil, fmt.Errorf("could not find segment for thumbnail SegmentID=%s", thumbnail.SegmentID) 55 55 - } 56 56 - 57 57 - thumbnail.Segment = seg 58 58 - 59 59 - return &thumbnail, nil 60 60 - }

+1 -1

pkg/spxrpc/app_bsky_feed.go

··· 56 56 outCursor = fmt.Sprintf("%d::%s", ts, last.CID) 57 57 } 58 58 } else if name == FeedLiveStreams { 59 59 - segs, err := s.model.MostRecentSegments() 59 59 + segs, err := s.localDB.MostRecentSegments() 60 60 if err != nil { 61 61 return nil, echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("failed to get recent segments: %v", err)) 62 62 }

+4 -4

pkg/spxrpc/com_atproto_moderation.go

··· 13 13 "github.com/labstack/echo/v4" 14 14 "github.com/streamplace/oatproxy/pkg/oatproxy" 15 15 "stream.place/streamplace/pkg/config" 16 16 + "stream.place/streamplace/pkg/localdb" 16 17 "stream.place/streamplace/pkg/log" 17 18 "stream.place/streamplace/pkg/media" 18 18 - "stream.place/streamplace/pkg/model" 19 19 ) 20 20 21 21 func (s *Server) handleComAtprotoModerationCreateReport(ctx context.Context, body *comatprototypes.ModerationCreateReport_Input) (*comatprototypes.ModerationCreateReport_Output, error) { ··· 76 76 return nil, echo.NewHTTPError(http.StatusBadRequest, "invalid subject") 77 77 } 78 78 79 79 - clipID, err := makeClip(ctx, s.cli, s.model, did) 79 79 + clipID, err := makeClip(ctx, s.cli, s.localDB, did) 80 80 if err != nil { 81 81 // we still want the report to go through! 82 82 log.Error(ctx, "failed to make clip for report", "error", err) ··· 99 99 return &output, nil 100 100 } 101 101 102 102 - func makeClip(ctx context.Context, cli *config.CLI, mod model.Model, did string) (string, error) { 102 102 + func makeClip(ctx context.Context, cli *config.CLI, localDB localdb.LocalDB, did string) (string, error) { 103 103 after := time.Now().Add(-time.Duration(60) * time.Second) 104 104 105 105 uu, err := uuid.NewV7() ··· 113 113 } 114 114 defer fd.Close() 115 115 116 116 - err = media.ClipUser(ctx, mod, cli, did, fd, nil, &after) 116 116 + err = media.ClipUser(ctx, localDB, cli, did, fd, nil, &after) 117 117 if err != nil { 118 118 return "", echo.NewHTTPError(http.StatusInternalServerError, "failed to clip user") 119 119 }

+4 -4

pkg/spxrpc/place_stream_live.go

··· 82 82 beforeTime = &parsedTime 83 83 } 84 84 85 85 - segments, err := s.model.LatestSegmentsForUser(userDID, limit, beforeTime, nil) 85 85 + segments, err := s.localDB.LatestSegmentsForUser(userDID, limit, beforeTime, nil) 86 86 if err != nil { 87 87 return nil, echo.NewHTTPError(http.StatusInternalServerError, "Failed to fetch segments") 88 88 } ··· 223 223 } 224 224 225 225 // Filter for only live streamers 226 226 - liveStreamers, err := s.model.FilterLiveRepoDIDs(streamers) 226 226 + liveStreamers, err := s.localDB.FilterLiveRepoDIDs(streamers) 227 227 if err != nil { 228 228 return nil, echo.NewHTTPError(http.StatusInternalServerError, "Failed to filter live streamers") 229 229 } ··· 256 256 followDIDs[i] = follow.SubjectDID 257 257 } 258 258 259 259 - liveFollows, err := s.model.FilterLiveRepoDIDs(followDIDs) 259 259 + liveFollows, err := s.localDB.FilterLiveRepoDIDs(followDIDs) 260 260 if err != nil { 261 261 return nil, echo.NewHTTPError(http.StatusInternalServerError, "Failed to filter live follows") 262 262 } ··· 281 281 // Final fallback: use host's default recommendations 282 282 defaultStreamers := s.cli.DefaultRecommendedStreamers 283 283 if len(defaultStreamers) > 0 { 284 284 - liveDefaults, err := s.model.FilterLiveRepoDIDs(defaultStreamers) 284 284 + liveDefaults, err := s.localDB.FilterLiveRepoDIDs(defaultStreamers) 285 285 if err != nil { 286 286 return nil, echo.NewHTTPError(http.StatusInternalServerError, "Failed to filter default streamers") 287 287 }

+4 -1

pkg/spxrpc/spxrpc.go

··· 18 18 "stream.place/streamplace/pkg/atproto" 19 19 "stream.place/streamplace/pkg/bus" 20 20 "stream.place/streamplace/pkg/config" 21 21 + "stream.place/streamplace/pkg/localdb" 21 22 "stream.place/streamplace/pkg/log" 22 23 "stream.place/streamplace/pkg/model" 23 24 "stream.place/streamplace/pkg/statedb" ··· 33 34 statefulDB *statedb.StatefulDB 34 35 bus *bus.Bus 35 36 op *oatproxy.OATProxy 37 37 + localDB localdb.LocalDB 36 38 } 37 39 38 38 - func NewServer(ctx context.Context, cli *config.CLI, model model.Model, statefulDB *statedb.StatefulDB, op *oatproxy.OATProxy, mdlw middleware.Middleware, atsync *atproto.ATProtoSynchronizer, bus *bus.Bus) (*Server, error) { 40 40 + func NewServer(ctx context.Context, cli *config.CLI, model model.Model, statefulDB *statedb.StatefulDB, op *oatproxy.OATProxy, mdlw middleware.Middleware, atsync *atproto.ATProtoSynchronizer, bus *bus.Bus, ldb localdb.LocalDB) (*Server, error) { 39 41 e := echo.New() 40 42 s := &Server{ 41 43 e: e, ··· 47 49 statefulDB: statefulDB, 48 50 bus: bus, 49 51 op: op, 52 52 + localDB: ldb, 50 53 } 51 54 e.Use(s.ErrorHandlingMiddleware()) 52 55 e.Use(s.ContextPreservingMiddleware())

+6 -6

pkg/storage/storage.go

··· 10 10 "golang.org/x/sync/errgroup" 11 11 "stream.place/streamplace/pkg/aqtime" 12 12 "stream.place/streamplace/pkg/config" 13 13 + "stream.place/streamplace/pkg/localdb" 13 14 "stream.place/streamplace/pkg/log" 14 14 - "stream.place/streamplace/pkg/model" 15 15 ) 16 16 17 17 const moderationRetention = 120 * time.Second 18 18 19 19 - func StartSegmentCleaner(ctx context.Context, mod model.Model, cli *config.CLI) error { 19 19 + func StartSegmentCleaner(ctx context.Context, localDB localdb.LocalDB, cli *config.CLI) error { 20 20 ctx = log.WithLogValues(ctx, "func", "StartSegmentCleaner") 21 21 g, ctx := errgroup.WithContext(ctx) 22 22 g.Go(func() error { ··· 25 25 case <-ctx.Done(): 26 26 return nil 27 27 case <-time.After(60 * time.Second): 28 28 - expiredSegments, err := mod.GetExpiredSegments(ctx) 28 28 + expiredSegments, err := localDB.GetExpiredSegments(ctx) 29 29 if err != nil { 30 30 return err 31 31 } 32 32 log.Log(ctx, "Cleaning expired segments", "count", len(expiredSegments)) 33 33 for _, seg := range expiredSegments { 34 34 g.Go(func() error { 35 35 - err := deleteSegment(ctx, mod, cli, seg) 35 35 + err := deleteSegment(ctx, localDB, cli, seg) 36 36 if err != nil { 37 37 log.Error(ctx, "Failed to delete segment", "error", err) 38 38 } ··· 47 47 return g.Wait() 48 48 } 49 49 50 50 - func deleteSegment(ctx context.Context, mod model.Model, cli *config.CLI, seg model.Segment) error { 50 50 + func deleteSegment(ctx context.Context, localDB localdb.LocalDB, cli *config.CLI, seg localdb.Segment) error { 51 51 if time.Since(seg.StartTime) < moderationRetention { 52 52 log.Debug(ctx, "Skipping deletion of segment", "id", seg.ID, "time since start", time.Since(seg.StartTime)) 53 53 return nil ··· 61 61 if err != nil && !errors.Is(err, os.ErrNotExist) { 62 62 return err 63 63 } 64 64 - err = mod.DeleteSegment(ctx, seg.ID) 64 64 + err = localDB.DeleteSegment(ctx, seg.ID) 65 65 if err != nil { 66 66 return err 67 67 }