+15 -10

cmd/server/main.go

··· 16 16 17 17 "arabica/internal/atproto" 18 18 "arabica/internal/database/boltstore" 19 19 + "arabica/internal/database/sqlitestore" 19 20 "arabica/internal/email" 20 21 "arabica/internal/feed" 21 22 "arabica/internal/firehose" ··· 103 104 104 105 // Get specialized stores 105 106 sessionStore := store.SessionStore() 106 106 - feedStore := store.FeedStore() 107 107 108 108 // Initialize OAuth manager with persistent session store 109 109 // For local development, localhost URLs trigger special localhost mode in indigo ··· 130 130 log.Fatal().Err(err).Msg("Failed to initialize OAuth") 131 131 } 132 132 133 133 - // Initialize feed registry with persistent store 134 134 - // This loads existing registered DIDs from the database 135 135 - feedRegistry := feed.NewPersistentRegistry(feedStore) 133 133 + // Initialize feed registry (in-memory; populated from SQLite after feedIndex opens) 134 134 + feedRegistry := feed.NewRegistry() 136 135 feedService := feed.NewService(feedRegistry) 137 136 138 138 - log.Info(). 139 139 - Int("registered_users", feedRegistry.Count()). 140 140 - Msg("Feed service initialized with persistent registry") 141 141 - 142 137 // Setup context for graceful shutdown 143 138 ctx, cancel := context.WithCancel(context.Background()) 144 139 defer cancel() ··· 159 154 } 160 155 dataDir = filepath.Join(home, ".local", "share") 161 156 } 162 162 - feedIndexPath = filepath.Join(dataDir, "arabica", "feed-index.db") 157 157 + feedIndexPath = filepath.Join(dataDir, "arabica", "feed-index.sqlite") 163 158 } 164 159 165 160 // Create firehose config ··· 181 176 182 177 log.Info().Str("path", feedIndexPath).Msg("Feed index opened") 183 178 179 179 + // Populate feed registry from SQLite known_dids (replaces BoltDB feed registry) 180 180 + if knownDIDs, err := feedIndex.GetKnownDIDs(); err == nil { 181 181 + for _, did := range knownDIDs { 182 182 + feedRegistry.Register(did) 183 183 + } 184 184 + log.Info().Int("registered_users", feedRegistry.Count()).Msg("Feed registry populated from index") 185 185 + } else { 186 186 + log.Warn().Err(err).Msg("Failed to load known DIDs into feed registry") 187 187 + } 188 188 + 184 189 // Create and start consumer 185 190 firehoseConsumer := firehose.NewConsumer(firehoseConfig, feedIndex) 186 191 firehoseConsumer.Start(ctx) ··· 190 195 feedService.SetFirehoseIndex(adapter) 191 196 192 197 // Wire up moderation filtering for the feed 193 193 - moderationStore := store.ModerationStore() 198 198 + moderationStore := sqlitestore.NewModerationStore(feedIndex.DB()) 194 199 feedService.SetModerationFilter(moderationStore) 195 200 196 201 log.Info().Msg("Firehose consumer started")

+10 -2

go.mod

··· 14 14 github.com/stretchr/testify v1.11.1 15 15 go.etcd.io/bbolt v1.3.8 16 16 golang.org/x/sync v0.19.0 17 17 + modernc.org/sqlite v1.46.1 17 18 ) 18 19 19 20 require ( 20 21 github.com/beorn7/perks v1.0.1 // indirect 21 22 github.com/cespare/xxhash/v2 v2.3.0 // indirect 22 23 github.com/davecgh/go-spew v1.1.1 // indirect 24 24 + github.com/dustin/go-humanize v1.0.1 // indirect 23 25 github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect 24 26 github.com/felixge/httpsnoop v1.0.4 // indirect 25 27 github.com/go-logr/logr v1.4.1 // indirect 26 28 github.com/go-logr/stdr v1.2.2 // indirect 27 29 github.com/gogo/protobuf v1.3.2 // indirect 28 30 github.com/golang-jwt/jwt/v5 v5.2.2 // indirect 29 29 - github.com/google/uuid v1.4.0 // indirect 31 31 + github.com/google/uuid v1.6.0 // indirect 30 32 github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 31 33 github.com/hashicorp/go-retryablehttp v0.7.5 // indirect 32 34 github.com/hashicorp/golang-lru v1.0.2 // indirect ··· 55 57 github.com/multiformats/go-multihash v0.2.3 // indirect 56 58 github.com/multiformats/go-varint v0.0.7 // indirect 57 59 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 60 60 + github.com/ncruces/go-strftime v1.0.0 // indirect 58 61 github.com/opentracing/opentracing-go v1.2.0 // indirect 59 62 github.com/pmezard/go-difflib v1.0.0 // indirect 60 63 github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect 61 64 github.com/prometheus/common v0.66.1 // indirect 62 65 github.com/prometheus/procfs v0.16.1 // indirect 66 66 + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 63 67 github.com/spaolacci/murmur3 v1.1.0 // indirect 64 68 github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e // indirect 65 69 gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect ··· 73 77 go.uber.org/zap v1.26.0 // indirect 74 78 go.yaml.in/yaml/v2 v2.4.2 // indirect 75 79 golang.org/x/crypto v0.40.0 // indirect 76 76 - golang.org/x/sys v0.36.0 // indirect 80 80 + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect 81 81 + golang.org/x/sys v0.37.0 // indirect 77 82 golang.org/x/time v0.3.0 // indirect 78 83 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 79 84 google.golang.org/protobuf v1.36.8 // indirect 80 85 gopkg.in/yaml.v3 v3.0.1 // indirect 81 86 lukechampine.com/blake3 v1.2.1 // indirect 87 87 + modernc.org/libc v1.67.6 // indirect 88 88 + modernc.org/mathutil v1.7.1 // indirect 89 89 + modernc.org/memory v1.11.0 // indirect 82 90 )

+46 -4

go.sum

··· 13 13 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 14 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 15 15 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 16 + github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 17 17 + github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 16 18 github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg= 17 19 github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw= 18 20 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= ··· 33 35 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 34 36 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 35 37 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 38 38 + github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= 39 39 + github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= 36 40 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 37 37 - github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= 38 38 - github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 41 41 + github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 42 42 + github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 39 43 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 40 44 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 41 45 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= ··· 120 124 github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= 121 125 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 122 126 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 127 127 + github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= 128 128 + github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= 123 129 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= 124 130 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= 125 131 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= ··· 136 142 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= 137 143 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= 138 144 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= 145 145 + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= 146 146 + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 139 147 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 140 148 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 141 149 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= ··· 203 211 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 204 212 golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= 205 213 golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= 214 214 + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= 215 215 + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= 206 216 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 207 217 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 208 218 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 209 219 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 210 220 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 221 221 + golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= 222 222 + golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= 211 223 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 212 224 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 213 225 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= ··· 231 243 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 232 244 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 233 245 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 234 234 - golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= 235 235 - golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 246 246 + golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= 247 247 + golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 236 248 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 237 249 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 238 250 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= ··· 248 260 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 249 261 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 250 262 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 263 263 + golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= 264 264 + golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= 251 265 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 252 266 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 253 267 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= ··· 270 284 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 271 285 lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= 272 286 lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= 287 287 + modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= 288 288 + modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= 289 289 + modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc= 290 290 + modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM= 291 291 + modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= 292 292 + modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= 293 293 + modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= 294 294 + modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= 295 295 + modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE= 296 296 + modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= 297 297 + modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= 298 298 + modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= 299 299 + modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= 300 300 + modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= 301 301 + modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= 302 302 + modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= 303 303 + modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= 304 304 + modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= 305 305 + modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= 306 306 + modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= 307 307 + modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= 308 308 + modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= 309 309 + modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= 310 310 + modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= 311 311 + modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= 312 312 + modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= 313 313 + modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= 314 314 + modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

-157

internal/database/boltstore/feed_store.go

··· 1 1 - package boltstore 2 2 - 3 3 - import ( 4 4 - "encoding/json" 5 5 - "time" 6 6 - 7 7 - bolt "go.etcd.io/bbolt" 8 8 - ) 9 9 - 10 10 - // FeedUser represents a registered user in the feed registry. 11 11 - type FeedUser struct { 12 12 - DID string `json:"did"` 13 13 - RegisteredAt time.Time `json:"registered_at"` 14 14 - } 15 15 - 16 16 - // FeedStore provides persistent storage for the feed registry. 17 17 - // It stores DIDs of users who have logged in and should appear in the community feed. 18 18 - type FeedStore struct { 19 19 - db *bolt.DB 20 20 - } 21 21 - 22 22 - // Register adds a DID to the feed registry. 23 23 - // If the DID already exists, this is a no-op. 24 24 - func (s *FeedStore) Register(did string) error { 25 25 - return s.db.Update(func(tx *bolt.Tx) error { 26 26 - bucket := tx.Bucket(BucketFeedRegistry) 27 27 - if bucket == nil { 28 28 - return nil 29 29 - } 30 30 - 31 31 - // Check if already registered 32 32 - existing := bucket.Get([]byte(did)) 33 33 - if existing != nil { 34 34 - // Already registered, no-op 35 35 - return nil 36 36 - } 37 37 - 38 38 - // Create new registration 39 39 - user := FeedUser{ 40 40 - DID: did, 41 41 - RegisteredAt: time.Now(), 42 42 - } 43 43 - 44 44 - data, err := json.Marshal(user) 45 45 - if err != nil { 46 46 - return err 47 47 - } 48 48 - 49 49 - return bucket.Put([]byte(did), data) 50 50 - }) 51 51 - } 52 52 - 53 53 - // Unregister removes a DID from the feed registry. 54 54 - func (s *FeedStore) Unregister(did string) error { 55 55 - return s.db.Update(func(tx *bolt.Tx) error { 56 56 - bucket := tx.Bucket(BucketFeedRegistry) 57 57 - if bucket == nil { 58 58 - return nil 59 59 - } 60 60 - 61 61 - return bucket.Delete([]byte(did)) 62 62 - }) 63 63 - } 64 64 - 65 65 - // IsRegistered checks if a DID is in the feed registry. 66 66 - func (s *FeedStore) IsRegistered(did string) bool { 67 67 - var registered bool 68 68 - 69 69 - s.db.View(func(tx *bolt.Tx) error { 70 70 - bucket := tx.Bucket(BucketFeedRegistry) 71 71 - if bucket == nil { 72 72 - return nil 73 73 - } 74 74 - 75 75 - registered = bucket.Get([]byte(did)) != nil 76 76 - return nil 77 77 - }) 78 78 - 79 79 - return registered 80 80 - } 81 81 - 82 82 - // List returns all registered DIDs. 83 83 - func (s *FeedStore) List() []string { 84 84 - var dids []string 85 85 - 86 86 - s.db.View(func(tx *bolt.Tx) error { 87 87 - bucket := tx.Bucket(BucketFeedRegistry) 88 88 - if bucket == nil { 89 89 - return nil 90 90 - } 91 91 - 92 92 - return bucket.ForEach(func(k, v []byte) error { 93 93 - dids = append(dids, string(k)) 94 94 - return nil 95 95 - }) 96 96 - }) 97 97 - 98 98 - return dids 99 99 - } 100 100 - 101 101 - // ListWithMetadata returns all registered users with their metadata. 102 102 - func (s *FeedStore) ListWithMetadata() []FeedUser { 103 103 - var users []FeedUser 104 104 - 105 105 - s.db.View(func(tx *bolt.Tx) error { 106 106 - bucket := tx.Bucket(BucketFeedRegistry) 107 107 - if bucket == nil { 108 108 - return nil 109 109 - } 110 110 - 111 111 - return bucket.ForEach(func(k, v []byte) error { 112 112 - var user FeedUser 113 113 - if err := json.Unmarshal(v, &user); err != nil { 114 114 - // Fallback for simple keys without metadata 115 115 - user = FeedUser{DID: string(k)} 116 116 - } 117 117 - users = append(users, user) 118 118 - return nil 119 119 - }) 120 120 - }) 121 121 - 122 122 - return users 123 123 - } 124 124 - 125 125 - // Count returns the number of registered users. 126 126 - func (s *FeedStore) Count() int { 127 127 - var count int 128 128 - 129 129 - s.db.View(func(tx *bolt.Tx) error { 130 130 - bucket := tx.Bucket(BucketFeedRegistry) 131 131 - if bucket == nil { 132 132 - return nil 133 133 - } 134 134 - 135 135 - count = bucket.Stats().KeyN 136 136 - return nil 137 137 - }) 138 138 - 139 139 - return count 140 140 - } 141 141 - 142 142 - // Clear removes all entries from the feed registry. 143 143 - // Use with caution - primarily for testing. 144 144 - func (s *FeedStore) Clear() error { 145 145 - return s.db.Update(func(tx *bolt.Tx) error { 146 146 - // Delete and recreate the bucket 147 147 - if err := tx.DeleteBucket(BucketFeedRegistry); err != nil { 148 148 - // Bucket might not exist, that's ok 149 149 - if err != bolt.ErrBucketNotFound { 150 150 - return err 151 151 - } 152 152 - } 153 153 - 154 154 - _, err := tx.CreateBucket(BucketFeedRegistry) 155 155 - return err 156 156 - }) 157 157 - }

-139

internal/database/boltstore/feed_store_test.go

··· 1 1 - package boltstore 2 2 - 3 3 - import ( 4 4 - "path/filepath" 5 5 - "testing" 6 6 - 7 7 - "github.com/stretchr/testify/assert" 8 8 - "github.com/stretchr/testify/require" 9 9 - ) 10 10 - 11 11 - func setupTestFeedStore(t *testing.T) *FeedStore { 12 12 - tmpDir := t.TempDir() 13 13 - dbPath := filepath.Join(tmpDir, "test.db") 14 14 - 15 15 - store, err := Open(Options{Path: dbPath}) 16 16 - require.NoError(t, err) 17 17 - 18 18 - t.Cleanup(func() { 19 19 - store.Close() 20 20 - }) 21 21 - 22 22 - return store.FeedStore() 23 23 - } 24 24 - 25 25 - func TestFeedStore_Register(t *testing.T) { 26 26 - store := setupTestFeedStore(t) 27 27 - 28 28 - t.Run("register new DID", func(t *testing.T) { 29 29 - err := store.Register("did:plc:user1") 30 30 - require.NoError(t, err) 31 31 - assert.True(t, store.IsRegistered("did:plc:user1")) 32 32 - }) 33 33 - 34 34 - t.Run("register is idempotent", func(t *testing.T) { 35 35 - err := store.Register("did:plc:user2") 36 36 - require.NoError(t, err) 37 37 - 38 38 - err = store.Register("did:plc:user2") 39 39 - require.NoError(t, err) 40 40 - 41 41 - assert.Equal(t, 1, countDID(store, "did:plc:user2")) 42 42 - }) 43 43 - } 44 44 - 45 45 - // countDID counts how many times a DID appears in the list (should be 0 or 1). 46 46 - func countDID(store *FeedStore, did string) int { 47 47 - count := 0 48 48 - for _, d := range store.List() { 49 49 - if d == did { 50 50 - count++ 51 51 - } 52 52 - } 53 53 - return count 54 54 - } 55 55 - 56 56 - func TestFeedStore_Unregister(t *testing.T) { 57 57 - store := setupTestFeedStore(t) 58 58 - 59 59 - err := store.Register("did:plc:unreg") 60 60 - require.NoError(t, err) 61 61 - assert.True(t, store.IsRegistered("did:plc:unreg")) 62 62 - 63 63 - err = store.Unregister("did:plc:unreg") 64 64 - require.NoError(t, err) 65 65 - assert.False(t, store.IsRegistered("did:plc:unreg")) 66 66 - } 67 67 - 68 68 - func TestFeedStore_IsRegistered(t *testing.T) { 69 69 - store := setupTestFeedStore(t) 70 70 - 71 71 - assert.False(t, store.IsRegistered("did:plc:nobody")) 72 72 - 73 73 - store.Register("did:plc:somebody") 74 74 - assert.True(t, store.IsRegistered("did:plc:somebody")) 75 75 - } 76 76 - 77 77 - func TestFeedStore_List(t *testing.T) { 78 78 - store := setupTestFeedStore(t) 79 79 - 80 80 - t.Run("empty store", func(t *testing.T) { 81 81 - dids := store.List() 82 82 - assert.Empty(t, dids) 83 83 - }) 84 84 - 85 85 - t.Run("multiple registrations", func(t *testing.T) { 86 86 - store.Register("did:plc:a") 87 87 - store.Register("did:plc:b") 88 88 - store.Register("did:plc:c") 89 89 - 90 90 - dids := store.List() 91 91 - assert.Len(t, dids, 3) 92 92 - assert.Contains(t, dids, "did:plc:a") 93 93 - assert.Contains(t, dids, "did:plc:b") 94 94 - assert.Contains(t, dids, "did:plc:c") 95 95 - }) 96 96 - } 97 97 - 98 98 - func TestFeedStore_ListWithMetadata(t *testing.T) { 99 99 - store := setupTestFeedStore(t) 100 100 - 101 101 - store.Register("did:plc:meta1") 102 102 - store.Register("did:plc:meta2") 103 103 - 104 104 - users := store.ListWithMetadata() 105 105 - assert.Len(t, users, 2) 106 106 - 107 107 - for _, u := range users { 108 108 - assert.NotEmpty(t, u.DID) 109 109 - assert.False(t, u.RegisteredAt.IsZero()) 110 110 - } 111 111 - } 112 112 - 113 113 - func TestFeedStore_Count(t *testing.T) { 114 114 - store := setupTestFeedStore(t) 115 115 - 116 116 - assert.Equal(t, 0, store.Count()) 117 117 - 118 118 - store.Register("did:plc:c1") 119 119 - assert.Equal(t, 1, store.Count()) 120 120 - 121 121 - store.Register("did:plc:c2") 122 122 - assert.Equal(t, 2, store.Count()) 123 123 - 124 124 - store.Unregister("did:plc:c1") 125 125 - assert.Equal(t, 1, store.Count()) 126 126 - } 127 127 - 128 128 - func TestFeedStore_Clear(t *testing.T) { 129 129 - store := setupTestFeedStore(t) 130 130 - 131 131 - store.Register("did:plc:clear1") 132 132 - store.Register("did:plc:clear2") 133 133 - assert.Equal(t, 2, store.Count()) 134 134 - 135 135 - err := store.Clear() 136 136 - require.NoError(t, err) 137 137 - assert.Equal(t, 0, store.Count()) 138 138 - assert.False(t, store.IsRegistered("did:plc:clear1")) 139 139 - }

-591

internal/database/boltstore/moderation_store.go

··· 1 1 - package boltstore 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "encoding/json" 6 6 - "fmt" 7 7 - "time" 8 8 - 9 9 - "arabica/internal/moderation" 10 10 - 11 11 - bolt "go.etcd.io/bbolt" 12 12 - ) 13 13 - 14 14 - // ModerationStore provides persistent storage for moderation data. 15 15 - type ModerationStore struct { 16 16 - db *bolt.DB 17 17 - } 18 18 - 19 19 - // HideRecord stores a hidden record entry. 20 20 - func (s *ModerationStore) HideRecord(ctx context.Context, entry moderation.HiddenRecord) error { 21 21 - return s.db.Update(func(tx *bolt.Tx) error { 22 22 - bucket := tx.Bucket(BucketModerationHiddenRecords) 23 23 - if bucket == nil { 24 24 - return fmt.Errorf("bucket not found: %s", BucketModerationHiddenRecords) 25 25 - } 26 26 - 27 27 - data, err := json.Marshal(entry) 28 28 - if err != nil { 29 29 - return fmt.Errorf("failed to marshal hidden record: %w", err) 30 30 - } 31 31 - 32 32 - return bucket.Put([]byte(entry.ATURI), data) 33 33 - }) 34 34 - } 35 35 - 36 36 - // UnhideRecord removes a record from the hidden list. 37 37 - func (s *ModerationStore) UnhideRecord(ctx context.Context, atURI string) error { 38 38 - return s.db.Update(func(tx *bolt.Tx) error { 39 39 - bucket := tx.Bucket(BucketModerationHiddenRecords) 40 40 - if bucket == nil { 41 41 - return nil 42 42 - } 43 43 - 44 44 - return bucket.Delete([]byte(atURI)) 45 45 - }) 46 46 - } 47 47 - 48 48 - // IsRecordHidden checks if a record is hidden. 49 49 - func (s *ModerationStore) IsRecordHidden(ctx context.Context, atURI string) bool { 50 50 - var hidden bool 51 51 - 52 52 - s.db.View(func(tx *bolt.Tx) error { 53 53 - bucket := tx.Bucket(BucketModerationHiddenRecords) 54 54 - if bucket == nil { 55 55 - return nil 56 56 - } 57 57 - 58 58 - hidden = bucket.Get([]byte(atURI)) != nil 59 59 - return nil 60 60 - }) 61 61 - 62 62 - return hidden 63 63 - } 64 64 - 65 65 - // GetHiddenRecord retrieves a hidden record entry by AT-URI. 66 66 - func (s *ModerationStore) GetHiddenRecord(ctx context.Context, atURI string) (*moderation.HiddenRecord, error) { 67 67 - var record *moderation.HiddenRecord 68 68 - 69 69 - err := s.db.View(func(tx *bolt.Tx) error { 70 70 - bucket := tx.Bucket(BucketModerationHiddenRecords) 71 71 - if bucket == nil { 72 72 - return nil 73 73 - } 74 74 - 75 75 - data := bucket.Get([]byte(atURI)) 76 76 - if data == nil { 77 77 - return nil 78 78 - } 79 79 - 80 80 - record = &moderation.HiddenRecord{} 81 81 - return json.Unmarshal(data, record) 82 82 - }) 83 83 - 84 84 - return record, err 85 85 - } 86 86 - 87 87 - // ListHiddenRecords returns all hidden records. 88 88 - func (s *ModerationStore) ListHiddenRecords(ctx context.Context) ([]moderation.HiddenRecord, error) { 89 89 - var records []moderation.HiddenRecord 90 90 - 91 91 - err := s.db.View(func(tx *bolt.Tx) error { 92 92 - bucket := tx.Bucket(BucketModerationHiddenRecords) 93 93 - if bucket == nil { 94 94 - return nil 95 95 - } 96 96 - 97 97 - return bucket.ForEach(func(k, v []byte) error { 98 98 - var record moderation.HiddenRecord 99 99 - if err := json.Unmarshal(v, &record); err != nil { 100 100 - return err 101 101 - } 102 102 - records = append(records, record) 103 103 - return nil 104 104 - }) 105 105 - }) 106 106 - 107 107 - return records, err 108 108 - } 109 109 - 110 110 - // BlacklistUser adds a user to the blacklist. 111 111 - func (s *ModerationStore) BlacklistUser(ctx context.Context, entry moderation.BlacklistedUser) error { 112 112 - return s.db.Update(func(tx *bolt.Tx) error { 113 113 - bucket := tx.Bucket(BucketModerationBlacklist) 114 114 - if bucket == nil { 115 115 - return fmt.Errorf("bucket not found: %s", BucketModerationBlacklist) 116 116 - } 117 117 - 118 118 - data, err := json.Marshal(entry) 119 119 - if err != nil { 120 120 - return fmt.Errorf("failed to marshal blacklisted user: %w", err) 121 121 - } 122 122 - 123 123 - return bucket.Put([]byte(entry.DID), data) 124 124 - }) 125 125 - } 126 126 - 127 127 - // UnblacklistUser removes a user from the blacklist. 128 128 - func (s *ModerationStore) UnblacklistUser(ctx context.Context, did string) error { 129 129 - return s.db.Update(func(tx *bolt.Tx) error { 130 130 - bucket := tx.Bucket(BucketModerationBlacklist) 131 131 - if bucket == nil { 132 132 - return nil 133 133 - } 134 134 - 135 135 - return bucket.Delete([]byte(did)) 136 136 - }) 137 137 - } 138 138 - 139 139 - // IsBlacklisted checks if a user is blacklisted. 140 140 - func (s *ModerationStore) IsBlacklisted(ctx context.Context, did string) bool { 141 141 - var blacklisted bool 142 142 - 143 143 - s.db.View(func(tx *bolt.Tx) error { 144 144 - bucket := tx.Bucket(BucketModerationBlacklist) 145 145 - if bucket == nil { 146 146 - return nil 147 147 - } 148 148 - 149 149 - blacklisted = bucket.Get([]byte(did)) != nil 150 150 - return nil 151 151 - }) 152 152 - 153 153 - return blacklisted 154 154 - } 155 155 - 156 156 - // GetBlacklistedUser retrieves a blacklisted user entry by DID. 157 157 - func (s *ModerationStore) GetBlacklistedUser(ctx context.Context, did string) (*moderation.BlacklistedUser, error) { 158 158 - var user *moderation.BlacklistedUser 159 159 - 160 160 - err := s.db.View(func(tx *bolt.Tx) error { 161 161 - bucket := tx.Bucket(BucketModerationBlacklist) 162 162 - if bucket == nil { 163 163 - return nil 164 164 - } 165 165 - 166 166 - data := bucket.Get([]byte(did)) 167 167 - if data == nil { 168 168 - return nil 169 169 - } 170 170 - 171 171 - user = &moderation.BlacklistedUser{} 172 172 - return json.Unmarshal(data, user) 173 173 - }) 174 174 - 175 175 - return user, err 176 176 - } 177 177 - 178 178 - // ListBlacklistedUsers returns all blacklisted users. 179 179 - func (s *ModerationStore) ListBlacklistedUsers(ctx context.Context) ([]moderation.BlacklistedUser, error) { 180 180 - var users []moderation.BlacklistedUser 181 181 - 182 182 - err := s.db.View(func(tx *bolt.Tx) error { 183 183 - bucket := tx.Bucket(BucketModerationBlacklist) 184 184 - if bucket == nil { 185 185 - return nil 186 186 - } 187 187 - 188 188 - return bucket.ForEach(func(k, v []byte) error { 189 189 - var user moderation.BlacklistedUser 190 190 - if err := json.Unmarshal(v, &user); err != nil { 191 191 - return err 192 192 - } 193 193 - users = append(users, user) 194 194 - return nil 195 195 - }) 196 196 - }) 197 197 - 198 198 - return users, err 199 199 - } 200 200 - 201 201 - // CreateReport stores a new report. 202 202 - func (s *ModerationStore) CreateReport(ctx context.Context, report moderation.Report) error { 203 203 - return s.db.Update(func(tx *bolt.Tx) error { 204 204 - // Store the report 205 205 - bucket := tx.Bucket(BucketModerationReports) 206 206 - if bucket == nil { 207 207 - return fmt.Errorf("bucket not found: %s", BucketModerationReports) 208 208 - } 209 209 - 210 210 - data, err := json.Marshal(report) 211 211 - if err != nil { 212 212 - return fmt.Errorf("failed to marshal report: %w", err) 213 213 - } 214 214 - 215 215 - if err := bucket.Put([]byte(report.ID), data); err != nil { 216 216 - return err 217 217 - } 218 218 - 219 219 - // Index by subject URI 220 220 - uriIndex := tx.Bucket(BucketModerationReportsByURI) 221 221 - if uriIndex != nil { 222 222 - // Store report ID in a list for this URI 223 223 - key := []byte(report.SubjectURI + ":" + report.ID) 224 224 - if err := uriIndex.Put(key, []byte(report.ID)); err != nil { 225 225 - return err 226 226 - } 227 227 - } 228 228 - 229 229 - // Index by subject DID 230 230 - didIndex := tx.Bucket(BucketModerationReportsByDID) 231 231 - if didIndex != nil { 232 232 - // Store report ID in a list for this DID 233 233 - key := []byte(report.SubjectDID + ":" + report.ID) 234 234 - if err := didIndex.Put(key, []byte(report.ID)); err != nil { 235 235 - return err 236 236 - } 237 237 - } 238 238 - 239 239 - return nil 240 240 - }) 241 241 - } 242 242 - 243 243 - // GetReport retrieves a report by ID. 244 244 - func (s *ModerationStore) GetReport(ctx context.Context, id string) (*moderation.Report, error) { 245 245 - var report *moderation.Report 246 246 - 247 247 - err := s.db.View(func(tx *bolt.Tx) error { 248 248 - bucket := tx.Bucket(BucketModerationReports) 249 249 - if bucket == nil { 250 250 - return nil 251 251 - } 252 252 - 253 253 - data := bucket.Get([]byte(id)) 254 254 - if data == nil { 255 255 - return nil 256 256 - } 257 257 - 258 258 - report = &moderation.Report{} 259 259 - return json.Unmarshal(data, report) 260 260 - }) 261 261 - 262 262 - return report, err 263 263 - } 264 264 - 265 265 - // ListPendingReports returns all reports with pending status. 266 266 - func (s *ModerationStore) ListPendingReports(ctx context.Context) ([]moderation.Report, error) { 267 267 - var reports []moderation.Report 268 268 - 269 269 - err := s.db.View(func(tx *bolt.Tx) error { 270 270 - bucket := tx.Bucket(BucketModerationReports) 271 271 - if bucket == nil { 272 272 - return nil 273 273 - } 274 274 - 275 275 - return bucket.ForEach(func(k, v []byte) error { 276 276 - var report moderation.Report 277 277 - if err := json.Unmarshal(v, &report); err != nil { 278 278 - return err 279 279 - } 280 280 - if report.Status == moderation.ReportStatusPending { 281 281 - reports = append(reports, report) 282 282 - } 283 283 - return nil 284 284 - }) 285 285 - }) 286 286 - 287 287 - return reports, err 288 288 - } 289 289 - 290 290 - // ListAllReports returns all reports regardless of status. 291 291 - func (s *ModerationStore) ListAllReports(ctx context.Context) ([]moderation.Report, error) { 292 292 - var reports []moderation.Report 293 293 - 294 294 - err := s.db.View(func(tx *bolt.Tx) error { 295 295 - bucket := tx.Bucket(BucketModerationReports) 296 296 - if bucket == nil { 297 297 - return nil 298 298 - } 299 299 - 300 300 - return bucket.ForEach(func(k, v []byte) error { 301 301 - var report moderation.Report 302 302 - if err := json.Unmarshal(v, &report); err != nil { 303 303 - return err 304 304 - } 305 305 - reports = append(reports, report) 306 306 - return nil 307 307 - }) 308 308 - }) 309 309 - 310 310 - return reports, err 311 311 - } 312 312 - 313 313 - // ResolveReport updates a report's status and resolution info. 314 314 - func (s *ModerationStore) ResolveReport(ctx context.Context, id string, status moderation.ReportStatus, resolvedBy string) error { 315 315 - return s.db.Update(func(tx *bolt.Tx) error { 316 316 - bucket := tx.Bucket(BucketModerationReports) 317 317 - if bucket == nil { 318 318 - return fmt.Errorf("bucket not found: %s", BucketModerationReports) 319 319 - } 320 320 - 321 321 - data := bucket.Get([]byte(id)) 322 322 - if data == nil { 323 323 - return fmt.Errorf("report not found: %s", id) 324 324 - } 325 325 - 326 326 - var report moderation.Report 327 327 - if err := json.Unmarshal(data, &report); err != nil { 328 328 - return err 329 329 - } 330 330 - 331 331 - report.Status = status 332 332 - report.ResolvedBy = resolvedBy 333 333 - now := time.Now() 334 334 - report.ResolvedAt = &now 335 335 - 336 336 - newData, err := json.Marshal(report) 337 337 - if err != nil { 338 338 - return err 339 339 - } 340 340 - 341 341 - return bucket.Put([]byte(id), newData) 342 342 - }) 343 343 - } 344 344 - 345 345 - // CountReportsForURI returns the number of reports for a given AT-URI. 346 346 - func (s *ModerationStore) CountReportsForURI(ctx context.Context, atURI string) (int, error) { 347 347 - var count int 348 348 - 349 349 - err := s.db.View(func(tx *bolt.Tx) error { 350 350 - bucket := tx.Bucket(BucketModerationReportsByURI) 351 351 - if bucket == nil { 352 352 - return nil 353 353 - } 354 354 - 355 355 - cursor := bucket.Cursor() 356 356 - prefix := []byte(atURI + ":") 357 357 - 358 358 - for k, _ := cursor.Seek(prefix); k != nil && hasPrefix(k, prefix); k, _ = cursor.Next() { 359 359 - count++ 360 360 - } 361 361 - 362 362 - return nil 363 363 - }) 364 364 - 365 365 - return count, err 366 366 - } 367 367 - 368 368 - // CountReportsForDID returns the number of reports for content by a given DID. 369 369 - func (s *ModerationStore) CountReportsForDID(ctx context.Context, did string) (int, error) { 370 370 - var count int 371 371 - 372 372 - err := s.db.View(func(tx *bolt.Tx) error { 373 373 - bucket := tx.Bucket(BucketModerationReportsByDID) 374 374 - if bucket == nil { 375 375 - return nil 376 376 - } 377 377 - 378 378 - cursor := bucket.Cursor() 379 379 - prefix := []byte(did + ":") 380 380 - 381 381 - for k, _ := cursor.Seek(prefix); k != nil && hasPrefix(k, prefix); k, _ = cursor.Next() { 382 382 - count++ 383 383 - } 384 384 - 385 385 - return nil 386 386 - }) 387 387 - 388 388 - return count, err 389 389 - } 390 390 - 391 391 - // HasReportedURI checks if a user has already reported a specific URI. 392 392 - func (s *ModerationStore) HasReportedURI(ctx context.Context, reporterDID, subjectURI string) (bool, error) { 393 393 - var found bool 394 394 - 395 395 - err := s.db.View(func(tx *bolt.Tx) error { 396 396 - bucket := tx.Bucket(BucketModerationReports) 397 397 - if bucket == nil { 398 398 - return nil 399 399 - } 400 400 - 401 401 - return bucket.ForEach(func(k, v []byte) error { 402 402 - var report moderation.Report 403 403 - if err := json.Unmarshal(v, &report); err != nil { 404 404 - return nil // Skip malformed entries 405 405 - } 406 406 - if report.ReporterDID == reporterDID && report.SubjectURI == subjectURI { 407 407 - found = true 408 408 - } 409 409 - return nil 410 410 - }) 411 411 - }) 412 412 - 413 413 - return found, err 414 414 - } 415 415 - 416 416 - // LogAction stores a moderation action in the audit log. 417 417 - func (s *ModerationStore) LogAction(ctx context.Context, entry moderation.AuditEntry) error { 418 418 - return s.db.Update(func(tx *bolt.Tx) error { 419 419 - bucket := tx.Bucket(BucketModerationAuditLog) 420 420 - if bucket == nil { 421 421 - return fmt.Errorf("bucket not found: %s", BucketModerationAuditLog) 422 422 - } 423 423 - 424 424 - data, err := json.Marshal(entry) 425 425 - if err != nil { 426 426 - return fmt.Errorf("failed to marshal audit entry: %w", err) 427 427 - } 428 428 - 429 429 - // Use timestamp-based key for chronological ordering 430 430 - // Format: timestamp:id for uniqueness 431 431 - key := fmt.Sprintf("%d:%s", entry.Timestamp.UnixNano(), entry.ID) 432 432 - 433 433 - return bucket.Put([]byte(key), data) 434 434 - }) 435 435 - } 436 436 - 437 437 - // ListAuditLog returns the most recent audit log entries. 438 438 - // Entries are returned in reverse chronological order (newest first). 439 439 - func (s *ModerationStore) ListAuditLog(ctx context.Context, limit int) ([]moderation.AuditEntry, error) { 440 440 - var entries []moderation.AuditEntry 441 441 - 442 442 - err := s.db.View(func(tx *bolt.Tx) error { 443 443 - bucket := tx.Bucket(BucketModerationAuditLog) 444 444 - if bucket == nil { 445 445 - return nil 446 446 - } 447 447 - 448 448 - // Collect all entries first (BoltDB cursors iterate in key order) 449 449 - var all []moderation.AuditEntry 450 450 - err := bucket.ForEach(func(k, v []byte) error { 451 451 - var entry moderation.AuditEntry 452 452 - if err := json.Unmarshal(v, &entry); err != nil { 453 453 - return nil // Skip malformed entries 454 454 - } 455 455 - all = append(all, entry) 456 456 - return nil 457 457 - }) 458 458 - if err != nil { 459 459 - return err 460 460 - } 461 461 - 462 462 - // Reverse to get newest first 463 463 - for i := len(all) - 1; i >= 0 && len(entries) < limit; i-- { 464 464 - entries = append(entries, all[i]) 465 465 - } 466 466 - 467 467 - return nil 468 468 - }) 469 469 - 470 470 - return entries, err 471 471 - } 472 472 - 473 473 - // CountReportsFromUserSince counts reports submitted by a user since a given time. 474 474 - // Used for rate limiting report submissions. 475 475 - func (s *ModerationStore) CountReportsFromUserSince(ctx context.Context, reporterDID string, since time.Time) (int, error) { 476 476 - var count int 477 477 - 478 478 - err := s.db.View(func(tx *bolt.Tx) error { 479 479 - bucket := tx.Bucket(BucketModerationReports) 480 480 - if bucket == nil { 481 481 - return nil 482 482 - } 483 483 - 484 484 - return bucket.ForEach(func(k, v []byte) error { 485 485 - var report moderation.Report 486 486 - if err := json.Unmarshal(v, &report); err != nil { 487 487 - return nil // Skip malformed entries 488 488 - } 489 489 - if report.ReporterDID == reporterDID && report.CreatedAt.After(since) { 490 490 - count++ 491 491 - } 492 492 - return nil 493 493 - }) 494 494 - }) 495 495 - 496 496 - return count, err 497 497 - } 498 498 - 499 499 - // SetAutoHideReset stores a reset timestamp for a user's auto-hide counter. 500 500 - // Reports created before this timestamp are ignored when checking the per-user auto-hide threshold. 501 501 - func (s *ModerationStore) SetAutoHideReset(ctx context.Context, did string, resetAt time.Time) error { 502 502 - return s.db.Update(func(tx *bolt.Tx) error { 503 503 - bucket := tx.Bucket(BucketModerationAutoHideResets) 504 504 - if bucket == nil { 505 505 - return fmt.Errorf("bucket not found: %s", BucketModerationAutoHideResets) 506 506 - } 507 507 - 508 508 - data, err := resetAt.MarshalBinary() 509 509 - if err != nil { 510 510 - return fmt.Errorf("failed to marshal reset time: %w", err) 511 511 - } 512 512 - 513 513 - return bucket.Put([]byte(did), data) 514 514 - }) 515 515 - } 516 516 - 517 517 - // GetAutoHideReset returns the auto-hide reset timestamp for a user, or zero time if none set. 518 518 - func (s *ModerationStore) GetAutoHideReset(ctx context.Context, did string) (time.Time, error) { 519 519 - var resetAt time.Time 520 520 - 521 521 - err := s.db.View(func(tx *bolt.Tx) error { 522 522 - bucket := tx.Bucket(BucketModerationAutoHideResets) 523 523 - if bucket == nil { 524 524 - return nil 525 525 - } 526 526 - 527 527 - data := bucket.Get([]byte(did)) 528 528 - if data == nil { 529 529 - return nil 530 530 - } 531 531 - 532 532 - return resetAt.UnmarshalBinary(data) 533 533 - }) 534 534 - 535 535 - return resetAt, err 536 536 - } 537 537 - 538 538 - // CountReportsForDIDSince returns the number of reports for content by a given DID 539 539 - // created after the specified time. 540 540 - func (s *ModerationStore) CountReportsForDIDSince(ctx context.Context, did string, since time.Time) (int, error) { 541 541 - var count int 542 542 - 543 543 - err := s.db.View(func(tx *bolt.Tx) error { 544 544 - didIndex := tx.Bucket(BucketModerationReportsByDID) 545 545 - if didIndex == nil { 546 546 - return nil 547 547 - } 548 548 - 549 549 - reportsBucket := tx.Bucket(BucketModerationReports) 550 550 - if reportsBucket == nil { 551 551 - return nil 552 552 - } 553 553 - 554 554 - cursor := didIndex.Cursor() 555 555 - prefix := []byte(did + ":") 556 556 - 557 557 - for k, v := cursor.Seek(prefix); k != nil && hasPrefix(k, prefix); k, v = cursor.Next() { 558 558 - // v is the report ID 559 559 - reportData := reportsBucket.Get(v) 560 560 - if reportData == nil { 561 561 - continue 562 562 - } 563 563 - 564 564 - var report moderation.Report 565 565 - if err := json.Unmarshal(reportData, &report); err != nil { 566 566 - continue 567 567 - } 568 568 - 569 569 - if report.CreatedAt.After(since) { 570 570 - count++ 571 571 - } 572 572 - } 573 573 - 574 574 - return nil 575 575 - }) 576 576 - 577 577 - return count, err 578 578 - } 579 579 - 580 580 - // hasPrefix checks if a byte slice has a given prefix. 581 581 - func hasPrefix(s, prefix []byte) bool { 582 582 - if len(s) < len(prefix) { 583 583 - return false 584 584 - } 585 585 - for i, b := range prefix { 586 586 - if s[i] != b { 587 587 - return false 588 588 - } 589 589 - } 590 590 - return true 591 591 - }

-453

internal/database/boltstore/moderation_store_test.go

··· 1 1 - package boltstore 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "path/filepath" 6 6 - "testing" 7 7 - "time" 8 8 - 9 9 - "arabica/internal/moderation" 10 10 - 11 11 - "github.com/stretchr/testify/assert" 12 12 - "github.com/stretchr/testify/require" 13 13 - ) 14 14 - 15 15 - func setupTestModerationStore(t *testing.T) *ModerationStore { 16 16 - tmpDir := t.TempDir() 17 17 - dbPath := filepath.Join(tmpDir, "test.db") 18 18 - 19 19 - store, err := Open(Options{Path: dbPath}) 20 20 - require.NoError(t, err) 21 21 - 22 22 - t.Cleanup(func() { 23 23 - store.Close() 24 24 - }) 25 25 - 26 26 - return store.ModerationStore() 27 27 - } 28 28 - 29 29 - func TestHiddenRecords(t *testing.T) { 30 30 - ctx := context.Background() 31 31 - store := setupTestModerationStore(t) 32 32 - 33 33 - t.Run("hide and check record", func(t *testing.T) { 34 34 - entry := moderation.HiddenRecord{ 35 35 - ATURI: "at://did:plc:test/app.bsky.feed.post/abc123", 36 36 - HiddenAt: time.Now(), 37 37 - HiddenBy: "did:plc:admin", 38 38 - Reason: "Spam content", 39 39 - AutoHidden: false, 40 40 - } 41 41 - 42 42 - err := store.HideRecord(ctx, entry) 43 43 - require.NoError(t, err) 44 44 - 45 45 - assert.True(t, store.IsRecordHidden(ctx, entry.ATURI)) 46 46 - assert.False(t, store.IsRecordHidden(ctx, "at://did:plc:other/app.bsky.feed.post/xyz")) 47 47 - }) 48 48 - 49 49 - t.Run("get hidden record", func(t *testing.T) { 50 50 - uri := "at://did:plc:test/social.arabica.alpha.brew/get123" 51 51 - entry := moderation.HiddenRecord{ 52 52 - ATURI: uri, 53 53 - HiddenAt: time.Now(), 54 54 - HiddenBy: "did:plc:mod", 55 55 - Reason: "Inappropriate", 56 56 - AutoHidden: true, 57 57 - } 58 58 - 59 59 - err := store.HideRecord(ctx, entry) 60 60 - require.NoError(t, err) 61 61 - 62 62 - retrieved, err := store.GetHiddenRecord(ctx, uri) 63 63 - require.NoError(t, err) 64 64 - require.NotNil(t, retrieved) 65 65 - 66 66 - assert.Equal(t, uri, retrieved.ATURI) 67 67 - assert.Equal(t, "did:plc:mod", retrieved.HiddenBy) 68 68 - assert.Equal(t, "Inappropriate", retrieved.Reason) 69 69 - assert.True(t, retrieved.AutoHidden) 70 70 - }) 71 71 - 72 72 - t.Run("unhide record", func(t *testing.T) { 73 73 - uri := "at://did:plc:test/social.arabica.alpha.brew/unhide123" 74 74 - entry := moderation.HiddenRecord{ 75 75 - ATURI: uri, 76 76 - HiddenAt: time.Now(), 77 77 - HiddenBy: "did:plc:admin", 78 78 - } 79 79 - 80 80 - err := store.HideRecord(ctx, entry) 81 81 - require.NoError(t, err) 82 82 - assert.True(t, store.IsRecordHidden(ctx, uri)) 83 83 - 84 84 - err = store.UnhideRecord(ctx, uri) 85 85 - require.NoError(t, err) 86 86 - assert.False(t, store.IsRecordHidden(ctx, uri)) 87 87 - }) 88 88 - 89 89 - t.Run("list hidden records", func(t *testing.T) { 90 90 - // Clear by unhiding previous test records 91 91 - store.UnhideRecord(ctx, "at://did:plc:test/app.bsky.feed.post/abc123") 92 92 - store.UnhideRecord(ctx, "at://did:plc:test/social.arabica.alpha.brew/get123") 93 93 - 94 94 - // Add fresh records 95 95 - for i := 0; i < 3; i++ { 96 96 - entry := moderation.HiddenRecord{ 97 97 - ATURI: "at://did:plc:list/social.arabica.alpha.brew/list" + string(rune('0'+i)), 98 98 - HiddenAt: time.Now(), 99 99 - HiddenBy: "did:plc:admin", 100 100 - } 101 101 - require.NoError(t, store.HideRecord(ctx, entry)) 102 102 - } 103 103 - 104 104 - records, err := store.ListHiddenRecords(ctx) 105 105 - require.NoError(t, err) 106 106 - assert.GreaterOrEqual(t, len(records), 3) 107 107 - }) 108 108 - } 109 109 - 110 110 - func TestBlacklist(t *testing.T) { 111 111 - ctx := context.Background() 112 112 - store := setupTestModerationStore(t) 113 113 - 114 114 - t.Run("blacklist and check user", func(t *testing.T) { 115 115 - entry := moderation.BlacklistedUser{ 116 116 - DID: "did:plc:baduser", 117 117 - BlacklistedAt: time.Now(), 118 118 - BlacklistedBy: "did:plc:admin", 119 119 - Reason: "Repeated violations", 120 120 - } 121 121 - 122 122 - err := store.BlacklistUser(ctx, entry) 123 123 - require.NoError(t, err) 124 124 - 125 125 - assert.True(t, store.IsBlacklisted(ctx, "did:plc:baduser")) 126 126 - assert.False(t, store.IsBlacklisted(ctx, "did:plc:gooduser")) 127 127 - }) 128 128 - 129 129 - t.Run("get blacklisted user", func(t *testing.T) { 130 130 - did := "did:plc:getblacklist" 131 131 - entry := moderation.BlacklistedUser{ 132 132 - DID: did, 133 133 - BlacklistedAt: time.Now(), 134 134 - BlacklistedBy: "did:plc:admin", 135 135 - Reason: "Test reason", 136 136 - } 137 137 - 138 138 - err := store.BlacklistUser(ctx, entry) 139 139 - require.NoError(t, err) 140 140 - 141 141 - retrieved, err := store.GetBlacklistedUser(ctx, did) 142 142 - require.NoError(t, err) 143 143 - require.NotNil(t, retrieved) 144 144 - 145 145 - assert.Equal(t, did, retrieved.DID) 146 146 - assert.Equal(t, "did:plc:admin", retrieved.BlacklistedBy) 147 147 - assert.Equal(t, "Test reason", retrieved.Reason) 148 148 - }) 149 149 - 150 150 - t.Run("unblacklist user", func(t *testing.T) { 151 151 - did := "did:plc:unblacklist" 152 152 - entry := moderation.BlacklistedUser{ 153 153 - DID: did, 154 154 - BlacklistedAt: time.Now(), 155 155 - BlacklistedBy: "did:plc:admin", 156 156 - } 157 157 - 158 158 - err := store.BlacklistUser(ctx, entry) 159 159 - require.NoError(t, err) 160 160 - assert.True(t, store.IsBlacklisted(ctx, did)) 161 161 - 162 162 - err = store.UnblacklistUser(ctx, did) 163 163 - require.NoError(t, err) 164 164 - assert.False(t, store.IsBlacklisted(ctx, did)) 165 165 - }) 166 166 - 167 167 - t.Run("list blacklisted users", func(t *testing.T) { 168 168 - users, err := store.ListBlacklistedUsers(ctx) 169 169 - require.NoError(t, err) 170 170 - assert.GreaterOrEqual(t, len(users), 1) 171 171 - }) 172 172 - } 173 173 - 174 174 - func TestReports(t *testing.T) { 175 175 - ctx := context.Background() 176 176 - store := setupTestModerationStore(t) 177 177 - 178 178 - t.Run("create and get report", func(t *testing.T) { 179 179 - report := moderation.Report{ 180 180 - ID: "report001", 181 181 - SubjectURI: "at://did:plc:subject/social.arabica.alpha.brew/abc", 182 182 - SubjectDID: "did:plc:subject", 183 183 - ReporterDID: "did:plc:reporter", 184 184 - Reason: "This is spam", 185 185 - CreatedAt: time.Now(), 186 186 - Status: moderation.ReportStatusPending, 187 187 - } 188 188 - 189 189 - err := store.CreateReport(ctx, report) 190 190 - require.NoError(t, err) 191 191 - 192 192 - retrieved, err := store.GetReport(ctx, "report001") 193 193 - require.NoError(t, err) 194 194 - require.NotNil(t, retrieved) 195 195 - 196 196 - assert.Equal(t, "report001", retrieved.ID) 197 197 - assert.Equal(t, "did:plc:reporter", retrieved.ReporterDID) 198 198 - assert.Equal(t, moderation.ReportStatusPending, retrieved.Status) 199 199 - }) 200 200 - 201 201 - t.Run("list pending reports", func(t *testing.T) { 202 202 - // Create a mix of pending and resolved reports 203 203 - pending := moderation.Report{ 204 204 - ID: "report_pending", 205 205 - SubjectURI: "at://did:plc:sub/social.arabica.alpha.brew/p1", 206 206 - SubjectDID: "did:plc:sub", 207 207 - ReporterDID: "did:plc:rep1", 208 208 - Status: moderation.ReportStatusPending, 209 209 - CreatedAt: time.Now(), 210 210 - } 211 211 - require.NoError(t, store.CreateReport(ctx, pending)) 212 212 - 213 213 - dismissed := moderation.Report{ 214 214 - ID: "report_dismissed", 215 215 - SubjectURI: "at://did:plc:sub/social.arabica.alpha.brew/p2", 216 216 - SubjectDID: "did:plc:sub", 217 217 - ReporterDID: "did:plc:rep2", 218 218 - Status: moderation.ReportStatusDismissed, 219 219 - CreatedAt: time.Now(), 220 220 - } 221 221 - require.NoError(t, store.CreateReport(ctx, dismissed)) 222 222 - 223 223 - reports, err := store.ListPendingReports(ctx) 224 224 - require.NoError(t, err) 225 225 - 226 226 - // Should only include pending reports 227 227 - for _, r := range reports { 228 228 - assert.Equal(t, moderation.ReportStatusPending, r.Status) 229 229 - } 230 230 - }) 231 231 - 232 232 - t.Run("resolve report", func(t *testing.T) { 233 233 - report := moderation.Report{ 234 234 - ID: "report_to_resolve", 235 235 - SubjectURI: "at://did:plc:sub/social.arabica.alpha.brew/resolve", 236 236 - SubjectDID: "did:plc:sub", 237 237 - ReporterDID: "did:plc:rep", 238 238 - Status: moderation.ReportStatusPending, 239 239 - CreatedAt: time.Now(), 240 240 - } 241 241 - require.NoError(t, store.CreateReport(ctx, report)) 242 242 - 243 243 - err := store.ResolveReport(ctx, "report_to_resolve", moderation.ReportStatusActioned, "did:plc:mod") 244 244 - require.NoError(t, err) 245 245 - 246 246 - retrieved, err := store.GetReport(ctx, "report_to_resolve") 247 247 - require.NoError(t, err) 248 248 - 249 249 - assert.Equal(t, moderation.ReportStatusActioned, retrieved.Status) 250 250 - assert.Equal(t, "did:plc:mod", retrieved.ResolvedBy) 251 251 - assert.NotNil(t, retrieved.ResolvedAt) 252 252 - }) 253 253 - 254 254 - t.Run("count reports for URI", func(t *testing.T) { 255 255 - uri := "at://did:plc:counted/social.arabica.alpha.brew/count" 256 256 - 257 257 - for i := 0; i < 3; i++ { 258 258 - report := moderation.Report{ 259 259 - ID: "count_uri_" + string(rune('0'+i)), 260 260 - SubjectURI: uri, 261 261 - SubjectDID: "did:plc:counted", 262 262 - ReporterDID: "did:plc:reporter" + string(rune('0'+i)), 263 263 - Status: moderation.ReportStatusPending, 264 264 - CreatedAt: time.Now(), 265 265 - } 266 266 - require.NoError(t, store.CreateReport(ctx, report)) 267 267 - } 268 268 - 269 269 - count, err := store.CountReportsForURI(ctx, uri) 270 270 - require.NoError(t, err) 271 271 - assert.Equal(t, 3, count) 272 272 - }) 273 273 - 274 274 - t.Run("count reports for DID", func(t *testing.T) { 275 275 - did := "did:plc:counteddid" 276 276 - 277 277 - for i := 0; i < 2; i++ { 278 278 - report := moderation.Report{ 279 279 - ID: "count_did_" + string(rune('0'+i)), 280 280 - SubjectURI: "at://" + did + "/social.arabica.alpha.brew/post" + string(rune('0'+i)), 281 281 - SubjectDID: did, 282 282 - ReporterDID: "did:plc:reporter", 283 283 - Status: moderation.ReportStatusPending, 284 284 - CreatedAt: time.Now(), 285 285 - } 286 286 - require.NoError(t, store.CreateReport(ctx, report)) 287 287 - } 288 288 - 289 289 - count, err := store.CountReportsForDID(ctx, did) 290 290 - require.NoError(t, err) 291 291 - assert.Equal(t, 2, count) 292 292 - }) 293 293 - 294 294 - t.Run("has reported URI", func(t *testing.T) { 295 295 - uri := "at://did:plc:hasreported/social.arabica.alpha.brew/check" 296 296 - reporter := "did:plc:checker" 297 297 - 298 298 - report := moderation.Report{ 299 299 - ID: "has_reported_check", 300 300 - SubjectURI: uri, 301 301 - SubjectDID: "did:plc:hasreported", 302 302 - ReporterDID: reporter, 303 303 - Status: moderation.ReportStatusPending, 304 304 - CreatedAt: time.Now(), 305 305 - } 306 306 - require.NoError(t, store.CreateReport(ctx, report)) 307 307 - 308 308 - has, err := store.HasReportedURI(ctx, reporter, uri) 309 309 - require.NoError(t, err) 310 310 - assert.True(t, has) 311 311 - 312 312 - has, err = store.HasReportedURI(ctx, "did:plc:other", uri) 313 313 - require.NoError(t, err) 314 314 - assert.False(t, has) 315 315 - }) 316 316 - 317 317 - t.Run("count reports from user since", func(t *testing.T) { 318 318 - reporter := "did:plc:ratelimituser" 319 319 - now := time.Now() 320 320 - 321 321 - // Create reports at different times 322 322 - for i := 0; i < 5; i++ { 323 323 - report := moderation.Report{ 324 324 - ID: "ratelimit_" + string(rune('a'+i)), 325 325 - SubjectURI: "at://did:plc:target/social.arabica.alpha.brew/rl" + string(rune('0'+i)), 326 326 - SubjectDID: "did:plc:target", 327 327 - ReporterDID: reporter, 328 328 - Status: moderation.ReportStatusPending, 329 329 - CreatedAt: now.Add(-time.Duration(i*30) * time.Minute), // 0, -30, -60, -90, -120 mins 330 330 - } 331 331 - require.NoError(t, store.CreateReport(ctx, report)) 332 332 - } 333 333 - 334 334 - // Count reports in the last hour (should be 2: 0min and -30min) 335 335 - oneHourAgo := now.Add(-1 * time.Hour) 336 336 - count, err := store.CountReportsFromUserSince(ctx, reporter, oneHourAgo) 337 337 - require.NoError(t, err) 338 338 - assert.Equal(t, 2, count) 339 339 - 340 340 - // Count reports in the last 2 hours (should be 4: 0, -30, -60, -90 mins) 341 341 - twoHoursAgo := now.Add(-2 * time.Hour) 342 342 - count, err = store.CountReportsFromUserSince(ctx, reporter, twoHoursAgo) 343 343 - require.NoError(t, err) 344 344 - assert.Equal(t, 4, count) 345 345 - 346 346 - // Count reports from a different user (should be 0) 347 347 - count, err = store.CountReportsFromUserSince(ctx, "did:plc:otheruser", oneHourAgo) 348 348 - require.NoError(t, err) 349 349 - assert.Equal(t, 0, count) 350 350 - }) 351 351 - } 352 352 - 353 353 - func TestAuditLog(t *testing.T) { 354 354 - ctx := context.Background() 355 355 - store := setupTestModerationStore(t) 356 356 - 357 357 - t.Run("log action", func(t *testing.T) { 358 358 - entry := moderation.AuditEntry{ 359 359 - ID: "audit001", 360 360 - Action: moderation.AuditActionHideRecord, 361 361 - ActorDID: "did:plc:mod", 362 362 - TargetURI: "at://did:plc:target/social.arabica.alpha.brew/abc", 363 363 - Reason: "Spam", 364 364 - Timestamp: time.Now(), 365 365 - AutoMod: false, 366 366 - } 367 367 - 368 368 - err := store.LogAction(ctx, entry) 369 369 - require.NoError(t, err) 370 370 - }) 371 371 - 372 372 - t.Run("list audit log", func(t *testing.T) { 373 373 - // Add several entries with different timestamps 374 374 - now := time.Now() 375 375 - for i := 0; i < 5; i++ { 376 376 - entry := moderation.AuditEntry{ 377 377 - ID: "audit_list_" + string(rune('0'+i)), 378 378 - Action: moderation.AuditActionHideRecord, 379 379 - ActorDID: "did:plc:mod", 380 380 - TargetURI: "at://did:plc:target/social.arabica.alpha.brew/" + string(rune('0'+i)), 381 381 - Timestamp: now.Add(time.Duration(i) * time.Second), 382 382 - } 383 383 - require.NoError(t, store.LogAction(ctx, entry)) 384 384 - } 385 385 - 386 386 - entries, err := store.ListAuditLog(ctx, 3) 387 387 - require.NoError(t, err) 388 388 - assert.Len(t, entries, 3) 389 389 - 390 390 - // Should be in reverse chronological order (newest first) 391 391 - for i := 1; i < len(entries); i++ { 392 392 - assert.True(t, entries[i-1].Timestamp.After(entries[i].Timestamp) || 393 393 - entries[i-1].Timestamp.Equal(entries[i].Timestamp)) 394 394 - } 395 395 - }) 396 396 - 397 397 - t.Run("automod entry", func(t *testing.T) { 398 398 - entry := moderation.AuditEntry{ 399 399 - ID: "audit_automod", 400 400 - Action: moderation.AuditActionHideRecord, 401 401 - ActorDID: "automod", 402 402 - TargetURI: "at://did:plc:auto/social.arabica.alpha.brew/auto", 403 403 - Reason: "Exceeded report threshold", 404 404 - Timestamp: time.Now(), 405 405 - AutoMod: true, 406 406 - } 407 407 - 408 408 - err := store.LogAction(ctx, entry) 409 409 - require.NoError(t, err) 410 410 - 411 411 - entries, err := store.ListAuditLog(ctx, 100) 412 412 - require.NoError(t, err) 413 413 - 414 414 - var found bool 415 415 - for _, e := range entries { 416 416 - if e.ID == "audit_automod" { 417 417 - assert.True(t, e.AutoMod) 418 418 - found = true 419 419 - break 420 420 - } 421 421 - } 422 422 - assert.True(t, found, "automod entry not found") 423 423 - }) 424 424 - } 425 425 - 426 426 - func TestNonExistentRecords(t *testing.T) { 427 427 - ctx := context.Background() 428 428 - store := setupTestModerationStore(t) 429 429 - 430 430 - t.Run("get nonexistent hidden record", func(t *testing.T) { 431 431 - record, err := store.GetHiddenRecord(ctx, "at://nonexistent") 432 432 - require.NoError(t, err) 433 433 - assert.Nil(t, record) 434 434 - }) 435 435 - 436 436 - t.Run("get nonexistent blacklisted user", func(t *testing.T) { 437 437 - user, err := store.GetBlacklistedUser(ctx, "did:plc:nonexistent") 438 438 - require.NoError(t, err) 439 439 - assert.Nil(t, user) 440 440 - }) 441 441 - 442 442 - t.Run("get nonexistent report", func(t *testing.T) { 443 443 - report, err := store.GetReport(ctx, "nonexistent") 444 444 - require.NoError(t, err) 445 445 - assert.Nil(t, report) 446 446 - }) 447 447 - 448 448 - t.Run("resolve nonexistent report", func(t *testing.T) { 449 449 - err := store.ResolveReport(ctx, "nonexistent", moderation.ReportStatusDismissed, "did:plc:mod") 450 450 - assert.Error(t, err) 451 451 - assert.Contains(t, err.Error(), "not found") 452 452 - }) 453 453 - }

+1 -43

internal/database/boltstore/store.go

··· 1 1 // Package boltstore provides persistent storage using BoltDB (bbolt). 2 2 // It implements the oauth.ClientAuthStore interface for session persistence 3 3 - // and provides storage for the feed registry. 3 3 + // and provides storage for join requests. 4 4 package boltstore 5 5 6 6 import ( ··· 20 20 // BucketAuthRequests stores pending OAuth auth requests keyed by state 21 21 BucketAuthRequests = []byte("oauth_auth_requests") 22 22 23 23 - // BucketFeedRegistry stores registered user DIDs for the community feed 24 24 - BucketFeedRegistry = []byte("feed_registry") 25 25 - 26 26 - // BucketModerationHiddenRecords stores AT-URIs of hidden records 27 27 - BucketModerationHiddenRecords = []byte("moderation_hidden_records") 28 28 - 29 29 - // BucketModerationBlacklist stores blacklisted user DIDs 30 30 - BucketModerationBlacklist = []byte("moderation_blacklist") 31 31 - 32 32 - // BucketModerationReports stores user reports on content 33 33 - BucketModerationReports = []byte("moderation_reports") 34 34 - 35 35 - // BucketModerationReportsByURI indexes reports by subject AT-URI 36 36 - BucketModerationReportsByURI = []byte("moderation_reports_by_uri") 37 37 - 38 38 - // BucketModerationReportsByDID indexes reports by subject DID 39 39 - BucketModerationReportsByDID = []byte("moderation_reports_by_did") 40 40 - 41 41 - // BucketModerationAuditLog stores moderation action audit trail 42 42 - BucketModerationAuditLog = []byte("moderation_audit_log") 43 43 - 44 44 - // BucketModerationAutoHideResets stores DID -> timestamp for auto-hide counter resets 45 45 - BucketModerationAutoHideResets = []byte("moderation_autohide_resets") 46 46 - 47 23 // BucketJoinRequests stores PDS account join requests 48 24 BucketJoinRequests = []byte("join_requests") 49 25 ) ··· 110 86 buckets := [][]byte{ 111 87 BucketSessions, 112 88 BucketAuthRequests, 113 113 - BucketFeedRegistry, 114 114 - BucketModerationHiddenRecords, 115 115 - BucketModerationBlacklist, 116 116 - BucketModerationReports, 117 117 - BucketModerationReportsByURI, 118 118 - BucketModerationReportsByDID, 119 119 - BucketModerationAuditLog, 120 120 - BucketModerationAutoHideResets, 121 89 BucketJoinRequests, 122 90 } 123 91 ··· 154 122 // SessionStore returns an OAuth session store backed by this database. 155 123 func (s *Store) SessionStore() *SessionStore { 156 124 return &SessionStore{db: s.db} 157 157 - } 158 158 - 159 159 - // FeedStore returns a feed registry store backed by this database. 160 160 - func (s *Store) FeedStore() *FeedStore { 161 161 - return &FeedStore{db: s.db} 162 162 - } 163 163 - 164 164 - // ModerationStore returns a moderation store backed by this database. 165 165 - func (s *Store) ModerationStore() *ModerationStore { 166 166 - return &ModerationStore{db: s.db} 167 125 } 168 126 169 127 // JoinStore returns a join request store backed by this database.

+378

internal/database/sqlitestore/moderation.go

··· 1 1 + // Package sqlitestore provides SQLite-backed store implementations. 2 2 + package sqlitestore 3 3 + 4 4 + import ( 5 5 + "context" 6 6 + "database/sql" 7 7 + "encoding/json" 8 8 + "fmt" 9 9 + "time" 10 10 + 11 11 + "arabica/internal/moderation" 12 12 + ) 13 13 + 14 14 + // ModerationStore implements moderation.Store using SQLite. 15 15 + // It shares the database connection with the firehose FeedIndex. 16 16 + type ModerationStore struct { 17 17 + db *sql.DB 18 18 + } 19 19 + 20 20 + // NewModerationStore creates a ModerationStore backed by the given database. 21 21 + // The database must already have the moderation schema applied. 22 22 + func NewModerationStore(db *sql.DB) *ModerationStore { 23 23 + return &ModerationStore{db: db} 24 24 + } 25 25 + 26 26 + // Ensure ModerationStore implements the interface at compile time. 27 27 + var _ moderation.Store = (*ModerationStore)(nil) 28 28 + 29 29 + // ========== Hidden Records ========== 30 30 + 31 31 + func (s *ModerationStore) HideRecord(ctx context.Context, entry moderation.HiddenRecord) error { 32 32 + autoHidden := 0 33 33 + if entry.AutoHidden { 34 34 + autoHidden = 1 35 35 + } 36 36 + _, err := s.db.ExecContext(ctx, ` 37 37 + INSERT INTO moderation_hidden_records (uri, hidden_at, hidden_by, reason, auto_hidden) 38 38 + VALUES (?, ?, ?, ?, ?) 39 39 + ON CONFLICT(uri) DO UPDATE SET 40 40 + hidden_at = excluded.hidden_at, 41 41 + hidden_by = excluded.hidden_by, 42 42 + reason = excluded.reason, 43 43 + auto_hidden = excluded.auto_hidden 44 44 + `, entry.ATURI, entry.HiddenAt.Format(time.RFC3339Nano), entry.HiddenBy, entry.Reason, autoHidden) 45 45 + if err != nil { 46 46 + return fmt.Errorf("hide record: %w", err) 47 47 + } 48 48 + return nil 49 49 + } 50 50 + 51 51 + func (s *ModerationStore) UnhideRecord(ctx context.Context, atURI string) error { 52 52 + _, err := s.db.ExecContext(ctx, `DELETE FROM moderation_hidden_records WHERE uri = ?`, atURI) 53 53 + return err 54 54 + } 55 55 + 56 56 + func (s *ModerationStore) IsRecordHidden(ctx context.Context, atURI string) bool { 57 57 + var exists int 58 58 + _ = s.db.QueryRowContext(ctx, `SELECT 1 FROM moderation_hidden_records WHERE uri = ?`, atURI).Scan(&exists) 59 59 + return exists == 1 60 60 + } 61 61 + 62 62 + func (s *ModerationStore) GetHiddenRecord(ctx context.Context, atURI string) (*moderation.HiddenRecord, error) { 63 63 + var r moderation.HiddenRecord 64 64 + var hiddenAtStr string 65 65 + var autoHidden int 66 66 + err := s.db.QueryRowContext(ctx, ` 67 67 + SELECT uri, hidden_at, hidden_by, reason, auto_hidden 68 68 + FROM moderation_hidden_records WHERE uri = ? 69 69 + `, atURI).Scan(&r.ATURI, &hiddenAtStr, &r.HiddenBy, &r.Reason, &autoHidden) 70 70 + if err == sql.ErrNoRows { 71 71 + return nil, nil 72 72 + } 73 73 + if err != nil { 74 74 + return nil, err 75 75 + } 76 76 + r.HiddenAt, _ = time.Parse(time.RFC3339Nano, hiddenAtStr) 77 77 + r.AutoHidden = autoHidden == 1 78 78 + return &r, nil 79 79 + } 80 80 + 81 81 + func (s *ModerationStore) ListHiddenRecords(ctx context.Context) ([]moderation.HiddenRecord, error) { 82 82 + rows, err := s.db.QueryContext(ctx, ` 83 83 + SELECT uri, hidden_at, hidden_by, reason, auto_hidden 84 84 + FROM moderation_hidden_records ORDER BY hidden_at DESC 85 85 + `) 86 86 + if err != nil { 87 87 + return nil, err 88 88 + } 89 89 + defer rows.Close() 90 90 + 91 91 + var records []moderation.HiddenRecord 92 92 + for rows.Next() { 93 93 + var r moderation.HiddenRecord 94 94 + var hiddenAtStr string 95 95 + var autoHidden int 96 96 + if err := rows.Scan(&r.ATURI, &hiddenAtStr, &r.HiddenBy, &r.Reason, &autoHidden); err != nil { 97 97 + continue 98 98 + } 99 99 + r.HiddenAt, _ = time.Parse(time.RFC3339Nano, hiddenAtStr) 100 100 + r.AutoHidden = autoHidden == 1 101 101 + records = append(records, r) 102 102 + } 103 103 + return records, rows.Err() 104 104 + } 105 105 + 106 106 + // ========== Blacklist ========== 107 107 + 108 108 + func (s *ModerationStore) BlacklistUser(ctx context.Context, entry moderation.BlacklistedUser) error { 109 109 + _, err := s.db.ExecContext(ctx, ` 110 110 + INSERT INTO moderation_blacklist (did, blacklisted_at, blacklisted_by, reason) 111 111 + VALUES (?, ?, ?, ?) 112 112 + ON CONFLICT(did) DO UPDATE SET 113 113 + blacklisted_at = excluded.blacklisted_at, 114 114 + blacklisted_by = excluded.blacklisted_by, 115 115 + reason = excluded.reason 116 116 + `, entry.DID, entry.BlacklistedAt.Format(time.RFC3339Nano), entry.BlacklistedBy, entry.Reason) 117 117 + if err != nil { 118 118 + return fmt.Errorf("blacklist user: %w", err) 119 119 + } 120 120 + return nil 121 121 + } 122 122 + 123 123 + func (s *ModerationStore) UnblacklistUser(ctx context.Context, did string) error { 124 124 + _, err := s.db.ExecContext(ctx, `DELETE FROM moderation_blacklist WHERE did = ?`, did) 125 125 + return err 126 126 + } 127 127 + 128 128 + func (s *ModerationStore) IsBlacklisted(ctx context.Context, did string) bool { 129 129 + var exists int 130 130 + _ = s.db.QueryRowContext(ctx, `SELECT 1 FROM moderation_blacklist WHERE did = ?`, did).Scan(&exists) 131 131 + return exists == 1 132 132 + } 133 133 + 134 134 + func (s *ModerationStore) GetBlacklistedUser(ctx context.Context, did string) (*moderation.BlacklistedUser, error) { 135 135 + var u moderation.BlacklistedUser 136 136 + var blacklistedAtStr string 137 137 + err := s.db.QueryRowContext(ctx, ` 138 138 + SELECT did, blacklisted_at, blacklisted_by, reason 139 139 + FROM moderation_blacklist WHERE did = ? 140 140 + `, did).Scan(&u.DID, &blacklistedAtStr, &u.BlacklistedBy, &u.Reason) 141 141 + if err == sql.ErrNoRows { 142 142 + return nil, nil 143 143 + } 144 144 + if err != nil { 145 145 + return nil, err 146 146 + } 147 147 + u.BlacklistedAt, _ = time.Parse(time.RFC3339Nano, blacklistedAtStr) 148 148 + return &u, nil 149 149 + } 150 150 + 151 151 + func (s *ModerationStore) ListBlacklistedUsers(ctx context.Context) ([]moderation.BlacklistedUser, error) { 152 152 + rows, err := s.db.QueryContext(ctx, ` 153 153 + SELECT did, blacklisted_at, blacklisted_by, reason 154 154 + FROM moderation_blacklist ORDER BY blacklisted_at DESC 155 155 + `) 156 156 + if err != nil { 157 157 + return nil, err 158 158 + } 159 159 + defer rows.Close() 160 160 + 161 161 + var users []moderation.BlacklistedUser 162 162 + for rows.Next() { 163 163 + var u moderation.BlacklistedUser 164 164 + var blacklistedAtStr string 165 165 + if err := rows.Scan(&u.DID, &blacklistedAtStr, &u.BlacklistedBy, &u.Reason); err != nil { 166 166 + continue 167 167 + } 168 168 + u.BlacklistedAt, _ = time.Parse(time.RFC3339Nano, blacklistedAtStr) 169 169 + users = append(users, u) 170 170 + } 171 171 + return users, rows.Err() 172 172 + } 173 173 + 174 174 + // ========== Reports ========== 175 175 + 176 176 + func (s *ModerationStore) CreateReport(ctx context.Context, report moderation.Report) error { 177 177 + _, err := s.db.ExecContext(ctx, ` 178 178 + INSERT INTO moderation_reports 179 179 + (id, subject_uri, subject_did, reporter_did, reason, created_at, status, resolved_by, resolved_at) 180 180 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) 181 181 + `, report.ID, report.SubjectURI, report.SubjectDID, report.ReporterDID, report.Reason, 182 182 + report.CreatedAt.Format(time.RFC3339Nano), string(report.Status), report.ResolvedBy, nil) 183 183 + if err != nil { 184 184 + return fmt.Errorf("create report: %w", err) 185 185 + } 186 186 + return nil 187 187 + } 188 188 + 189 189 + func (s *ModerationStore) GetReport(ctx context.Context, id string) (*moderation.Report, error) { 190 190 + var r moderation.Report 191 191 + var createdAtStr string 192 192 + var resolvedAtStr sql.NullString 193 193 + err := s.db.QueryRowContext(ctx, ` 194 194 + SELECT id, subject_uri, subject_did, reporter_did, reason, created_at, status, resolved_by, resolved_at 195 195 + FROM moderation_reports WHERE id = ? 196 196 + `, id).Scan(&r.ID, &r.SubjectURI, &r.SubjectDID, &r.ReporterDID, &r.Reason, 197 197 + &createdAtStr, &r.Status, &r.ResolvedBy, &resolvedAtStr) 198 198 + if err == sql.ErrNoRows { 199 199 + return nil, nil 200 200 + } 201 201 + if err != nil { 202 202 + return nil, err 203 203 + } 204 204 + r.CreatedAt, _ = time.Parse(time.RFC3339Nano, createdAtStr) 205 205 + if resolvedAtStr.Valid { 206 206 + t, _ := time.Parse(time.RFC3339Nano, resolvedAtStr.String) 207 207 + r.ResolvedAt = &t 208 208 + } 209 209 + return &r, nil 210 210 + } 211 211 + 212 212 + func (s *ModerationStore) ListPendingReports(ctx context.Context) ([]moderation.Report, error) { 213 213 + return s.listReports(ctx, `WHERE status = 'pending' ORDER BY created_at DESC`) 214 214 + } 215 215 + 216 216 + func (s *ModerationStore) ListAllReports(ctx context.Context) ([]moderation.Report, error) { 217 217 + return s.listReports(ctx, `ORDER BY created_at DESC`) 218 218 + } 219 219 + 220 220 + func (s *ModerationStore) listReports(ctx context.Context, clause string) ([]moderation.Report, error) { 221 221 + rows, err := s.db.QueryContext(ctx, ` 222 222 + SELECT id, subject_uri, subject_did, reporter_did, reason, created_at, status, resolved_by, resolved_at 223 223 + FROM moderation_reports `+clause) 224 224 + if err != nil { 225 225 + return nil, err 226 226 + } 227 227 + defer rows.Close() 228 228 + return scanReports(rows) 229 229 + } 230 230 + 231 231 + func scanReports(rows *sql.Rows) ([]moderation.Report, error) { 232 232 + var reports []moderation.Report 233 233 + for rows.Next() { 234 234 + var r moderation.Report 235 235 + var createdAtStr string 236 236 + var resolvedAtStr sql.NullString 237 237 + if err := rows.Scan(&r.ID, &r.SubjectURI, &r.SubjectDID, &r.ReporterDID, &r.Reason, 238 238 + &createdAtStr, &r.Status, &r.ResolvedBy, &resolvedAtStr); err != nil { 239 239 + continue 240 240 + } 241 241 + r.CreatedAt, _ = time.Parse(time.RFC3339Nano, createdAtStr) 242 242 + if resolvedAtStr.Valid { 243 243 + t, _ := time.Parse(time.RFC3339Nano, resolvedAtStr.String) 244 244 + r.ResolvedAt = &t 245 245 + } 246 246 + reports = append(reports, r) 247 247 + } 248 248 + return reports, rows.Err() 249 249 + } 250 250 + 251 251 + func (s *ModerationStore) ResolveReport(ctx context.Context, id string, status moderation.ReportStatus, resolvedBy string) error { 252 252 + now := time.Now().Format(time.RFC3339Nano) 253 253 + res, err := s.db.ExecContext(ctx, ` 254 254 + UPDATE moderation_reports SET status = ?, resolved_by = ?, resolved_at = ? WHERE id = ? 255 255 + `, string(status), resolvedBy, now, id) 256 256 + if err != nil { 257 257 + return fmt.Errorf("resolve report: %w", err) 258 258 + } 259 259 + n, _ := res.RowsAffected() 260 260 + if n == 0 { 261 261 + return fmt.Errorf("report not found: %s", id) 262 262 + } 263 263 + return nil 264 264 + } 265 265 + 266 266 + func (s *ModerationStore) CountReportsForURI(ctx context.Context, atURI string) (int, error) { 267 267 + var count int 268 268 + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM moderation_reports WHERE subject_uri = ?`, atURI).Scan(&count) 269 269 + return count, err 270 270 + } 271 271 + 272 272 + func (s *ModerationStore) CountReportsForDID(ctx context.Context, did string) (int, error) { 273 273 + var count int 274 274 + err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM moderation_reports WHERE subject_did = ?`, did).Scan(&count) 275 275 + return count, err 276 276 + } 277 277 + 278 278 + func (s *ModerationStore) CountReportsForDIDSince(ctx context.Context, did string, since time.Time) (int, error) { 279 279 + var count int 280 280 + err := s.db.QueryRowContext(ctx, ` 281 281 + SELECT COUNT(*) FROM moderation_reports WHERE subject_did = ? AND created_at > ? 282 282 + `, did, since.Format(time.RFC3339Nano)).Scan(&count) 283 283 + return count, err 284 284 + } 285 285 + 286 286 + func (s *ModerationStore) HasReportedURI(ctx context.Context, reporterDID, subjectURI string) (bool, error) { 287 287 + var exists int 288 288 + err := s.db.QueryRowContext(ctx, ` 289 289 + SELECT 1 FROM moderation_reports WHERE reporter_did = ? AND subject_uri = ? LIMIT 1 290 290 + `, reporterDID, subjectURI).Scan(&exists) 291 291 + if err == sql.ErrNoRows { 292 292 + return false, nil 293 293 + } 294 294 + return exists == 1, err 295 295 + } 296 296 + 297 297 + func (s *ModerationStore) CountReportsFromUserSince(ctx context.Context, reporterDID string, since time.Time) (int, error) { 298 298 + var count int 299 299 + err := s.db.QueryRowContext(ctx, ` 300 300 + SELECT COUNT(*) FROM moderation_reports WHERE reporter_did = ? AND created_at > ? 301 301 + `, reporterDID, since.Format(time.RFC3339Nano)).Scan(&count) 302 302 + return count, err 303 303 + } 304 304 + 305 305 + // ========== Audit Log ========== 306 306 + 307 307 + func (s *ModerationStore) LogAction(ctx context.Context, entry moderation.AuditEntry) error { 308 308 + details, err := json.Marshal(entry.Details) 309 309 + if err != nil { 310 310 + details = []byte("{}") 311 311 + } 312 312 + autoMod := 0 313 313 + if entry.AutoMod { 314 314 + autoMod = 1 315 315 + } 316 316 + _, err = s.db.ExecContext(ctx, ` 317 317 + INSERT INTO moderation_audit_log (id, action, actor_did, target_uri, reason, details, timestamp, auto_mod) 318 318 + VALUES (?, ?, ?, ?, ?, ?, ?, ?) 319 319 + `, entry.ID, string(entry.Action), entry.ActorDID, entry.TargetURI, entry.Reason, 320 320 + string(details), entry.Timestamp.Format(time.RFC3339Nano), autoMod) 321 321 + if err != nil { 322 322 + return fmt.Errorf("log action: %w", err) 323 323 + } 324 324 + return nil 325 325 + } 326 326 + 327 327 + func (s *ModerationStore) ListAuditLog(ctx context.Context, limit int) ([]moderation.AuditEntry, error) { 328 328 + rows, err := s.db.QueryContext(ctx, ` 329 329 + SELECT id, action, actor_did, target_uri, reason, details, timestamp, auto_mod 330 330 + FROM moderation_audit_log ORDER BY timestamp DESC LIMIT ? 331 331 + `, limit) 332 332 + if err != nil { 333 333 + return nil, err 334 334 + } 335 335 + defer rows.Close() 336 336 + 337 337 + var entries []moderation.AuditEntry 338 338 + for rows.Next() { 339 339 + var e moderation.AuditEntry 340 340 + var timestampStr, detailsStr string 341 341 + var autoMod int 342 342 + if err := rows.Scan(&e.ID, &e.Action, &e.ActorDID, &e.TargetURI, &e.Reason, 343 343 + &detailsStr, &timestampStr, &autoMod); err != nil { 344 344 + continue 345 345 + } 346 346 + e.Timestamp, _ = time.Parse(time.RFC3339Nano, timestampStr) 347 347 + e.AutoMod = autoMod == 1 348 348 + _ = json.Unmarshal([]byte(detailsStr), &e.Details) 349 349 + entries = append(entries, e) 350 350 + } 351 351 + return entries, rows.Err() 352 352 + } 353 353 + 354 354 + // ========== Auto-hide Resets ========== 355 355 + 356 356 + func (s *ModerationStore) SetAutoHideReset(ctx context.Context, did string, resetAt time.Time) error { 357 357 + _, err := s.db.ExecContext(ctx, ` 358 358 + INSERT INTO moderation_autohide_resets (did, reset_at) VALUES (?, ?) 359 359 + ON CONFLICT(did) DO UPDATE SET reset_at = excluded.reset_at 360 360 + `, did, resetAt.Format(time.RFC3339Nano)) 361 361 + if err != nil { 362 362 + return fmt.Errorf("set autohide reset: %w", err) 363 363 + } 364 364 + return nil 365 365 + } 366 366 + 367 367 + func (s *ModerationStore) GetAutoHideReset(ctx context.Context, did string) (time.Time, error) { 368 368 + var resetAtStr string 369 369 + err := s.db.QueryRowContext(ctx, `SELECT reset_at FROM moderation_autohide_resets WHERE did = ?`, did).Scan(&resetAtStr) 370 370 + if err == sql.ErrNoRows { 371 371 + return time.Time{}, nil 372 372 + } 373 373 + if err != nil { 374 374 + return time.Time{}, err 375 375 + } 376 376 + t, _ := time.Parse(time.RFC3339Nano, resetAtStr) 377 377 + return t, nil 378 378 + }

+2 -2

internal/firehose/config.go

··· 1 1 // Package firehose provides real-time AT Protocol event consumption via Jetstream. 2 2 - // It indexes Arabica records into a local BoltDB database for fast feed queries. 2 2 + // It indexes Arabica records into a local SQLite database for fast feed queries. 3 3 package firehose 4 4 5 5 import ( ··· 36 36 // Compress enables zstd compression (~56% bandwidth reduction) 37 37 Compress bool 38 38 39 39 - // IndexPath is the path to the BoltDB feed index database 39 39 + // IndexPath is the path to the SQLite feed index database 40 40 IndexPath string 41 41 42 42 // ProfileCacheTTL is how long to cache profile data

+479 -890

internal/firehose/index.go

··· 1 1 package firehose 2 2 3 3 import ( 4 4 - "bytes" 5 4 "context" 6 6 - "encoding/binary" 7 7 - "encoding/hex" 5 5 + "database/sql" 8 6 "encoding/json" 9 7 "fmt" 10 8 "os" ··· 19 17 "arabica/internal/models" 20 18 21 19 "github.com/rs/zerolog/log" 22 22 - bolt "go.etcd.io/bbolt" 23 23 - ) 24 24 - 25 25 - // Bucket names for the feed index 26 26 - var ( 27 27 - // BucketRecords stores full record data: {at-uri} -> {IndexedRecord JSON} 28 28 - BucketRecords = []byte("records") 29 29 - 30 30 - // BucketByTime stores records by timestamp for chronological queries: {timestamp:at-uri} -> {} 31 31 - BucketByTime = []byte("by_time") 32 32 - 33 33 - // BucketByDID stores records by DID for user-specific queries: {did:at-uri} -> {} 34 34 - BucketByDID = []byte("by_did") 35 35 - 36 36 - // BucketByCollection stores records by type: {collection:timestamp:at-uri} -> {} 37 37 - BucketByCollection = []byte("by_collection") 38 38 - 39 39 - // BucketProfiles stores cached profile data: {did} -> {CachedProfile JSON} 40 40 - BucketProfiles = []byte("profiles") 41 41 - 42 42 - // BucketMeta stores metadata like cursor position: {key} -> {value} 43 43 - BucketMeta = []byte("meta") 44 44 - 45 45 - // BucketKnownDIDs stores all DIDs we've seen with Arabica records 46 46 - BucketKnownDIDs = []byte("known_dids") 47 47 - 48 48 - // BucketBackfilled stores DIDs that have been backfilled: {did} -> {timestamp} 49 49 - BucketBackfilled = []byte("backfilled") 50 50 - 51 51 - // BucketLikes stores like mappings: {subject_uri:actor_did} -> {rkey} 52 52 - BucketLikes = []byte("likes") 53 53 - 54 54 - // BucketLikeCounts stores aggregated like counts: {subject_uri} -> {uint64 count} 55 55 - BucketLikeCounts = []byte("like_counts") 56 56 - 57 57 - // BucketLikesByActor stores likes by actor for lookup: {actor_did:subject_uri} -> {rkey} 58 58 - BucketLikesByActor = []byte("likes_by_actor") 59 59 - 60 60 - // BucketComments stores comment data: {subject_uri:timestamp:actor_did} -> {comment JSON} 61 61 - BucketComments = []byte("comments") 62 62 - 63 63 - // BucketCommentCounts stores aggregated comment counts: {subject_uri} -> {uint64 count} 64 64 - BucketCommentCounts = []byte("comment_counts") 65 65 - 66 66 - // BucketCommentsByActor stores comments by actor for lookup: {actor_did:rkey} -> {subject_uri} 67 67 - BucketCommentsByActor = []byte("comments_by_actor") 68 68 - 69 69 - // BucketCommentChildren stores parent-child relationships: {parent_uri:child_rkey} -> {child_actor_did} 70 70 - BucketCommentChildren = []byte("comment_children") 20 20 + _ "modernc.org/sqlite" 71 21 ) 72 22 73 23 // FeedableRecordTypes are the record types that should appear as feed items. ··· 89 39 Record json.RawMessage `json:"record"` 90 40 CID string `json:"cid"` 91 41 IndexedAt time.Time `json:"indexed_at"` 92 92 - CreatedAt time.Time `json:"created_at"` // Parsed from record 42 42 + CreatedAt time.Time `json:"created_at"` 93 43 } 94 44 95 45 // CachedProfile stores profile data with TTL ··· 101 51 102 52 // FeedIndex provides persistent storage for firehose events 103 53 type FeedIndex struct { 104 104 - db *bolt.DB 54 54 + db *sql.DB 105 55 publicClient *atproto.PublicClient 106 56 profileTTL time.Duration 107 57 ··· 123 73 124 74 // FeedQuery specifies filtering, sorting, and pagination for feed queries 125 75 type FeedQuery struct { 126 126 - Limit int // Max items to return 127 127 - Cursor string // Opaque cursor for pagination (base64-encoded time key) 128 128 - TypeFilter lexicons.RecordType // Filter to a specific record type (empty = all) 129 129 - Sort FeedSort // Sort order (default: recent) 76 76 + Limit int // Max items to return 77 77 + Cursor string // Opaque cursor for pagination (created_at|uri) 78 78 + TypeFilter lexicons.RecordType // Filter to a specific record type (empty = all) 79 79 + Sort FeedSort // Sort order (default: recent) 130 80 } 131 81 132 82 // FeedResult contains feed items plus pagination info ··· 135 85 NextCursor string // Empty if no more results 136 86 } 137 87 138 138 - // NewFeedIndex creates a new feed index backed by BoltDB 88 88 + const schemaNoTrailingPragma = ` 89 89 + CREATE TABLE IF NOT EXISTS records ( 90 90 + uri TEXT PRIMARY KEY, 91 91 + did TEXT NOT NULL, 92 92 + collection TEXT NOT NULL, 93 93 + rkey TEXT NOT NULL, 94 94 + record TEXT NOT NULL, 95 95 + cid TEXT NOT NULL DEFAULT '', 96 96 + indexed_at TEXT NOT NULL, 97 97 + created_at TEXT NOT NULL 98 98 + ); 99 99 + CREATE INDEX IF NOT EXISTS idx_records_created ON records(created_at DESC); 100 100 + CREATE INDEX IF NOT EXISTS idx_records_did ON records(did); 101 101 + CREATE INDEX IF NOT EXISTS idx_records_coll_created ON records(collection, created_at DESC); 102 102 + 103 103 + CREATE TABLE IF NOT EXISTS meta ( 104 104 + key TEXT PRIMARY KEY, 105 105 + value BLOB 106 106 + ); 107 107 + 108 108 + CREATE TABLE IF NOT EXISTS known_dids (did TEXT PRIMARY KEY); 109 109 + CREATE TABLE IF NOT EXISTS backfilled (did TEXT PRIMARY KEY, backfilled_at TEXT NOT NULL); 110 110 + 111 111 + CREATE TABLE IF NOT EXISTS profiles ( 112 112 + did TEXT PRIMARY KEY, 113 113 + data TEXT NOT NULL, 114 114 + expires_at TEXT NOT NULL 115 115 + ); 116 116 + 117 117 + CREATE TABLE IF NOT EXISTS likes ( 118 118 + subject_uri TEXT NOT NULL, 119 119 + actor_did TEXT NOT NULL, 120 120 + rkey TEXT NOT NULL, 121 121 + PRIMARY KEY (subject_uri, actor_did) 122 122 + ); 123 123 + CREATE INDEX IF NOT EXISTS idx_likes_actor ON likes(actor_did, subject_uri); 124 124 + 125 125 + CREATE TABLE IF NOT EXISTS comments ( 126 126 + actor_did TEXT NOT NULL, 127 127 + rkey TEXT NOT NULL, 128 128 + subject_uri TEXT NOT NULL, 129 129 + parent_uri TEXT NOT NULL DEFAULT '', 130 130 + parent_rkey TEXT NOT NULL DEFAULT '', 131 131 + cid TEXT NOT NULL DEFAULT '', 132 132 + text TEXT NOT NULL, 133 133 + created_at TEXT NOT NULL, 134 134 + PRIMARY KEY (actor_did, rkey) 135 135 + ); 136 136 + CREATE INDEX IF NOT EXISTS idx_comments_subject ON comments(subject_uri, created_at); 137 137 + 138 138 + CREATE TABLE IF NOT EXISTS notifications ( 139 139 + id TEXT NOT NULL, 140 140 + target_did TEXT NOT NULL, 141 141 + type TEXT NOT NULL, 142 142 + actor_did TEXT NOT NULL, 143 143 + subject_uri TEXT NOT NULL, 144 144 + created_at TEXT NOT NULL 145 145 + ); 146 146 + CREATE INDEX IF NOT EXISTS idx_notif_target ON notifications(target_did, created_at DESC); 147 147 + CREATE UNIQUE INDEX IF NOT EXISTS idx_notif_dedup ON notifications(target_did, type, actor_did, subject_uri); 148 148 + 149 149 + CREATE TABLE IF NOT EXISTS notifications_meta ( 150 150 + target_did TEXT PRIMARY KEY, 151 151 + last_read TEXT NOT NULL 152 152 + ); 153 153 + 154 154 + CREATE TABLE IF NOT EXISTS moderation_hidden_records ( 155 155 + uri TEXT PRIMARY KEY, 156 156 + hidden_at TEXT NOT NULL, 157 157 + hidden_by TEXT NOT NULL, 158 158 + reason TEXT NOT NULL DEFAULT '', 159 159 + auto_hidden INTEGER NOT NULL DEFAULT 0 160 160 + ); 161 161 + 162 162 + CREATE TABLE IF NOT EXISTS moderation_blacklist ( 163 163 + did TEXT PRIMARY KEY, 164 164 + blacklisted_at TEXT NOT NULL, 165 165 + blacklisted_by TEXT NOT NULL, 166 166 + reason TEXT NOT NULL DEFAULT '' 167 167 + ); 168 168 + 169 169 + CREATE TABLE IF NOT EXISTS moderation_reports ( 170 170 + id TEXT PRIMARY KEY, 171 171 + subject_uri TEXT NOT NULL DEFAULT '', 172 172 + subject_did TEXT NOT NULL DEFAULT '', 173 173 + reporter_did TEXT NOT NULL, 174 174 + reason TEXT NOT NULL, 175 175 + created_at TEXT NOT NULL, 176 176 + status TEXT NOT NULL DEFAULT 'pending', 177 177 + resolved_by TEXT NOT NULL DEFAULT '', 178 178 + resolved_at TEXT 179 179 + ); 180 180 + CREATE INDEX IF NOT EXISTS idx_modreports_uri ON moderation_reports(subject_uri); 181 181 + CREATE INDEX IF NOT EXISTS idx_modreports_did ON moderation_reports(subject_did); 182 182 + CREATE INDEX IF NOT EXISTS idx_modreports_reporter ON moderation_reports(reporter_did, created_at); 183 183 + CREATE INDEX IF NOT EXISTS idx_modreports_status ON moderation_reports(status); 184 184 + 185 185 + CREATE TABLE IF NOT EXISTS moderation_audit_log ( 186 186 + id TEXT PRIMARY KEY, 187 187 + action TEXT NOT NULL, 188 188 + actor_did TEXT NOT NULL, 189 189 + target_uri TEXT NOT NULL DEFAULT '', 190 190 + reason TEXT NOT NULL DEFAULT '', 191 191 + details TEXT NOT NULL DEFAULT '{}', 192 192 + timestamp TEXT NOT NULL, 193 193 + auto_mod INTEGER NOT NULL DEFAULT 0 194 194 + ); 195 195 + CREATE INDEX IF NOT EXISTS idx_modaudit_ts ON moderation_audit_log(timestamp DESC); 196 196 + 197 197 + CREATE TABLE IF NOT EXISTS moderation_autohide_resets ( 198 198 + did TEXT PRIMARY KEY, 199 199 + reset_at TEXT NOT NULL 200 200 + ); 201 201 + ` 202 202 + 203 203 + // NewFeedIndex creates a new feed index backed by SQLite 139 204 func NewFeedIndex(path string, profileTTL time.Duration) (*FeedIndex, error) { 140 205 if path == "" { 141 206 return nil, fmt.Errorf("index path is required") ··· 149 214 } 150 215 } 151 216 152 152 - db, err := bolt.Open(path, 0600, &bolt.Options{ 153 153 - Timeout: 5 * time.Second, 154 154 - }) 217 217 + db, err := sql.Open("sqlite", "file:"+path+"?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)&_pragma=temp_store(MEMORY)&_pragma=mmap_size(134217728)&_pragma=cache_size(-65536)") 155 218 if err != nil { 156 219 return nil, fmt.Errorf("failed to open index database: %w", err) 157 220 } 158 221 159 159 - // Create buckets 160 160 - err = db.Update(func(tx *bolt.Tx) error { 161 161 - buckets := [][]byte{ 162 162 - BucketRecords, 163 163 - BucketByTime, 164 164 - BucketByDID, 165 165 - BucketByCollection, 166 166 - BucketProfiles, 167 167 - BucketMeta, 168 168 - BucketKnownDIDs, 169 169 - BucketBackfilled, 170 170 - BucketLikes, 171 171 - BucketLikeCounts, 172 172 - BucketLikesByActor, 173 173 - BucketComments, 174 174 - BucketCommentCounts, 175 175 - BucketCommentsByActor, 176 176 - BucketCommentChildren, 177 177 - BucketNotifications, 178 178 - BucketNotificationsMeta, 179 179 - } 180 180 - for _, bucket := range buckets { 181 181 - if _, err := tx.CreateBucketIfNotExists(bucket); err != nil { 182 182 - return fmt.Errorf("failed to create bucket %s: %w", bucket, err) 183 183 - } 184 184 - } 185 185 - return nil 186 186 - }) 187 187 - if err != nil { 222 222 + // WAL mode allows concurrent reads with a single writer. 223 223 + // Allow multiple reader connections but limit to avoid file descriptor exhaustion. 224 224 + db.SetMaxOpenConns(4) 225 225 + db.SetMaxIdleConns(4) 226 226 + 227 227 + // Execute schema (skip PRAGMAs — already set via DSN) 228 228 + if _, err := db.Exec(schemaNoTrailingPragma); err != nil { 188 229 _ = db.Close() 189 189 - return nil, err 230 230 + return nil, fmt.Errorf("failed to initialize schema: %w", err) 190 231 } 191 232 192 233 idx := &FeedIndex{ ··· 199 240 return idx, nil 200 241 } 201 242 243 243 + // DB returns the underlying database connection for shared use by other stores. 244 244 + func (idx *FeedIndex) DB() *sql.DB { 245 245 + return idx.db 246 246 + } 247 247 + 202 248 // Close closes the index database 203 249 func (idx *FeedIndex) Close() error { 204 250 if idx.db != nil { ··· 224 270 // GetCursor returns the last processed cursor (microseconds timestamp) 225 271 func (idx *FeedIndex) GetCursor() (int64, error) { 226 272 var cursor int64 227 227 - err := idx.db.View(func(tx *bolt.Tx) error { 228 228 - b := tx.Bucket(BucketMeta) 229 229 - v := b.Get([]byte("cursor")) 230 230 - if len(v) == 8 { 231 231 - cursor = int64(binary.BigEndian.Uint64(v)) 232 232 - } 233 233 - return nil 234 234 - }) 273 273 + err := idx.db.QueryRow(`SELECT value FROM meta WHERE key = 'cursor'`).Scan(&cursor) 274 274 + if err == sql.ErrNoRows { 275 275 + return 0, nil 276 276 + } 235 277 return cursor, err 236 278 } 237 279 238 280 // SetCursor stores the cursor position 239 281 func (idx *FeedIndex) SetCursor(cursor int64) error { 240 240 - return idx.db.Update(func(tx *bolt.Tx) error { 241 241 - b := tx.Bucket(BucketMeta) 242 242 - buf := make([]byte, 8) 243 243 - binary.BigEndian.PutUint64(buf, uint64(cursor)) 244 244 - return b.Put([]byte("cursor"), buf) 245 245 - }) 282 282 + _, err := idx.db.Exec(`INSERT OR REPLACE INTO meta (key, value) VALUES ('cursor', ?)`, cursor) 283 283 + return err 246 284 } 247 285 248 286 // UpsertRecord adds or updates a record in the index ··· 260 298 } 261 299 } 262 300 263 263 - indexed := &IndexedRecord{ 264 264 - URI: uri, 265 265 - DID: did, 266 266 - Collection: collection, 267 267 - RKey: rkey, 268 268 - Record: record, 269 269 - CID: cid, 270 270 - IndexedAt: time.Now(), 271 271 - CreatedAt: createdAt, 301 301 + now := time.Now() 302 302 + 303 303 + _, err := idx.db.Exec(` 304 304 + INSERT INTO records (uri, did, collection, rkey, record, cid, indexed_at, created_at) 305 305 + VALUES (?, ?, ?, ?, ?, ?, ?, ?) 306 306 + ON CONFLICT(uri) DO UPDATE SET 307 307 + record = excluded.record, 308 308 + cid = excluded.cid, 309 309 + indexed_at = excluded.indexed_at, 310 310 + created_at = excluded.created_at 311 311 + `, uri, did, collection, rkey, string(record), cid, 312 312 + now.Format(time.RFC3339Nano), createdAt.Format(time.RFC3339Nano)) 313 313 + if err != nil { 314 314 + return fmt.Errorf("failed to upsert record: %w", err) 272 315 } 273 316 274 274 - data, err := json.Marshal(indexed) 317 317 + // Track known DID 318 318 + _, err = idx.db.Exec(`INSERT OR IGNORE INTO known_dids (did) VALUES (?)`, did) 275 319 if err != nil { 276 276 - return fmt.Errorf("failed to marshal record: %w", err) 320 320 + return fmt.Errorf("failed to track known DID: %w", err) 277 321 } 278 322 279 279 - return idx.db.Update(func(tx *bolt.Tx) error { 280 280 - // Store the record 281 281 - records := tx.Bucket(BucketRecords) 282 282 - if err := records.Put([]byte(uri), data); err != nil { 283 283 - return err 284 284 - } 285 285 - 286 286 - // Index by time (use createdAt for sorting, not event time) 287 287 - byTime := tx.Bucket(BucketByTime) 288 288 - timeKey := makeTimeKey(createdAt, uri) 289 289 - if err := byTime.Put(timeKey, nil); err != nil { 290 290 - return err 291 291 - } 292 292 - 293 293 - // Index by DID 294 294 - byDID := tx.Bucket(BucketByDID) 295 295 - didKey := []byte(did + ":" + uri) 296 296 - if err := byDID.Put(didKey, nil); err != nil { 297 297 - return err 298 298 - } 299 299 - 300 300 - // Index by collection 301 301 - byCollection := tx.Bucket(BucketByCollection) 302 302 - collKey := []byte(collection + ":" + string(timeKey)) 303 303 - if err := byCollection.Put(collKey, nil); err != nil { 304 304 - return err 305 305 - } 306 306 - 307 307 - // Track known DID 308 308 - knownDIDs := tx.Bucket(BucketKnownDIDs) 309 309 - if err := knownDIDs.Put([]byte(did), []byte("1")); err != nil { 310 310 - return err 311 311 - } 312 312 - 313 313 - return nil 314 314 - }) 323 323 + return nil 315 324 } 316 325 317 326 // DeleteRecord removes a record from the index 318 327 func (idx *FeedIndex) DeleteRecord(did, collection, rkey string) error { 319 328 uri := atproto.BuildATURI(did, collection, rkey) 329 329 + _, err := idx.db.Exec(`DELETE FROM records WHERE uri = ?`, uri) 330 330 + return err 331 331 + } 320 332 321 321 - return idx.db.Update(func(tx *bolt.Tx) error { 322 322 - // Get the existing record to find its timestamp 323 323 - records := tx.Bucket(BucketRecords) 324 324 - existingData := records.Get([]byte(uri)) 325 325 - if existingData == nil { 326 326 - // Record doesn't exist, nothing to delete 327 327 - return nil 328 328 - } 333 333 + // GetRecord retrieves a single record by URI 334 334 + func (idx *FeedIndex) GetRecord(uri string) (*IndexedRecord, error) { 335 335 + var rec IndexedRecord 336 336 + var recordStr, indexedAtStr, createdAtStr string 329 337 330 330 - var existing IndexedRecord 331 331 - if err := json.Unmarshal(existingData, &existing); err != nil { 332 332 - // Can't parse, just delete the main record 333 333 - return records.Delete([]byte(uri)) 334 334 - } 338 338 + err := idx.db.QueryRow(` 339 339 + SELECT uri, did, collection, rkey, record, cid, indexed_at, created_at 340 340 + FROM records WHERE uri = ? 341 341 + `, uri).Scan(&rec.URI, &rec.DID, &rec.Collection, &rec.RKey, 342 342 + &recordStr, &rec.CID, &indexedAtStr, &createdAtStr) 343 343 + if err == sql.ErrNoRows { 344 344 + return nil, nil 345 345 + } 346 346 + if err != nil { 347 347 + return nil, err 348 348 + } 335 349 336 336 - // Delete from records 337 337 - if err := records.Delete([]byte(uri)); err != nil { 338 338 - return err 339 339 - } 340 340 - 341 341 - // Delete from by_time index 342 342 - byTime := tx.Bucket(BucketByTime) 343 343 - timeKey := makeTimeKey(existing.CreatedAt, uri) 344 344 - if err := byTime.Delete(timeKey); err != nil { 345 345 - return err 346 346 - } 350 350 + rec.Record = json.RawMessage(recordStr) 351 351 + rec.IndexedAt, _ = time.Parse(time.RFC3339Nano, indexedAtStr) 352 352 + rec.CreatedAt, _ = time.Parse(time.RFC3339Nano, createdAtStr) 347 353 348 348 - // Delete from by_did index 349 349 - byDID := tx.Bucket(BucketByDID) 350 350 - didKey := []byte(did + ":" + uri) 351 351 - if err := byDID.Delete(didKey); err != nil { 352 352 - return err 353 353 - } 354 354 - 355 355 - // Delete from by_collection index 356 356 - byCollection := tx.Bucket(BucketByCollection) 357 357 - collKey := []byte(collection + ":" + string(timeKey)) 358 358 - if err := byCollection.Delete(collKey); err != nil { 359 359 - return err 360 360 - } 361 361 - 362 362 - return nil 363 363 - }) 364 364 - } 365 365 - 366 366 - // GetRecord retrieves a single record by URI 367 367 - func (idx *FeedIndex) GetRecord(uri string) (*IndexedRecord, error) { 368 368 - var record *IndexedRecord 369 369 - err := idx.db.View(func(tx *bolt.Tx) error { 370 370 - b := tx.Bucket(BucketRecords) 371 371 - data := b.Get([]byte(uri)) 372 372 - if data == nil { 373 373 - return nil 374 374 - } 375 375 - record = &IndexedRecord{} 376 376 - return json.Unmarshal(data, record) 377 377 - }) 378 378 - return record, err 354 354 + return &rec, nil 379 355 } 380 356 381 357 // FeedItem represents an item in the feed (matches feed.FeedItem structure) ··· 404 380 405 381 // GetRecentFeed returns recent feed items from the index 406 382 func (idx *FeedIndex) GetRecentFeed(ctx context.Context, limit int) ([]*FeedItem, error) { 407 407 - var records []*IndexedRecord 408 408 - err := idx.db.View(func(tx *bolt.Tx) error { 409 409 - byTime := tx.Bucket(BucketByTime) 410 410 - recordsBucket := tx.Bucket(BucketRecords) 411 411 - 412 412 - c := byTime.Cursor() 413 413 - 414 414 - // Iterate in reverse (newest first) 415 415 - count := 0 416 416 - for k, _ := c.First(); k != nil && count < limit*2; k, _ = c.Next() { 417 417 - // Extract URI from key (format: timestamp:uri) 418 418 - uri := extractURIFromTimeKey(k) 419 419 - if uri == "" { 420 420 - continue 421 421 - } 422 422 - 423 423 - data := recordsBucket.Get([]byte(uri)) 424 424 - if data == nil { 425 425 - continue 426 426 - } 427 427 - 428 428 - var record IndexedRecord 429 429 - if err := json.Unmarshal(data, &record); err != nil { 430 430 - continue 431 431 - } 432 432 - 433 433 - records = append(records, &record) 434 434 - count++ 435 435 - } 436 436 - 437 437 - return nil 438 438 - }) 439 439 - if err != nil { 440 440 - return nil, err 441 441 - } 442 442 - 443 443 - // Build lookup maps for reference resolution 444 444 - recordsByURI := make(map[string]*IndexedRecord) 445 445 - for _, r := range records { 446 446 - recordsByURI[r.URI] = r 447 447 - } 448 448 - 449 449 - // Also load additional records we might need for references 450 450 - err = idx.db.View(func(tx *bolt.Tx) error { 451 451 - recordsBucket := tx.Bucket(BucketRecords) 452 452 - return recordsBucket.ForEach(func(k, v []byte) error { 453 453 - uri := string(k) 454 454 - if _, exists := recordsByURI[uri]; exists { 455 455 - return nil 456 456 - } 457 457 - var record IndexedRecord 458 458 - if err := json.Unmarshal(v, &record); err != nil { 459 459 - return nil 460 460 - } 461 461 - // Only load beans, roasters, grinders, brewers for reference resolution 462 462 - switch record.Collection { 463 463 - case atproto.NSIDBean, atproto.NSIDRoaster, atproto.NSIDGrinder, atproto.NSIDBrewer: 464 464 - recordsByURI[uri] = &record 465 465 - } 466 466 - return nil 467 467 - }) 468 468 - }) 469 469 - if err != nil { 470 470 - return nil, err 471 471 - } 472 472 - 473 473 - // Convert to FeedItems 474 474 - items := make([]*FeedItem, 0, len(records)) 475 475 - for _, record := range records { 476 476 - // Skip likes - they're indexed for like counts but not displayed as feed items 477 477 - if record.Collection == atproto.NSIDLike { 478 478 - continue 479 479 - } 480 480 - 481 481 - item, err := idx.recordToFeedItem(ctx, record, recordsByURI) 482 482 - if err != nil { 483 483 - log.Warn().Err(err).Str("uri", record.URI).Msg("failed to convert record to feed item") 484 484 - continue 485 485 - } 486 486 - if !FeedableRecordTypes[item.RecordType] { 487 487 - continue 488 488 - } 489 489 - items = append(items, item) 490 490 - } 491 491 - 492 492 - // Sort by timestamp descending 493 493 - sort.Slice(items, func(i, j int) bool { 494 494 - return items[i].Timestamp.After(items[j].Timestamp) 495 495 - }) 496 496 - 497 497 - // Apply limit 498 498 - if len(items) > limit { 499 499 - items = items[:limit] 500 500 - } 501 501 - 502 502 - return items, nil 383 383 + return idx.getFeedItems(ctx, "", limit, "") 503 384 } 504 385 505 386 // recordTypeToNSID maps a lexicons.RecordType to its NSID collection string ··· 511 392 lexicons.RecordTypeBrewer: atproto.NSIDBrewer, 512 393 } 513 394 395 395 + // feedableCollections is the set of collection NSIDs that appear in the feed 396 396 + var feedableCollections = func() []string { 397 397 + out := make([]string, 0, len(recordTypeToNSID)) 398 398 + for _, nsid := range recordTypeToNSID { 399 399 + out = append(out, nsid) 400 400 + } 401 401 + return out 402 402 + }() 403 403 + 514 404 // GetFeedWithQuery returns feed items matching the given query with cursor-based pagination 515 405 func (idx *FeedIndex) GetFeedWithQuery(ctx context.Context, q FeedQuery) (*FeedResult, error) { 516 406 if q.Limit <= 0 { ··· 520 410 q.Sort = FeedSortRecent 521 411 } 522 412 523 523 - // For type-filtered queries, use BucketByCollection for efficiency 524 524 - // For unfiltered queries, use BucketByTime 525 525 - var records []*IndexedRecord 526 526 - var lastTimeKey []byte 527 527 - 528 528 - // Decode cursor if provided 529 529 - var cursorBytes []byte 530 530 - if q.Cursor != "" { 531 531 - var err error 532 532 - cursorBytes, err = decodeCursor(q.Cursor) 533 533 - if err != nil { 534 534 - return nil, fmt.Errorf("invalid cursor: %w", err) 413 413 + var collectionFilter string 414 414 + if q.TypeFilter != "" { 415 415 + nsid, ok := recordTypeToNSID[q.TypeFilter] 416 416 + if !ok { 417 417 + return nil, fmt.Errorf("unknown record type: %s", q.TypeFilter) 535 418 } 419 419 + collectionFilter = nsid 536 420 } 537 421 538 538 - // Fetch more than needed to account for filtering 539 539 - fetchLimit := q.Limit + 10 540 540 - 541 541 - err := idx.db.View(func(tx *bolt.Tx) error { 542 542 - recordsBucket := tx.Bucket(BucketRecords) 422 422 + items, err := idx.getFeedItems(ctx, collectionFilter, q.Limit+1, q.Cursor) 423 423 + if err != nil { 424 424 + return nil, err 425 425 + } 543 426 544 544 - if q.TypeFilter != "" { 545 545 - // Use BucketByCollection for filtered queries 546 546 - nsid, ok := recordTypeToNSID[q.TypeFilter] 547 547 - if !ok { 548 548 - return fmt.Errorf("unknown record type: %s", q.TypeFilter) 427 427 + // Sort based on query 428 428 + if q.Sort == FeedSortPopular { 429 429 + sort.Slice(items, func(i, j int) bool { 430 430 + scoreI := items[i].LikeCount*3 + items[i].CommentCount*2 431 431 + scoreJ := items[j].LikeCount*3 + items[j].CommentCount*2 432 432 + if scoreI != scoreJ { 433 433 + return scoreI > scoreJ 549 434 } 435 435 + return items[i].Timestamp.After(items[j].Timestamp) 436 436 + }) 437 437 + } 550 438 551 551 - byCollection := tx.Bucket(BucketByCollection) 552 552 - c := byCollection.Cursor() 439 439 + result := &FeedResult{Items: items} 440 440 + if len(items) > q.Limit { 441 441 + result.Items = items[:q.Limit] 442 442 + last := result.Items[q.Limit-1] 443 443 + result.NextCursor = last.Timestamp.Format(time.RFC3339Nano) + "|" + last.SubjectURI 444 444 + } 553 445 554 554 - // Collection keys: {collection}:{inverted_timestamp}:{uri} 555 555 - prefix := []byte(nsid + ":") 446 446 + return result, nil 447 447 + } 556 448 557 557 - var k []byte 558 558 - if cursorBytes != nil { 559 559 - // Seek to cursor position (cursor is the full collection key) 560 560 - k, _ = c.Seek(cursorBytes) 561 561 - // Skip the cursor key itself (it was the last item of previous page) 562 562 - if k != nil && string(k) == string(cursorBytes) { 563 563 - k, _ = c.Next() 564 564 - } 565 565 - } else { 566 566 - k, _ = c.Seek(prefix) 567 567 - } 449 449 + // getFeedItems fetches records from SQLite, resolves references, and returns FeedItems. 450 450 + func (idx *FeedIndex) getFeedItems(ctx context.Context, collectionFilter string, limit int, cursor string) ([]*FeedItem, error) { 451 451 + // Build query for feedable records 452 452 + var args []any 453 453 + query := `SELECT uri, did, collection, rkey, record, cid, indexed_at, created_at FROM records WHERE ` 568 454 569 569 - count := 0 570 570 - for ; k != nil && count < fetchLimit; k, _ = c.Next() { 571 571 - if !bytes.HasPrefix(k, prefix) { 572 572 - break 573 573 - } 455 455 + if collectionFilter != "" { 456 456 + query += `collection = ? ` 457 457 + args = append(args, collectionFilter) 458 458 + } else { 459 459 + // Only feedable collections 460 460 + placeholders := make([]string, len(feedableCollections)) 461 461 + for i, c := range feedableCollections { 462 462 + placeholders[i] = "?" 463 463 + args = append(args, c) 464 464 + } 465 465 + query += `collection IN (` + strings.Join(placeholders, ",") + `) ` 466 466 + } 574 467 575 575 - // Extract URI from collection key: {collection}:{timestamp_bytes}:{uri} 576 576 - uri := extractURIFromCollectionKey(k, nsid) 577 577 - if uri == "" { 578 578 - continue 579 579 - } 468 468 + // Cursor-based pagination: cursor format is "created_at|uri" 469 469 + if cursor != "" { 470 470 + parts := strings.SplitN(cursor, "|", 2) 471 471 + if len(parts) == 2 { 472 472 + query += `AND (created_at < ? OR (created_at = ? AND uri < ?)) ` 473 473 + args = append(args, parts[0], parts[0], parts[1]) 474 474 + } 475 475 + } 580 476 581 581 - data := recordsBucket.Get([]byte(uri)) 582 582 - if data == nil { 583 583 - continue 584 584 - } 477 477 + query += `ORDER BY created_at DESC LIMIT ?` 478 478 + args = append(args, limit) 585 479 586 586 - var record IndexedRecord 587 587 - if err := json.Unmarshal(data, &record); err != nil { 588 588 - continue 589 589 - } 590 590 - records = append(records, &record) 591 591 - lastTimeKey = make([]byte, len(k)) 592 592 - copy(lastTimeKey, k) 593 593 - count++ 594 594 - } 595 595 - } else { 596 596 - // Use BucketByTime for unfiltered queries 597 597 - byTime := tx.Bucket(BucketByTime) 598 598 - c := byTime.Cursor() 480 480 + rows, err := idx.db.QueryContext(ctx, query, args...) 481 481 + if err != nil { 482 482 + return nil, err 483 483 + } 484 484 + defer rows.Close() 599 485 600 600 - var k []byte 601 601 - if cursorBytes != nil { 602 602 - k, _ = c.Seek(cursorBytes) 603 603 - if k != nil && string(k) == string(cursorBytes) { 604 604 - k, _ = c.Next() 605 605 - } 606 606 - } else { 607 607 - k, _ = c.First() 608 608 - } 486 486 + var records []*IndexedRecord 487 487 + refURIs := make(map[string]bool) // URIs we need to resolve 609 488 610 610 - count := 0 611 611 - for ; k != nil && count < fetchLimit; k, _ = c.Next() { 612 612 - uri := extractURIFromTimeKey(k) 613 613 - if uri == "" { 614 614 - continue 615 615 - } 489 489 + for rows.Next() { 490 490 + var rec IndexedRecord 491 491 + var recordStr, indexedAtStr, createdAtStr string 492 492 + if err := rows.Scan(&rec.URI, &rec.DID, &rec.Collection, &rec.RKey, 493 493 + &recordStr, &rec.CID, &indexedAtStr, &createdAtStr); err != nil { 494 494 + continue 495 495 + } 496 496 + rec.Record = json.RawMessage(recordStr) 497 497 + rec.IndexedAt, _ = time.Parse(time.RFC3339Nano, indexedAtStr) 498 498 + rec.CreatedAt, _ = time.Parse(time.RFC3339Nano, createdAtStr) 499 499 + records = append(records, &rec) 616 500 617 617 - data := recordsBucket.Get([]byte(uri)) 618 618 - if data == nil { 619 619 - continue 620 620 - } 621 621 - 622 622 - var record IndexedRecord 623 623 - if err := json.Unmarshal(data, &record); err != nil { 624 624 - continue 625 625 - } 626 626 - // Skip non-feedable records (likes, comments) so they don't 627 627 - // consume slots in the fetch limit, which would cause pagination 628 628 - // to break when many non-feedable records are intermixed. 629 629 - if record.Collection == atproto.NSIDLike || record.Collection == atproto.NSIDComment { 630 630 - continue 501 501 + // Collect reference URIs from the record data 502 502 + var recordData map[string]any 503 503 + if err := json.Unmarshal(rec.Record, &recordData); err == nil { 504 504 + for _, key := range []string{"beanRef", "roasterRef", "grinderRef", "brewerRef"} { 505 505 + if ref, ok := recordData[key].(string); ok && ref != "" { 506 506 + refURIs[ref] = true 631 507 } 632 632 - records = append(records, &record) 633 633 - lastTimeKey = make([]byte, len(k)) 634 634 - copy(lastTimeKey, k) 635 635 - count++ 636 508 } 637 509 } 638 638 - 639 639 - return nil 640 640 - }) 641 641 - if err != nil { 510 510 + } 511 511 + if err := rows.Err(); err != nil { 642 512 return nil, err 643 513 } 644 514 645 645 - // Build lookup maps for reference resolution 646 646 - recordsByURI := make(map[string]*IndexedRecord) 515 515 + // Build lookup map starting with the fetched records 516 516 + recordsByURI := make(map[string]*IndexedRecord, len(records)) 647 517 for _, r := range records { 648 518 recordsByURI[r.URI] = r 649 519 } 650 520 651 651 - // Load additional records for reference resolution 652 652 - err = idx.db.View(func(tx *bolt.Tx) error { 653 653 - recordsBucket := tx.Bucket(BucketRecords) 654 654 - return recordsBucket.ForEach(func(k, v []byte) error { 655 655 - uri := string(k) 656 656 - if _, exists := recordsByURI[uri]; exists { 657 657 - return nil 658 658 - } 659 659 - var record IndexedRecord 660 660 - if err := json.Unmarshal(v, &record); err != nil { 661 661 - return nil 662 662 - } 663 663 - switch record.Collection { 664 664 - case atproto.NSIDBean, atproto.NSIDRoaster, atproto.NSIDGrinder, atproto.NSIDBrewer: 665 665 - recordsByURI[uri] = &record 521 521 + // Fetch referenced records that we don't already have 522 522 + var missingURIs []string 523 523 + for uri := range refURIs { 524 524 + if _, ok := recordsByURI[uri]; !ok { 525 525 + missingURIs = append(missingURIs, uri) 526 526 + } 527 527 + } 528 528 + 529 529 + if len(missingURIs) > 0 { 530 530 + placeholders := make([]string, len(missingURIs)) 531 531 + refArgs := make([]any, len(missingURIs)) 532 532 + for i, uri := range missingURIs { 533 533 + placeholders[i] = "?" 534 534 + refArgs[i] = uri 535 535 + } 536 536 + refQuery := `SELECT uri, did, collection, rkey, record, cid, indexed_at, created_at FROM records WHERE uri IN (` + strings.Join(placeholders, ",") + `)` 537 537 + refRows, err := idx.db.QueryContext(ctx, refQuery, refArgs...) 538 538 + if err == nil { 539 539 + defer refRows.Close() 540 540 + for refRows.Next() { 541 541 + var rec IndexedRecord 542 542 + var recordStr, indexedAtStr, createdAtStr string 543 543 + if err := refRows.Scan(&rec.URI, &rec.DID, &rec.Collection, &rec.RKey, 544 544 + &recordStr, &rec.CID, &indexedAtStr, &createdAtStr); err != nil { 545 545 + continue 546 546 + } 547 547 + rec.Record = json.RawMessage(recordStr) 548 548 + rec.IndexedAt, _ = time.Parse(time.RFC3339Nano, indexedAtStr) 549 549 + rec.CreatedAt, _ = time.Parse(time.RFC3339Nano, createdAtStr) 550 550 + recordsByURI[rec.URI] = &rec 551 551 + 552 552 + // If this is a bean, check if it references a roaster we also need 553 553 + if rec.Collection == atproto.NSIDBean { 554 554 + var beanData map[string]any 555 555 + if err := json.Unmarshal(rec.Record, &beanData); err == nil { 556 556 + if roasterRef, ok := beanData["roasterRef"].(string); ok && roasterRef != "" { 557 557 + if _, ok := recordsByURI[roasterRef]; !ok { 558 558 + // Fetch this roaster too 559 559 + var rRec IndexedRecord 560 560 + var rStr, rIdxAt, rCreAt string 561 561 + err := idx.db.QueryRowContext(ctx, 562 562 + `SELECT uri, did, collection, rkey, record, cid, indexed_at, created_at FROM records WHERE uri = ?`, 563 563 + roasterRef).Scan(&rRec.URI, &rRec.DID, &rRec.Collection, &rRec.RKey, 564 564 + &rStr, &rRec.CID, &rIdxAt, &rCreAt) 565 565 + if err == nil { 566 566 + rRec.Record = json.RawMessage(rStr) 567 567 + rRec.IndexedAt, _ = time.Parse(time.RFC3339Nano, rIdxAt) 568 568 + rRec.CreatedAt, _ = time.Parse(time.RFC3339Nano, rCreAt) 569 569 + recordsByURI[rRec.URI] = &rRec 570 570 + } 571 571 + } 572 572 + } 573 573 + } 574 574 + } 666 575 } 667 667 - return nil 668 668 - }) 669 669 - }) 670 670 - if err != nil { 671 671 - return nil, err 576 576 + } 672 577 } 673 578 674 579 // Convert to FeedItems 675 580 items := make([]*FeedItem, 0, len(records)) 676 581 for _, record := range records { 677 677 - if record.Collection == atproto.NSIDLike || record.Collection == atproto.NSIDComment { 678 678 - continue 679 679 - } 680 680 - 681 582 item, err := idx.recordToFeedItem(ctx, record, recordsByURI) 682 583 if err != nil { 683 584 log.Warn().Err(err).Str("uri", record.URI).Msg("failed to convert record to feed item") ··· 689 590 items = append(items, item) 690 591 } 691 592 692 692 - // Sort based on query 693 693 - switch q.Sort { 694 694 - case FeedSortPopular: 695 695 - sort.Slice(items, func(i, j int) bool { 696 696 - scoreI := items[i].LikeCount*3 + items[i].CommentCount*2 697 697 - scoreJ := items[j].LikeCount*3 + items[j].CommentCount*2 698 698 - if scoreI != scoreJ { 699 699 - return scoreI > scoreJ 700 700 - } 701 701 - return items[i].Timestamp.After(items[j].Timestamp) 702 702 - }) 703 703 - default: // FeedSortRecent 704 704 - sort.Slice(items, func(i, j int) bool { 705 705 - return items[i].Timestamp.After(items[j].Timestamp) 706 706 - }) 707 707 - } 708 708 - 709 709 - // Build result with cursor 710 710 - result := &FeedResult{Items: items} 711 711 - 712 712 - if len(items) > q.Limit { 713 713 - result.Items = items[:q.Limit] 714 714 - // Cursor is the last time key we read from the DB 715 715 - if lastTimeKey != nil { 716 716 - result.NextCursor = encodeCursor(lastTimeKey) 717 717 - } 718 718 - } 719 719 - 720 720 - return result, nil 721 721 - } 722 722 - 723 723 - // extractURIFromCollectionKey extracts the URI from a collection key 724 724 - // Format: {collection}:{inverted_timestamp_8bytes}:{uri} 725 725 - func extractURIFromCollectionKey(key []byte, collection string) string { 726 726 - // prefix is collection + ":" 727 727 - prefixLen := len(collection) + 1 728 728 - // Then 8 bytes of timestamp + ":" 729 729 - minLen := prefixLen + 8 + 1 + 1 // prefix + timestamp + ":" + at least 1 char 730 730 - if len(key) < minLen { 731 731 - return "" 732 732 - } 733 733 - return string(key[prefixLen+9:]) 734 734 - } 735 735 - 736 736 - func encodeCursor(key []byte) string { 737 737 - return hex.EncodeToString(key) 738 738 - } 739 739 - 740 740 - func decodeCursor(s string) ([]byte, error) { 741 741 - return hex.DecodeString(s) 593 593 + return items, nil 742 594 } 743 595 744 596 // recordToFeedItem converts an IndexedRecord to a FeedItem ··· 757 609 profile, err := idx.GetProfile(ctx, record.DID) 758 610 if err != nil { 759 611 log.Warn().Err(err).Str("did", record.DID).Msg("failed to get profile") 760 760 - // Use a placeholder profile 761 612 profile = &atproto.Profile{ 762 613 DID: record.DID, 763 763 - Handle: record.DID, // Use DID as handle if we can't resolve 614 614 + Handle: record.DID, 764 615 } 765 616 } 766 617 item.Author = profile ··· 869 720 item.Brewer = brewer 870 721 871 722 case atproto.NSIDLike: 872 872 - // This should never be reached - likes are filtered before calling recordToFeedItem 873 723 return nil, fmt.Errorf("unexpected: likes should be filtered before conversion") 874 724 875 725 default: ··· 896 746 idx.profileCacheMu.RUnlock() 897 747 898 748 // Check persistent cache 899 899 - var cached *CachedProfile 900 900 - err := idx.db.View(func(tx *bolt.Tx) error { 901 901 - b := tx.Bucket(BucketProfiles) 902 902 - data := b.Get([]byte(did)) 903 903 - if data == nil { 904 904 - return nil 749 749 + var dataStr, expiresAtStr string 750 750 + err := idx.db.QueryRow(`SELECT data, expires_at FROM profiles WHERE did = ?`, did).Scan(&dataStr, &expiresAtStr) 751 751 + if err == nil { 752 752 + expiresAt, _ := time.Parse(time.RFC3339Nano, expiresAtStr) 753 753 + if time.Now().Before(expiresAt) { 754 754 + cached := &CachedProfile{} 755 755 + if err := json.Unmarshal([]byte(dataStr), cached); err == nil { 756 756 + idx.profileCacheMu.Lock() 757 757 + idx.profileCache[did] = cached 758 758 + idx.profileCacheMu.Unlock() 759 759 + return cached.Profile, nil 760 760 + } 905 761 } 906 906 - cached = &CachedProfile{} 907 907 - return json.Unmarshal(data, cached) 908 908 - }) 909 909 - if err == nil && cached != nil && time.Now().Before(cached.ExpiresAt) { 910 910 - // Update in-memory cache 911 911 - idx.profileCacheMu.Lock() 912 912 - idx.profileCache[did] = cached 913 913 - idx.profileCacheMu.Unlock() 914 914 - return cached.Profile, nil 915 762 } 916 763 917 764 // Fetch from API ··· 922 769 923 770 // Cache the result 924 771 now := time.Now() 925 925 - cached = &CachedProfile{ 772 772 + cached := &CachedProfile{ 926 773 Profile: profile, 927 774 CachedAt: now, 928 775 ExpiresAt: now.Add(idx.profileTTL), ··· 935 782 936 783 // Persist to database 937 784 data, _ := json.Marshal(cached) 938 938 - _ = idx.db.Update(func(tx *bolt.Tx) error { 939 939 - b := tx.Bucket(BucketProfiles) 940 940 - return b.Put([]byte(did), data) 941 941 - }) 785 785 + _, _ = idx.db.Exec(`INSERT OR REPLACE INTO profiles (did, data, expires_at) VALUES (?, ?, ?)`, 786 786 + did, string(data), cached.ExpiresAt.Format(time.RFC3339Nano)) 942 787 943 788 return profile, nil 944 789 } 945 790 946 791 // GetKnownDIDs returns all DIDs that have created Arabica records 947 792 func (idx *FeedIndex) GetKnownDIDs() ([]string, error) { 793 793 + rows, err := idx.db.Query(`SELECT did FROM known_dids`) 794 794 + if err != nil { 795 795 + return nil, err 796 796 + } 797 797 + defer rows.Close() 798 798 + 948 799 var dids []string 949 949 - err := idx.db.View(func(tx *bolt.Tx) error { 950 950 - b := tx.Bucket(BucketKnownDIDs) 951 951 - return b.ForEach(func(k, v []byte) error { 952 952 - dids = append(dids, string(k)) 953 953 - return nil 954 954 - }) 955 955 - }) 956 956 - return dids, err 800 800 + for rows.Next() { 801 801 + var did string 802 802 + if err := rows.Scan(&did); err != nil { 803 803 + continue 804 804 + } 805 805 + dids = append(dids, did) 806 806 + } 807 807 + return dids, rows.Err() 808 808 + } 809 809 + 810 810 + // ListRecordsByCollection returns all indexed records for a given collection. 811 811 + func (idx *FeedIndex) ListRecordsByCollection(collection string) ([]IndexedRecord, error) { 812 812 + rows, err := idx.db.Query(` 813 813 + SELECT uri, did, collection, rkey, record, cid, indexed_at, created_at 814 814 + FROM records WHERE collection = ? ORDER BY created_at DESC 815 815 + `, collection) 816 816 + if err != nil { 817 817 + return nil, err 818 818 + } 819 819 + defer rows.Close() 820 820 + 821 821 + var records []IndexedRecord 822 822 + for rows.Next() { 823 823 + var rec IndexedRecord 824 824 + var recordStr, indexedAtStr, createdAtStr string 825 825 + if err := rows.Scan(&rec.URI, &rec.DID, &rec.Collection, &rec.RKey, 826 826 + &recordStr, &rec.CID, &indexedAtStr, &createdAtStr); err != nil { 827 827 + continue 828 828 + } 829 829 + rec.Record = json.RawMessage(recordStr) 830 830 + rec.IndexedAt, _ = time.Parse(time.RFC3339Nano, indexedAtStr) 831 831 + rec.CreatedAt, _ = time.Parse(time.RFC3339Nano, createdAtStr) 832 832 + records = append(records, rec) 833 833 + } 834 834 + return records, rows.Err() 957 835 } 958 836 959 837 // RecordCount returns the total number of indexed records 960 838 func (idx *FeedIndex) RecordCount() int { 961 839 var count int 962 962 - _ = idx.db.View(func(tx *bolt.Tx) error { 963 963 - b := tx.Bucket(BucketRecords) 964 964 - count = b.Stats().KeyN 965 965 - return nil 966 966 - }) 840 840 + _ = idx.db.QueryRow(`SELECT COUNT(*) FROM records`).Scan(&count) 967 841 return count 968 842 } 969 843 970 844 // KnownDIDCount returns the number of unique DIDs in the index 971 845 func (idx *FeedIndex) KnownDIDCount() int { 972 846 var count int 973 973 - _ = idx.db.View(func(tx *bolt.Tx) error { 974 974 - b := tx.Bucket(BucketKnownDIDs) 975 975 - count = b.Stats().KeyN 976 976 - return nil 977 977 - }) 847 847 + _ = idx.db.QueryRow(`SELECT COUNT(*) FROM known_dids`).Scan(&count) 978 848 return count 979 849 } 980 850 981 851 // TotalLikeCount returns the total number of likes indexed 982 852 func (idx *FeedIndex) TotalLikeCount() int { 983 853 var count int 984 984 - _ = idx.db.View(func(tx *bolt.Tx) error { 985 985 - b := tx.Bucket(BucketLikes) 986 986 - count = b.Stats().KeyN 987 987 - return nil 988 988 - }) 854 854 + _ = idx.db.QueryRow(`SELECT COUNT(*) FROM likes`).Scan(&count) 989 855 return count 990 856 } 991 857 992 858 // TotalCommentCount returns the total number of comments indexed 993 859 func (idx *FeedIndex) TotalCommentCount() int { 994 860 var count int 995 995 - _ = idx.db.View(func(tx *bolt.Tx) error { 996 996 - b := tx.Bucket(BucketCommentsByActor) 997 997 - count = b.Stats().KeyN 998 998 - return nil 999 999 - }) 861 861 + _ = idx.db.QueryRow(`SELECT COUNT(*) FROM comments`).Scan(&count) 1000 862 return count 1001 863 } 1002 864 1003 865 // RecordCountByCollection returns a breakdown of record counts by collection type 1004 866 func (idx *FeedIndex) RecordCountByCollection() map[string]int { 1005 867 counts := make(map[string]int) 1006 1006 - _ = idx.db.View(func(tx *bolt.Tx) error { 1007 1007 - records := tx.Bucket(BucketRecords) 1008 1008 - return records.ForEach(func(k, v []byte) error { 1009 1009 - var record IndexedRecord 1010 1010 - if err := json.Unmarshal(v, &record); err != nil { 1011 1011 - return nil 1012 1012 - } 1013 1013 - counts[record.Collection]++ 1014 1014 - return nil 1015 1015 - }) 1016 1016 - }) 868 868 + rows, err := idx.db.Query(`SELECT collection, COUNT(*) FROM records GROUP BY collection`) 869 869 + if err != nil { 870 870 + return counts 871 871 + } 872 872 + defer rows.Close() 873 873 + for rows.Next() { 874 874 + var collection string 875 875 + var count int 876 876 + if err := rows.Scan(&collection, &count); err == nil { 877 877 + counts[collection] = count 878 878 + } 879 879 + } 1017 880 return counts 1018 1018 - } 1019 1019 - 1020 1020 - // Helper functions 1021 1021 - 1022 1022 - func makeTimeKey(t time.Time, uri string) []byte { 1023 1023 - // Format: inverted timestamp (for reverse chronological order) + ":" + uri 1024 1024 - // Use nanoseconds for uniqueness 1025 1025 - inverted := ^uint64(t.UnixNano()) 1026 1026 - buf := make([]byte, 8) 1027 1027 - binary.BigEndian.PutUint64(buf, inverted) 1028 1028 - return append(buf, []byte(":"+uri)...) 1029 1029 - } 1030 1030 - 1031 1031 - func extractURIFromTimeKey(key []byte) string { 1032 1032 - if len(key) < 10 { // 8 bytes timestamp + ":" + at least 1 char 1033 1033 - return "" 1034 1034 - } 1035 1035 - // Skip 8 bytes timestamp + 1 byte ":" 1036 1036 - return string(key[9:]) 1037 881 } 1038 882 1039 883 func formatTimeAgo(t time.Time) string { ··· 1077 921 1078 922 // IsBackfilled checks if a DID has already been backfilled 1079 923 func (idx *FeedIndex) IsBackfilled(did string) bool { 1080 1080 - var exists bool 1081 1081 - _ = idx.db.View(func(tx *bolt.Tx) error { 1082 1082 - b := tx.Bucket(BucketBackfilled) 1083 1083 - exists = b.Get([]byte(did)) != nil 1084 1084 - return nil 1085 1085 - }) 1086 1086 - return exists 924 924 + var exists int 925 925 + err := idx.db.QueryRow(`SELECT 1 FROM backfilled WHERE did = ?`, did).Scan(&exists) 926 926 + return err == nil 1087 927 } 1088 928 1089 929 // MarkBackfilled marks a DID as backfilled with current timestamp 1090 930 func (idx *FeedIndex) MarkBackfilled(did string) error { 1091 1091 - return idx.db.Update(func(tx *bolt.Tx) error { 1092 1092 - b := tx.Bucket(BucketBackfilled) 1093 1093 - timestamp := []byte(time.Now().Format(time.RFC3339)) 1094 1094 - return b.Put([]byte(did), timestamp) 1095 1095 - }) 931 931 + _, err := idx.db.Exec(`INSERT OR IGNORE INTO backfilled (did, backfilled_at) VALUES (?, ?)`, 932 932 + did, time.Now().Format(time.RFC3339)) 933 933 + return err 1096 934 } 1097 935 1098 936 // BackfillUser fetches all existing records for a DID and adds them to the index 1099 1099 - // Returns early if the DID has already been backfilled 1100 937 func (idx *FeedIndex) BackfillUser(ctx context.Context, did string) error { 1101 1101 - // Check if already backfilled 1102 938 if idx.IsBackfilled(did) { 1103 939 log.Debug().Str("did", did).Msg("DID already backfilled, skipping") 1104 940 return nil ··· 1115 951 } 1116 952 1117 953 for _, record := range records.Records { 1118 1118 - // Extract rkey from URI 1119 954 parts := strings.Split(record.URI, "/") 1120 955 if len(parts) < 3 { 1121 956 continue ··· 1133 968 } 1134 969 recordCount++ 1135 970 1136 1136 - // Index likes and comments into their specialized buckets 1137 971 switch collection { 1138 972 case atproto.NSIDLike: 1139 973 if subject, ok := record.Value["subject"].(map[string]interface{}); ok { ··· 1170 1004 } 1171 1005 } 1172 1006 1173 1173 - // Mark as backfilled 1174 1007 if err := idx.MarkBackfilled(did); err != nil { 1175 1008 log.Warn().Err(err).Str("did", did).Msg("failed to mark DID as backfilled") 1176 1009 } ··· 1183 1016 1184 1017 // UpsertLike adds or updates a like in the index 1185 1018 func (idx *FeedIndex) UpsertLike(actorDID, rkey, subjectURI string) error { 1186 1186 - return idx.db.Update(func(tx *bolt.Tx) error { 1187 1187 - likes := tx.Bucket(BucketLikes) 1188 1188 - likeCounts := tx.Bucket(BucketLikeCounts) 1189 1189 - likesByActor := tx.Bucket(BucketLikesByActor) 1190 1190 - 1191 1191 - // Key format: {subject_uri}:{actor_did} 1192 1192 - likeKey := []byte(subjectURI + ":" + actorDID) 1193 1193 - 1194 1194 - // Check if this like already exists 1195 1195 - existingRKey := likes.Get(likeKey) 1196 1196 - if existingRKey != nil { 1197 1197 - // Already exists, nothing to do 1198 1198 - return nil 1199 1199 - } 1200 1200 - 1201 1201 - // Store the like mapping 1202 1202 - if err := likes.Put(likeKey, []byte(rkey)); err != nil { 1203 1203 - return err 1204 1204 - } 1205 1205 - 1206 1206 - // Store by actor for reverse lookup 1207 1207 - actorKey := []byte(actorDID + ":" + subjectURI) 1208 1208 - if err := likesByActor.Put(actorKey, []byte(rkey)); err != nil { 1209 1209 - return err 1210 1210 - } 1211 1211 - 1212 1212 - // Increment the like count 1213 1213 - countKey := []byte(subjectURI) 1214 1214 - currentCount := uint64(0) 1215 1215 - if countData := likeCounts.Get(countKey); len(countData) == 8 { 1216 1216 - currentCount = binary.BigEndian.Uint64(countData) 1217 1217 - } 1218 1218 - currentCount++ 1219 1219 - countBuf := make([]byte, 8) 1220 1220 - binary.BigEndian.PutUint64(countBuf, currentCount) 1221 1221 - return likeCounts.Put(countKey, countBuf) 1222 1222 - }) 1019 1019 + _, err := idx.db.Exec(`INSERT OR IGNORE INTO likes (subject_uri, actor_did, rkey) VALUES (?, ?, ?)`, 1020 1020 + subjectURI, actorDID, rkey) 1021 1021 + return err 1223 1022 } 1224 1023 1225 1024 // DeleteLike removes a like from the index 1226 1025 func (idx *FeedIndex) DeleteLike(actorDID, subjectURI string) error { 1227 1227 - return idx.db.Update(func(tx *bolt.Tx) error { 1228 1228 - likes := tx.Bucket(BucketLikes) 1229 1229 - likeCounts := tx.Bucket(BucketLikeCounts) 1230 1230 - likesByActor := tx.Bucket(BucketLikesByActor) 1231 1231 - 1232 1232 - // Key format: {subject_uri}:{actor_did} 1233 1233 - likeKey := []byte(subjectURI + ":" + actorDID) 1234 1234 - 1235 1235 - // Check if like exists 1236 1236 - if likes.Get(likeKey) == nil { 1237 1237 - // Doesn't exist, nothing to do 1238 1238 - return nil 1239 1239 - } 1240 1240 - 1241 1241 - // Delete the like mapping 1242 1242 - if err := likes.Delete(likeKey); err != nil { 1243 1243 - return err 1244 1244 - } 1245 1245 - 1246 1246 - // Delete by actor lookup 1247 1247 - actorKey := []byte(actorDID + ":" + subjectURI) 1248 1248 - if err := likesByActor.Delete(actorKey); err != nil { 1249 1249 - return err 1250 1250 - } 1251 1251 - 1252 1252 - // Decrement the like count 1253 1253 - countKey := []byte(subjectURI) 1254 1254 - currentCount := uint64(0) 1255 1255 - if countData := likeCounts.Get(countKey); len(countData) == 8 { 1256 1256 - currentCount = binary.BigEndian.Uint64(countData) 1257 1257 - } 1258 1258 - if currentCount > 0 { 1259 1259 - currentCount-- 1260 1260 - } 1261 1261 - if currentCount == 0 { 1262 1262 - return likeCounts.Delete(countKey) 1263 1263 - } 1264 1264 - countBuf := make([]byte, 8) 1265 1265 - binary.BigEndian.PutUint64(countBuf, currentCount) 1266 1266 - return likeCounts.Put(countKey, countBuf) 1267 1267 - }) 1026 1026 + _, err := idx.db.Exec(`DELETE FROM likes WHERE subject_uri = ? AND actor_did = ?`, 1027 1027 + subjectURI, actorDID) 1028 1028 + return err 1268 1029 } 1269 1030 1270 1031 // GetLikeCount returns the number of likes for a record 1271 1032 func (idx *FeedIndex) GetLikeCount(subjectURI string) int { 1272 1272 - var count uint64 1273 1273 - _ = idx.db.View(func(tx *bolt.Tx) error { 1274 1274 - likeCounts := tx.Bucket(BucketLikeCounts) 1275 1275 - countData := likeCounts.Get([]byte(subjectURI)) 1276 1276 - if len(countData) == 8 { 1277 1277 - count = binary.BigEndian.Uint64(countData) 1278 1278 - } 1279 1279 - return nil 1280 1280 - }) 1281 1281 - return int(count) 1033 1033 + var count int 1034 1034 + _ = idx.db.QueryRow(`SELECT COUNT(*) FROM likes WHERE subject_uri = ?`, subjectURI).Scan(&count) 1035 1035 + return count 1282 1036 } 1283 1037 1284 1038 // HasUserLiked checks if a user has liked a specific record 1285 1039 func (idx *FeedIndex) HasUserLiked(actorDID, subjectURI string) bool { 1286 1286 - var exists bool 1287 1287 - _ = idx.db.View(func(tx *bolt.Tx) error { 1288 1288 - likesByActor := tx.Bucket(BucketLikesByActor) 1289 1289 - actorKey := []byte(actorDID + ":" + subjectURI) 1290 1290 - exists = likesByActor.Get(actorKey) != nil 1291 1291 - return nil 1292 1292 - }) 1293 1293 - return exists 1040 1040 + var exists int 1041 1041 + err := idx.db.QueryRow(`SELECT 1 FROM likes WHERE actor_did = ? AND subject_uri = ? LIMIT 1`, 1042 1042 + actorDID, subjectURI).Scan(&exists) 1043 1043 + return err == nil 1294 1044 } 1295 1045 1296 1046 // GetUserLikeRKey returns the rkey of a user's like for a specific record, or empty string if not found 1297 1047 func (idx *FeedIndex) GetUserLikeRKey(actorDID, subjectURI string) string { 1298 1048 var rkey string 1299 1299 - _ = idx.db.View(func(tx *bolt.Tx) error { 1300 1300 - likesByActor := tx.Bucket(BucketLikesByActor) 1301 1301 - actorKey := []byte(actorDID + ":" + subjectURI) 1302 1302 - if data := likesByActor.Get(actorKey); data != nil { 1303 1303 - rkey = string(data) 1304 1304 - } 1305 1305 - return nil 1306 1306 - }) 1049 1049 + err := idx.db.QueryRow(`SELECT rkey FROM likes WHERE actor_did = ? AND subject_uri = ?`, 1050 1050 + actorDID, subjectURI).Scan(&rkey) 1051 1051 + if err != nil { 1052 1052 + return "" 1053 1053 + } 1307 1054 return rkey 1308 1055 } 1309 1056 ··· 1332 1079 1333 1080 // UpsertComment adds or updates a comment in the index 1334 1081 func (idx *FeedIndex) UpsertComment(actorDID, rkey, subjectURI, parentURI, cid, text string, createdAt time.Time) error { 1335 1335 - return idx.db.Update(func(tx *bolt.Tx) error { 1336 1336 - comments := tx.Bucket(BucketComments) 1337 1337 - commentCounts := tx.Bucket(BucketCommentCounts) 1338 1338 - commentsByActor := tx.Bucket(BucketCommentsByActor) 1339 1339 - commentChildren := tx.Bucket(BucketCommentChildren) 1340 1340 - 1341 1341 - // Key format: {subject_uri}:{timestamp}:{actor_did}:{rkey} 1342 1342 - // Using timestamp for chronological ordering 1343 1343 - commentKey := []byte(subjectURI + ":" + createdAt.Format(time.RFC3339Nano) + ":" + actorDID + ":" + rkey) 1344 1344 - 1345 1345 - // Check if this comment already exists (by actor key) 1346 1346 - actorKey := []byte(actorDID + ":" + rkey) 1347 1347 - existingSubject := commentsByActor.Get(actorKey) 1348 1348 - isNew := existingSubject == nil 1349 1349 - 1350 1350 - // If the comment already exists, delete the old entry from BucketComments 1351 1351 - // to prevent duplicates (the key includes timestamp which may differ between calls) 1352 1352 - if !isNew { 1353 1353 - oldPrefix := []byte(string(existingSubject) + ":") 1354 1354 - suffix := ":" + actorDID + ":" + rkey 1355 1355 - cur := comments.Cursor() 1356 1356 - for k, _ := cur.Seek(oldPrefix); k != nil && strings.HasPrefix(string(k), string(oldPrefix)); k, _ = cur.Next() { 1357 1357 - if strings.HasSuffix(string(k), suffix) { 1358 1358 - _ = comments.Delete(k) 1359 1359 - break 1360 1360 - } 1361 1361 - } 1362 1362 - } 1363 1363 - 1364 1364 - // Extract parent rkey from parent URI if present 1365 1365 - var parentRKey string 1366 1366 - if parentURI != "" { 1367 1367 - parts := strings.Split(parentURI, "/") 1368 1368 - if len(parts) > 0 { 1369 1369 - parentRKey = parts[len(parts)-1] 1370 1370 - } 1371 1371 - } 1372 1372 - 1373 1373 - // Store comment data as JSON 1374 1374 - commentData := IndexedComment{ 1375 1375 - RKey: rkey, 1376 1376 - SubjectURI: subjectURI, 1377 1377 - Text: text, 1378 1378 - ActorDID: actorDID, 1379 1379 - CreatedAt: createdAt, 1380 1380 - ParentURI: parentURI, 1381 1381 - ParentRKey: parentRKey, 1382 1382 - CID: cid, 1383 1383 - } 1384 1384 - commentJSON, err := json.Marshal(commentData) 1385 1385 - if err != nil { 1386 1386 - return fmt.Errorf("failed to marshal comment: %w", err) 1387 1387 - } 1388 1388 - 1389 1389 - // Store comment 1390 1390 - if err := comments.Put(commentKey, commentJSON); err != nil { 1391 1391 - return fmt.Errorf("failed to store comment: %w", err) 1392 1392 - } 1393 1393 - 1394 1394 - // Store actor lookup 1395 1395 - if err := commentsByActor.Put(actorKey, []byte(subjectURI)); err != nil { 1396 1396 - return fmt.Errorf("failed to store comment by actor: %w", err) 1397 1397 - } 1398 1398 - 1399 1399 - // Store parent-child relationship if this is a reply 1400 1400 - if parentURI != "" { 1401 1401 - childKey := []byte(parentURI + ":" + rkey) 1402 1402 - if err := commentChildren.Put(childKey, []byte(actorDID)); err != nil { 1403 1403 - return fmt.Errorf("failed to store comment child: %w", err) 1404 1404 - } 1405 1405 - } 1406 1406 - 1407 1407 - // Increment count only if this is a new comment 1408 1408 - if isNew { 1409 1409 - countKey := []byte(subjectURI) 1410 1410 - var count uint64 1411 1411 - if countData := commentCounts.Get(countKey); len(countData) == 8 { 1412 1412 - count = binary.BigEndian.Uint64(countData) 1413 1413 - } 1414 1414 - count++ 1415 1415 - countBytes := make([]byte, 8) 1416 1416 - binary.BigEndian.PutUint64(countBytes, count) 1417 1417 - if err := commentCounts.Put(countKey, countBytes); err != nil { 1418 1418 - return fmt.Errorf("failed to update comment count: %w", err) 1419 1419 - } 1082 1082 + // Extract parent rkey from parent URI if present 1083 1083 + var parentRKey string 1084 1084 + if parentURI != "" { 1085 1085 + parts := strings.Split(parentURI, "/") 1086 1086 + if len(parts) > 0 { 1087 1087 + parentRKey = parts[len(parts)-1] 1420 1088 } 1089 1089 + } 1421 1090 1422 1422 - return nil 1423 1423 - }) 1091 1091 + _, err := idx.db.Exec(` 1092 1092 + INSERT INTO comments (actor_did, rkey, subject_uri, parent_uri, parent_rkey, cid, text, created_at) 1093 1093 + VALUES (?, ?, ?, ?, ?, ?, ?, ?) 1094 1094 + ON CONFLICT(actor_did, rkey) DO UPDATE SET 1095 1095 + subject_uri = excluded.subject_uri, 1096 1096 + parent_uri = excluded.parent_uri, 1097 1097 + parent_rkey = excluded.parent_rkey, 1098 1098 + cid = excluded.cid, 1099 1099 + text = excluded.text, 1100 1100 + created_at = excluded.created_at 1101 1101 + `, actorDID, rkey, subjectURI, parentURI, parentRKey, cid, text, createdAt.Format(time.RFC3339Nano)) 1102 1102 + return err 1424 1103 } 1425 1104 1426 1105 // DeleteComment removes a comment from the index 1427 1106 func (idx *FeedIndex) DeleteComment(actorDID, rkey, subjectURI string) error { 1428 1428 - return idx.db.Update(func(tx *bolt.Tx) error { 1429 1429 - comments := tx.Bucket(BucketComments) 1430 1430 - commentCounts := tx.Bucket(BucketCommentCounts) 1431 1431 - commentsByActor := tx.Bucket(BucketCommentsByActor) 1432 1432 - commentChildren := tx.Bucket(BucketCommentChildren) 1433 1433 - 1434 1434 - actorKey := []byte(actorDID + ":" + rkey) 1435 1435 - 1436 1436 - // Get subject URI from the actor index, or use the provided one 1437 1437 - existingSubject := commentsByActor.Get(actorKey) 1438 1438 - if existingSubject != nil && subjectURI == "" { 1439 1439 - subjectURI = string(existingSubject) 1440 1440 - } 1441 1441 - 1442 1442 - // Find and delete the comment from BucketComments 1443 1443 - var parentURI string 1444 1444 - suffix := ":" + actorDID + ":" + rkey 1445 1445 - 1446 1446 - if subjectURI != "" { 1447 1447 - // Fast path: we know the subject URI, scan only that prefix 1448 1448 - prefix := []byte(subjectURI + ":") 1449 1449 - c := comments.Cursor() 1450 1450 - for k, v := c.Seek(prefix); k != nil && strings.HasPrefix(string(k), string(prefix)); k, v = c.Next() { 1451 1451 - if strings.HasSuffix(string(k), suffix) { 1452 1452 - var comment IndexedComment 1453 1453 - if err := json.Unmarshal(v, &comment); err == nil { 1454 1454 - parentURI = comment.ParentURI 1455 1455 - } 1456 1456 - if err := comments.Delete(k); err != nil { 1457 1457 - return fmt.Errorf("failed to delete comment: %w", err) 1458 1458 - } 1459 1459 - break 1460 1460 - } 1461 1461 - } 1462 1462 - } else { 1463 1463 - // Slow path: scan all comments to find this actor+rkey 1464 1464 - c := comments.Cursor() 1465 1465 - for k, v := c.First(); k != nil; k, v = c.Next() { 1466 1466 - if strings.HasSuffix(string(k), suffix) { 1467 1467 - var comment IndexedComment 1468 1468 - if err := json.Unmarshal(v, &comment); err == nil { 1469 1469 - parentURI = comment.ParentURI 1470 1470 - subjectURI = comment.SubjectURI 1471 1471 - } 1472 1472 - if err := comments.Delete(k); err != nil { 1473 1473 - return fmt.Errorf("failed to delete comment: %w", err) 1474 1474 - } 1475 1475 - break 1476 1476 - } 1477 1477 - } 1478 1478 - } 1479 1479 - 1480 1480 - // Delete actor lookup 1481 1481 - if existingSubject != nil { 1482 1482 - if err := commentsByActor.Delete(actorKey); err != nil { 1483 1483 - return fmt.Errorf("failed to delete comment by actor: %w", err) 1484 1484 - } 1485 1485 - } 1486 1486 - 1487 1487 - // Delete parent-child relationship if this was a reply 1488 1488 - if parentURI != "" { 1489 1489 - childKey := []byte(parentURI + ":" + rkey) 1490 1490 - if err := commentChildren.Delete(childKey); err != nil { 1491 1491 - return fmt.Errorf("failed to delete comment child: %w", err) 1492 1492 - } 1493 1493 - } 1494 1494 - 1495 1495 - // Decrement count 1496 1496 - countKey := []byte(subjectURI) 1497 1497 - var count uint64 1498 1498 - if countData := commentCounts.Get(countKey); len(countData) == 8 { 1499 1499 - count = binary.BigEndian.Uint64(countData) 1500 1500 - } 1501 1501 - if count > 0 { 1502 1502 - count-- 1503 1503 - } 1504 1504 - countBytes := make([]byte, 8) 1505 1505 - binary.BigEndian.PutUint64(countBytes, count) 1506 1506 - if err := commentCounts.Put(countKey, countBytes); err != nil { 1507 1507 - return fmt.Errorf("failed to update comment count: %w", err) 1508 1508 - } 1509 1509 - 1510 1510 - return nil 1511 1511 - }) 1107 1107 + _, err := idx.db.Exec(`DELETE FROM comments WHERE actor_did = ? AND rkey = ?`, actorDID, rkey) 1108 1108 + return err 1512 1109 } 1513 1110 1514 1111 // GetCommentCount returns the number of comments on a record 1515 1112 func (idx *FeedIndex) GetCommentCount(subjectURI string) int { 1516 1516 - var count uint64 1517 1517 - _ = idx.db.View(func(tx *bolt.Tx) error { 1518 1518 - commentCounts := tx.Bucket(BucketCommentCounts) 1519 1519 - countData := commentCounts.Get([]byte(subjectURI)) 1520 1520 - if len(countData) == 8 { 1521 1521 - count = binary.BigEndian.Uint64(countData) 1522 1522 - } 1523 1523 - return nil 1524 1524 - }) 1525 1525 - return int(count) 1113 1113 + var count int 1114 1114 + _ = idx.db.QueryRow(`SELECT COUNT(*) FROM comments WHERE subject_uri = ?`, subjectURI).Scan(&count) 1115 1115 + return count 1526 1116 } 1527 1117 1528 1118 // GetCommentsForSubject returns all comments for a specific record, ordered by creation time 1529 1529 - // This returns a flat list of comments without threading 1530 1119 func (idx *FeedIndex) GetCommentsForSubject(ctx context.Context, subjectURI string, limit int, viewerDID string) []IndexedComment { 1531 1531 - var comments []IndexedComment 1532 1532 - _ = idx.db.View(func(tx *bolt.Tx) error { 1533 1533 - bucket := tx.Bucket(BucketComments) 1534 1534 - prefix := []byte(subjectURI + ":") 1535 1535 - c := bucket.Cursor() 1120 1120 + query := `SELECT actor_did, rkey, subject_uri, parent_uri, parent_rkey, cid, text, created_at 1121 1121 + FROM comments WHERE subject_uri = ? ORDER BY created_at` 1122 1122 + var args []any 1123 1123 + args = append(args, subjectURI) 1124 1124 + if limit > 0 { 1125 1125 + query += ` LIMIT ?` 1126 1126 + args = append(args, limit) 1127 1127 + } 1536 1128 1537 1537 - for k, v := c.Seek(prefix); k != nil && strings.HasPrefix(string(k), string(prefix)); k, v = c.Next() { 1538 1538 - var comment IndexedComment 1539 1539 - if err := json.Unmarshal(v, &comment); err != nil { 1540 1540 - continue 1541 1541 - } 1542 1542 - comments = append(comments, comment) 1543 1543 - if limit > 0 && len(comments) >= limit { 1544 1544 - break 1545 1545 - } 1546 1546 - } 1129 1129 + rows, err := idx.db.QueryContext(ctx, query, args...) 1130 1130 + if err != nil { 1547 1131 return nil 1548 1548 - }) 1132 1132 + } 1133 1133 + defer rows.Close() 1134 1134 + 1135 1135 + var comments []IndexedComment 1136 1136 + for rows.Next() { 1137 1137 + var c IndexedComment 1138 1138 + var createdAtStr string 1139 1139 + if err := rows.Scan(&c.ActorDID, &c.RKey, &c.SubjectURI, &c.ParentURI, &c.ParentRKey, 1140 1140 + &c.CID, &c.Text, &createdAtStr); err != nil { 1141 1141 + continue 1142 1142 + } 1143 1143 + c.CreatedAt, _ = time.Parse(time.RFC3339Nano, createdAtStr) 1144 1144 + comments = append(comments, c) 1145 1145 + } 1549 1146 1550 1147 // Populate profile and like info for each comment 1551 1148 for i := range comments { 1552 1149 profile, err := idx.GetProfile(ctx, comments[i].ActorDID) 1553 1150 if err != nil { 1554 1554 - // Use DID as fallback handle 1555 1151 comments[i].Handle = comments[i].ActorDID 1556 1152 } else { 1557 1153 comments[i].Handle = profile.Handle ··· 1570 1166 } 1571 1167 1572 1168 // GetThreadedCommentsForSubject returns comments for a record in threaded order with depth 1573 1573 - // Comments are returned in depth-first order (parent followed by children) 1574 1574 - // Visual depth is capped at 2 levels for display purposes 1575 1169 func (idx *FeedIndex) GetThreadedCommentsForSubject(ctx context.Context, subjectURI string, limit int, viewerDID string) []IndexedComment { 1576 1576 - // First get all comments for this subject 1577 1577 - allComments := idx.GetCommentsForSubject(ctx, subjectURI, 0, viewerDID) // Get all, we'll limit after threading 1170 1170 + allComments := idx.GetCommentsForSubject(ctx, subjectURI, 0, viewerDID) 1578 1171 1579 1172 if len(allComments) == 0 { 1580 1173 return nil ··· 1593 1186 for i := range allComments { 1594 1187 comment := &allComments[i] 1595 1188 if comment.ParentRKey == "" { 1596 1596 - // Top-level comment 1597 1189 topLevel = append(topLevel, comment) 1598 1190 } else { 1599 1599 - // Reply - add to parent's children 1600 1191 childrenMap[comment.ParentRKey] = append(childrenMap[comment.ParentRKey], comment) 1601 1192 } 1602 1193 } ··· 1620 1211 if limit > 0 && len(result) >= limit { 1621 1212 return 1622 1213 } 1623 1623 - // Cap visual depth at 2 for display 1624 1214 visualDepth := depth 1625 1215 if visualDepth > 2 { 1626 1216 visualDepth = 2 ··· 1628 1218 comment.Depth = visualDepth 1629 1219 result = append(result, *comment) 1630 1220 1631 1631 - // Add children (if any) 1632 1221 if children, ok := childrenMap[comment.RKey]; ok { 1633 1222 for _, child := range children { 1634 1223 flatten(child, depth+1)

+75 -147

internal/firehose/notifications.go

··· 1 1 package firehose 2 2 3 3 import ( 4 4 - "encoding/json" 5 4 "fmt" 6 5 "strings" 7 6 "time" ··· 9 8 "arabica/internal/models" 10 9 11 10 "github.com/rs/zerolog/log" 12 12 - bolt "go.etcd.io/bbolt" 13 13 - ) 14 14 - 15 15 - // Bucket names for notifications 16 16 - var ( 17 17 - // BucketNotifications stores notifications: {target_did}:{inverted_timestamp}:{id} -> {Notification JSON} 18 18 - BucketNotifications = []byte("notifications") 19 19 - 20 20 - // BucketNotificationsMeta stores per-user metadata: {target_did}:last_read -> {timestamp RFC3339} 21 21 - BucketNotificationsMeta = []byte("notifications_meta") 22 11 ) 23 12 24 13 // CreateNotification stores a notification for the target user. 25 25 - // Deduplicates by (type + actorDID + subjectURI) to prevent duplicates from backfills. 14 14 + // Deduplicates by (type + actorDID + subjectURI) via unique index. 26 15 // Self-notifications (actorDID == targetDID) are silently skipped. 27 16 func (idx *FeedIndex) CreateNotification(targetDID string, notif models.Notification) error { 28 17 if targetDID == "" || targetDID == notif.ActorDID { 29 18 return nil // skip self-notifications 30 19 } 31 20 32 32 - return idx.db.Update(func(tx *bolt.Tx) error { 33 33 - b := tx.Bucket(BucketNotifications) 34 34 - 35 35 - // Deduplication: scan for existing notification with same type+actor+subject 36 36 - prefix := []byte(targetDID + ":") 37 37 - c := b.Cursor() 38 38 - for k, v := c.Seek(prefix); k != nil && strings.HasPrefix(string(k), string(prefix)); k, v = c.Next() { 39 39 - var existing models.Notification 40 40 - if err := json.Unmarshal(v, &existing); err != nil { 41 41 - continue 42 42 - } 43 43 - if existing.Type == notif.Type && existing.ActorDID == notif.ActorDID && existing.SubjectURI == notif.SubjectURI { 44 44 - return nil // duplicate, skip 45 45 - } 46 46 - } 47 47 - 48 48 - // Generate ID from timestamp 49 49 - if notif.ID == "" { 50 50 - notif.ID = fmt.Sprintf("%d", notif.CreatedAt.UnixNano()) 51 51 - } 52 52 - 53 53 - data, err := json.Marshal(notif) 54 54 - if err != nil { 55 55 - return fmt.Errorf("failed to marshal notification: %w", err) 56 56 - } 21 21 + // Generate ID from timestamp 22 22 + if notif.ID == "" { 23 23 + notif.ID = fmt.Sprintf("%d", notif.CreatedAt.UnixNano()) 24 24 + } 57 25 58 58 - // Key: {target_did}:{inverted_timestamp}:{id} for reverse chronological order 59 59 - inverted := ^uint64(notif.CreatedAt.UnixNano()) 60 60 - key := fmt.Sprintf("%s:%016x:%s", targetDID, inverted, notif.ID) 61 61 - return b.Put([]byte(key), data) 62 62 - }) 26 26 + // INSERT OR IGNORE deduplicates via the unique index on (target_did, type, actor_did, subject_uri) 27 27 + _, err := idx.db.Exec(` 28 28 + INSERT OR IGNORE INTO notifications (id, target_did, type, actor_did, subject_uri, created_at) 29 29 + VALUES (?, ?, ?, ?, ?, ?) 30 30 + `, notif.ID, targetDID, string(notif.Type), notif.ActorDID, notif.SubjectURI, 31 31 + notif.CreatedAt.Format(time.RFC3339Nano)) 32 32 + return err 63 33 } 64 34 65 35 // GetNotifications returns notifications for a user, newest first. 66 36 // Uses cursor-based pagination. Returns notifications, next cursor, and error. 67 37 func (idx *FeedIndex) GetNotifications(targetDID string, limit int, cursor string) ([]models.Notification, string, error) { 68 68 - var notifications []models.Notification 69 69 - var nextCursor string 70 70 - 71 38 if limit <= 0 { 72 39 limit = 20 73 40 } 74 41 75 75 - // Get last_read timestamp for marking read status 76 42 lastRead := idx.getLastRead(targetDID) 77 43 78 78 - err := idx.db.View(func(tx *bolt.Tx) error { 79 79 - b := tx.Bucket(BucketNotifications) 80 80 - c := b.Cursor() 44 44 + var args []any 45 45 + query := `SELECT id, type, actor_did, subject_uri, created_at 46 46 + FROM notifications WHERE target_did = ?` 47 47 + args = append(args, targetDID) 81 48 82 82 - prefix := []byte(targetDID + ":") 83 83 - var k, v []byte 49 49 + if cursor != "" { 50 50 + query += ` AND created_at < ?` 51 51 + args = append(args, cursor) 52 52 + } 84 53 85 85 - if cursor != "" { 86 86 - // Seek to cursor position, then advance past it 87 87 - k, v = c.Seek([]byte(cursor)) 88 88 - if k != nil && string(k) == cursor { 89 89 - k, v = c.Next() 90 90 - } 91 91 - } else { 92 92 - k, v = c.Seek(prefix) 93 93 - } 54 54 + query += ` ORDER BY created_at DESC LIMIT ?` 55 55 + // Fetch one extra to determine if there's a next page 56 56 + args = append(args, limit+1) 94 57 95 95 - var lastKey []byte 96 96 - count := 0 97 97 - for ; k != nil && strings.HasPrefix(string(k), string(prefix)); k, v = c.Next() { 98 98 - if count >= limit { 99 99 - // There are more items beyond our limit 100 100 - nextCursor = string(lastKey) 101 101 - break 102 102 - } 58 58 + rows, err := idx.db.Query(query, args...) 59 59 + if err != nil { 60 60 + return nil, "", err 61 61 + } 62 62 + defer rows.Close() 103 63 104 104 - var notif models.Notification 105 105 - if err := json.Unmarshal(v, &notif); err != nil { 106 106 - continue 107 107 - } 108 108 - 109 109 - // Determine read status based on last_read timestamp 110 110 - if !lastRead.IsZero() && !notif.CreatedAt.After(lastRead) { 111 111 - notif.Read = true 112 112 - } 64 64 + var notifications []models.Notification 65 65 + for rows.Next() { 66 66 + var notif models.Notification 67 67 + var typeStr, createdAtStr string 68 68 + if err := rows.Scan(&notif.ID, &typeStr, &notif.ActorDID, &notif.SubjectURI, &createdAtStr); err != nil { 69 69 + continue 70 70 + } 71 71 + notif.Type = models.NotificationType(typeStr) 72 72 + notif.CreatedAt, _ = time.Parse(time.RFC3339Nano, createdAtStr) 113 73 114 114 - notifications = append(notifications, notif) 115 115 - lastKey = make([]byte, len(k)) 116 116 - copy(lastKey, k) 117 117 - count++ 74 74 + if !lastRead.IsZero() && !notif.CreatedAt.After(lastRead) { 75 75 + notif.Read = true 118 76 } 119 77 120 120 - return nil 121 121 - }) 78 78 + notifications = append(notifications, notif) 79 79 + } 122 80 123 123 - return notifications, nextCursor, err 81 81 + var nextCursor string 82 82 + if len(notifications) > limit { 83 83 + // There are more results 84 84 + last := notifications[limit-1] 85 85 + nextCursor = last.CreatedAt.Format(time.RFC3339Nano) 86 86 + notifications = notifications[:limit] 87 87 + } 88 88 + 89 89 + return notifications, nextCursor, rows.Err() 124 90 } 125 91 126 92 // GetUnreadCount returns the number of unread notifications for a user. ··· 132 98 lastRead := idx.getLastRead(targetDID) 133 99 134 100 var count int 135 135 - _ = idx.db.View(func(tx *bolt.Tx) error { 136 136 - b := tx.Bucket(BucketNotifications) 137 137 - c := b.Cursor() 138 138 - 139 139 - prefix := []byte(targetDID + ":") 140 140 - for k, v := c.Seek(prefix); k != nil && strings.HasPrefix(string(k), string(prefix)); k, v = c.Next() { 141 141 - var notif models.Notification 142 142 - if err := json.Unmarshal(v, &notif); err != nil { 143 143 - continue 144 144 - } 145 145 - // If no last_read set, all are unread 146 146 - if lastRead.IsZero() || notif.CreatedAt.After(lastRead) { 147 147 - count++ 148 148 - } else { 149 149 - // Since keys are in reverse chronological order, 150 150 - // once we hit a read notification, all remaining are also read 151 151 - break 152 152 - } 153 153 - } 154 154 - return nil 155 155 - }) 101 101 + if lastRead.IsZero() { 102 102 + _ = idx.db.QueryRow(`SELECT COUNT(*) FROM notifications WHERE target_did = ?`, targetDID).Scan(&count) 103 103 + } else { 104 104 + _ = idx.db.QueryRow(`SELECT COUNT(*) FROM notifications WHERE target_did = ? AND created_at > ?`, 105 105 + targetDID, lastRead.Format(time.RFC3339Nano)).Scan(&count) 106 106 + } 156 107 157 108 return count 158 109 } 159 110 160 111 // MarkAllRead updates the last_read timestamp to now for the user. 161 112 func (idx *FeedIndex) MarkAllRead(targetDID string) error { 162 162 - return idx.db.Update(func(tx *bolt.Tx) error { 163 163 - b := tx.Bucket(BucketNotificationsMeta) 164 164 - key := []byte(targetDID + ":last_read") 165 165 - return b.Put(key, []byte(time.Now().Format(time.RFC3339Nano))) 166 166 - }) 113 113 + _, err := idx.db.Exec(`INSERT OR REPLACE INTO notifications_meta (target_did, last_read) VALUES (?, ?)`, 114 114 + targetDID, time.Now().Format(time.RFC3339Nano)) 115 115 + return err 167 116 } 168 117 169 118 // getLastRead returns the last_read timestamp for a user. 170 119 func (idx *FeedIndex) getLastRead(targetDID string) time.Time { 171 171 - var lastRead time.Time 172 172 - _ = idx.db.View(func(tx *bolt.Tx) error { 173 173 - b := tx.Bucket(BucketNotificationsMeta) 174 174 - v := b.Get([]byte(targetDID + ":last_read")) 175 175 - if v != nil { 176 176 - if t, err := time.Parse(time.RFC3339Nano, string(v)); err == nil { 177 177 - lastRead = t 178 178 - } 179 179 - } 180 180 - return nil 181 181 - }) 182 182 - return lastRead 120 120 + var lastReadStr string 121 121 + err := idx.db.QueryRow(`SELECT last_read FROM notifications_meta WHERE target_did = ?`, targetDID).Scan(&lastReadStr) 122 122 + if err != nil { 123 123 + return time.Time{} 124 124 + } 125 125 + t, _ := time.Parse(time.RFC3339Nano, lastReadStr) 126 126 + return t 183 127 } 184 128 185 129 // parseTargetDID extracts the DID from an AT-URI (at://did:plc:xxx/collection/rkey) ··· 206 150 return 207 151 } 208 152 209 209 - err := idx.db.Update(func(tx *bolt.Tx) error { 210 210 - b := tx.Bucket(BucketNotifications) 211 211 - prefix := []byte(targetDID + ":") 212 212 - c := b.Cursor() 213 213 - for k, v := c.Seek(prefix); k != nil && strings.HasPrefix(string(k), string(prefix)); k, v = c.Next() { 214 214 - var existing models.Notification 215 215 - if err := json.Unmarshal(v, &existing); err != nil { 216 216 - continue 217 217 - } 218 218 - if existing.Type == notifType && existing.ActorDID == actorDID && existing.SubjectURI == subjectURI { 219 219 - return b.Delete(k) 220 220 - } 221 221 - } 222 222 - return nil 223 223 - }) 153 153 + _, err := idx.db.Exec(` 154 154 + DELETE FROM notifications 155 155 + WHERE target_did = ? AND type = ? AND actor_did = ? AND subject_uri = ? 156 156 + `, targetDID, string(notifType), actorDID, subjectURI) 224 157 if err != nil { 225 158 log.Warn().Err(err).Str("target", targetDID).Str("actor", actorDID).Msg("failed to delete notification") 226 159 } ··· 251 184 // Returns empty string if not found. 252 185 func (idx *FeedIndex) GetCommentSubjectURI(actorDID, rkey string) string { 253 186 var subjectURI string 254 254 - _ = idx.db.View(func(tx *bolt.Tx) error { 255 255 - b := tx.Bucket(BucketCommentsByActor) 256 256 - v := b.Get([]byte(actorDID + ":" + rkey)) 257 257 - if v != nil { 258 258 - subjectURI = string(v) 259 259 - } 260 260 - return nil 261 261 - }) 187 187 + err := idx.db.QueryRow(`SELECT subject_uri FROM comments WHERE actor_did = ? AND rkey = ?`, 188 188 + actorDID, rkey).Scan(&subjectURI) 189 189 + if err != nil { 190 190 + return "" 191 191 + } 262 192 return subjectURI 263 193 } 264 194 ··· 301 231 } 302 232 303 233 // If this is a reply, also notify the parent comment's author. 304 304 - // We store the brew's subjectURI (not the parent comment URI) so the 305 305 - // notification links directly to the brew page with comments. 306 234 if parentURI != "" { 307 235 parentAuthorDID := parseTargetDID(parentURI) 308 236 if parentAuthorDID != "" && parentAuthorDID != actorDID && parentAuthorDID != targetDID {

-220

internal/firehose/suggestions.go

··· 1 1 - package firehose 2 2 - 3 3 - import ( 4 4 - "encoding/json" 5 5 - "sort" 6 6 - "strings" 7 7 - 8 8 - "arabica/internal/atproto" 9 9 - 10 10 - bolt "go.etcd.io/bbolt" 11 11 - ) 12 12 - 13 13 - // EntitySuggestion represents a suggestion for auto-completing an entity 14 14 - type EntitySuggestion struct { 15 15 - Name string `json:"name"` 16 16 - SourceURI string `json:"source_uri"` 17 17 - Fields map[string]string `json:"fields"` 18 18 - Count int `json:"count"` 19 19 - } 20 20 - 21 21 - // entityFieldConfig defines which fields to extract and search for each entity type 22 22 - type entityFieldConfig struct { 23 23 - allFields []string 24 24 - searchFields []string 25 25 - nameField string 26 26 - } 27 27 - 28 28 - var entityConfigs = map[string]entityFieldConfig{ 29 29 - atproto.NSIDRoaster: { 30 30 - allFields: []string{"name", "location", "website"}, 31 31 - searchFields: []string{"name", "location", "website"}, 32 32 - nameField: "name", 33 33 - }, 34 34 - atproto.NSIDGrinder: { 35 35 - allFields: []string{"name", "grinderType", "burrType"}, 36 36 - searchFields: []string{"name", "grinderType", "burrType"}, 37 37 - nameField: "name", 38 38 - }, 39 39 - atproto.NSIDBrewer: { 40 40 - allFields: []string{"name", "brewerType", "description"}, 41 41 - searchFields: []string{"name", "brewerType"}, 42 42 - nameField: "name", 43 43 - }, 44 44 - atproto.NSIDBean: { 45 45 - allFields: []string{"name", "origin", "roastLevel", "process"}, 46 46 - searchFields: []string{"name", "origin", "roastLevel"}, 47 47 - nameField: "name", 48 48 - }, 49 49 - } 50 50 - 51 51 - // SearchEntitySuggestions searches indexed records for entity suggestions matching a query. 52 52 - // It scans BucketByCollection for the given collection, matches against searchable fields, 53 53 - // deduplicates by normalized name, and returns results sorted by popularity. 54 54 - func (idx *FeedIndex) SearchEntitySuggestions(collection, query string, limit int) ([]EntitySuggestion, error) { 55 55 - if limit <= 0 { 56 56 - limit = 10 57 57 - } 58 58 - 59 59 - config, ok := entityConfigs[collection] 60 60 - if !ok { 61 61 - return nil, nil 62 62 - } 63 63 - 64 64 - queryLower := strings.ToLower(strings.TrimSpace(query)) 65 65 - if len(queryLower) < 2 { 66 66 - return nil, nil 67 67 - } 68 68 - 69 69 - // dedupKey -> aggregated suggestion 70 70 - type candidate struct { 71 71 - suggestion EntitySuggestion 72 72 - fieldCount int // number of non-empty fields (to pick best representative) 73 73 - dids map[string]struct{} 74 74 - } 75 75 - candidates := make(map[string]*candidate) 76 76 - 77 77 - err := idx.db.View(func(tx *bolt.Tx) error { 78 78 - byCollection := tx.Bucket(BucketByCollection) 79 79 - recordsBucket := tx.Bucket(BucketRecords) 80 80 - 81 81 - prefix := []byte(collection + ":") 82 82 - c := byCollection.Cursor() 83 83 - 84 84 - for k, _ := c.Seek(prefix); k != nil; k, _ = c.Next() { 85 85 - if !hasPrefix(k, prefix) { 86 86 - break 87 87 - } 88 88 - 89 89 - // Extract URI from collection key 90 90 - uri := extractURIFromCollectionKey(k, collection) 91 91 - if uri == "" { 92 92 - continue 93 93 - } 94 94 - 95 95 - data := recordsBucket.Get([]byte(uri)) 96 96 - if data == nil { 97 97 - continue 98 98 - } 99 99 - 100 100 - var indexed IndexedRecord 101 101 - if err := json.Unmarshal(data, &indexed); err != nil { 102 102 - continue 103 103 - } 104 104 - 105 105 - var recordData map[string]interface{} 106 106 - if err := json.Unmarshal(indexed.Record, &recordData); err != nil { 107 107 - continue 108 108 - } 109 109 - 110 110 - // Extract fields 111 111 - fields := make(map[string]string) 112 112 - for _, f := range config.allFields { 113 113 - if v, ok := recordData[f].(string); ok && v != "" { 114 114 - fields[f] = v 115 115 - } 116 116 - } 117 117 - 118 118 - name := fields[config.nameField] 119 119 - if name == "" { 120 120 - continue 121 121 - } 122 122 - 123 123 - // Check if any searchable field matches the query 124 124 - matched := false 125 125 - for _, sf := range config.searchFields { 126 126 - val := strings.ToLower(fields[sf]) 127 127 - if val == "" { 128 128 - continue 129 129 - } 130 130 - if strings.HasPrefix(val, queryLower) || strings.Contains(val, queryLower) { 131 131 - matched = true 132 132 - break 133 133 - } 134 134 - } 135 135 - if !matched { 136 136 - continue 137 137 - } 138 138 - 139 139 - // Deduplicate by normalized name 140 140 - normalizedName := strings.ToLower(strings.TrimSpace(name)) 141 141 - 142 142 - if existing, ok := candidates[normalizedName]; ok { 143 143 - existing.dids[indexed.DID] = struct{}{} 144 144 - // Keep the record with more complete fields 145 145 - nonEmpty := 0 146 146 - for _, v := range fields { 147 147 - if v != "" { 148 148 - nonEmpty++ 149 149 - } 150 150 - } 151 151 - if nonEmpty > existing.fieldCount { 152 152 - existing.suggestion.Name = name 153 153 - existing.suggestion.Fields = fields 154 154 - existing.suggestion.SourceURI = indexed.URI 155 155 - existing.fieldCount = nonEmpty 156 156 - } 157 157 - } else { 158 158 - nonEmpty := 0 159 159 - for _, v := range fields { 160 160 - if v != "" { 161 161 - nonEmpty++ 162 162 - } 163 163 - } 164 164 - candidates[normalizedName] = &candidate{ 165 165 - suggestion: EntitySuggestion{ 166 166 - Name: name, 167 167 - SourceURI: indexed.URI, 168 168 - Fields: fields, 169 169 - }, 170 170 - fieldCount: nonEmpty, 171 171 - dids: map[string]struct{}{indexed.DID: {}}, 172 172 - } 173 173 - } 174 174 - } 175 175 - 176 176 - return nil 177 177 - }) 178 178 - if err != nil { 179 179 - return nil, err 180 180 - } 181 181 - 182 182 - // Build results with counts 183 183 - results := make([]EntitySuggestion, 0, len(candidates)) 184 184 - for _, c := range candidates { 185 185 - c.suggestion.Count = len(c.dids) 186 186 - results = append(results, c.suggestion) 187 187 - } 188 188 - 189 189 - // Sort: prefix matches first, then by count desc, then alphabetically 190 190 - sort.Slice(results, func(i, j int) bool { 191 191 - iPrefix := strings.HasPrefix(strings.ToLower(results[i].Name), queryLower) 192 192 - jPrefix := strings.HasPrefix(strings.ToLower(results[j].Name), queryLower) 193 193 - if iPrefix != jPrefix { 194 194 - return iPrefix 195 195 - } 196 196 - if results[i].Count != results[j].Count { 197 197 - return results[i].Count > results[j].Count 198 198 - } 199 199 - return strings.ToLower(results[i].Name) < strings.ToLower(results[j].Name) 200 200 - }) 201 201 - 202 202 - if len(results) > limit { 203 203 - results = results[:limit] 204 204 - } 205 205 - 206 206 - return results, nil 207 207 - } 208 208 - 209 209 - // hasPrefix checks if a byte slice starts with a prefix (avoids import of bytes) 210 210 - func hasPrefix(s, prefix []byte) bool { 211 211 - if len(s) < len(prefix) { 212 212 - return false 213 213 - } 214 214 - for i, b := range prefix { 215 215 - if s[i] != b { 216 216 - return false 217 217 - } 218 218 - } 219 219 - return true 220 220 - }

-224

internal/firehose/suggestions_test.go

··· 1 1 - package firehose 2 2 - 3 3 - import ( 4 4 - "encoding/json" 5 5 - "os" 6 6 - "path/filepath" 7 7 - "testing" 8 8 - "time" 9 9 - 10 10 - "arabica/internal/atproto" 11 11 - 12 12 - "github.com/stretchr/testify/assert" 13 13 - ) 14 14 - 15 15 - func newTestFeedIndex(t *testing.T) *FeedIndex { 16 16 - t.Helper() 17 17 - dir := t.TempDir() 18 18 - path := filepath.Join(dir, "test-index.db") 19 19 - idx, err := NewFeedIndex(path, 1*time.Hour) 20 20 - assert.NoError(t, err) 21 21 - t.Cleanup(func() { 22 22 - idx.Close() 23 23 - os.Remove(path) 24 24 - }) 25 25 - return idx 26 26 - } 27 27 - 28 28 - func insertRecord(t *testing.T, idx *FeedIndex, did, collection, rkey string, fields map[string]interface{}) { 29 29 - t.Helper() 30 30 - fields["$type"] = collection 31 31 - fields["createdAt"] = time.Now().Format(time.RFC3339) 32 32 - data, err := json.Marshal(fields) 33 33 - assert.NoError(t, err) 34 34 - err = idx.UpsertRecord(did, collection, rkey, "cid-"+rkey, data, 0) 35 35 - assert.NoError(t, err) 36 36 - } 37 37 - 38 38 - func TestSearchEntitySuggestions_PrefixMatch(t *testing.T) { 39 39 - idx := newTestFeedIndex(t) 40 40 - 41 41 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 42 42 - "name": "Black & White Coffee", 43 43 - "location": "Raleigh, NC", 44 44 - }) 45 45 - insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{ 46 46 - "name": "Blue Bottle", 47 47 - "location": "Oakland, CA", 48 48 - }) 49 49 - 50 50 - results, err := idx.SearchEntitySuggestions(atproto.NSIDRoaster, "bl", 10) 51 51 - assert.NoError(t, err) 52 52 - assert.Len(t, results, 2) 53 53 - // Both match prefix "bl" 54 54 - assert.Equal(t, "Black & White Coffee", results[0].Name) 55 55 - assert.Equal(t, "Blue Bottle", results[1].Name) 56 56 - } 57 57 - 58 58 - func TestSearchEntitySuggestions_CaseInsensitive(t *testing.T) { 59 59 - idx := newTestFeedIndex(t) 60 60 - 61 61 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 62 62 - "name": "Stumptown Coffee", 63 63 - }) 64 64 - 65 65 - results, err := idx.SearchEntitySuggestions(atproto.NSIDRoaster, "STUMP", 10) 66 66 - assert.NoError(t, err) 67 67 - assert.Len(t, results, 1) 68 68 - assert.Equal(t, "Stumptown Coffee", results[0].Name) 69 69 - } 70 70 - 71 71 - func TestSearchEntitySuggestions_SubstringMatch(t *testing.T) { 72 72 - idx := newTestFeedIndex(t) 73 73 - 74 74 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 75 75 - "name": "Red Rooster Coffee", 76 76 - "location": "Floyd, VA", 77 77 - }) 78 78 - 79 79 - // Search by location substring 80 80 - results, err := idx.SearchEntitySuggestions(atproto.NSIDRoaster, "floyd", 10) 81 81 - assert.NoError(t, err) 82 82 - assert.Len(t, results, 1) 83 83 - assert.Equal(t, "Red Rooster Coffee", results[0].Name) 84 84 - } 85 85 - 86 86 - func TestSearchEntitySuggestions_Deduplication(t *testing.T) { 87 87 - idx := newTestFeedIndex(t) 88 88 - 89 89 - // Two users have the same roaster (different case/whitespace) 90 90 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 91 91 - "name": "Counter Culture", 92 92 - "location": "Durham, NC", 93 93 - "website": "https://counterculturecoffee.com", 94 94 - }) 95 95 - insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{ 96 96 - "name": "Counter Culture", 97 97 - }) 98 98 - 99 99 - results, err := idx.SearchEntitySuggestions(atproto.NSIDRoaster, "counter", 10) 100 100 - assert.NoError(t, err) 101 101 - assert.Len(t, results, 1) 102 102 - assert.Equal(t, 2, results[0].Count) 103 103 - // Should keep the more complete record (alice's with location + website) 104 104 - assert.Equal(t, "Durham, NC", results[0].Fields["location"]) 105 105 - } 106 106 - 107 107 - func TestSearchEntitySuggestions_Limit(t *testing.T) { 108 108 - idx := newTestFeedIndex(t) 109 109 - 110 110 - for i := 0; i < 5; i++ { 111 111 - rkey := "r" + string(rune('0'+i)) 112 112 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDGrinder, rkey, map[string]interface{}{ 113 113 - "name": "Grinder " + string(rune('A'+i)), 114 114 - "grinderType": "hand", 115 115 - }) 116 116 - } 117 117 - 118 118 - results, err := idx.SearchEntitySuggestions(atproto.NSIDGrinder, "grinder", 3) 119 119 - assert.NoError(t, err) 120 120 - assert.Len(t, results, 3) 121 121 - } 122 122 - 123 123 - func TestSearchEntitySuggestions_ShortQuery(t *testing.T) { 124 124 - idx := newTestFeedIndex(t) 125 125 - 126 126 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 127 127 - "name": "ABC", 128 128 - }) 129 129 - 130 130 - // Query too short (< 2 chars) 131 131 - results, err := idx.SearchEntitySuggestions(atproto.NSIDRoaster, "a", 10) 132 132 - assert.NoError(t, err) 133 133 - assert.Empty(t, results) 134 134 - 135 135 - // 2 chars should work 136 136 - results, err = idx.SearchEntitySuggestions(atproto.NSIDRoaster, "ab", 10) 137 137 - assert.NoError(t, err) 138 138 - assert.Len(t, results, 1) 139 139 - } 140 140 - 141 141 - func TestSearchEntitySuggestions_EmptyQuery(t *testing.T) { 142 142 - idx := newTestFeedIndex(t) 143 143 - 144 144 - results, err := idx.SearchEntitySuggestions(atproto.NSIDRoaster, "", 10) 145 145 - assert.NoError(t, err) 146 146 - assert.Empty(t, results) 147 147 - } 148 148 - 149 149 - func TestSearchEntitySuggestions_UnknownCollection(t *testing.T) { 150 150 - idx := newTestFeedIndex(t) 151 151 - 152 152 - results, err := idx.SearchEntitySuggestions("unknown.collection", "test", 10) 153 153 - assert.NoError(t, err) 154 154 - assert.Empty(t, results) 155 155 - } 156 156 - 157 157 - func TestSearchEntitySuggestions_GrinderFields(t *testing.T) { 158 158 - idx := newTestFeedIndex(t) 159 159 - 160 160 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDGrinder, "g1", map[string]interface{}{ 161 161 - "name": "1Zpresso JX Pro", 162 162 - "grinderType": "hand", 163 163 - "burrType": "conical", 164 164 - }) 165 165 - 166 166 - results, err := idx.SearchEntitySuggestions(atproto.NSIDGrinder, "1zp", 10) 167 167 - assert.NoError(t, err) 168 168 - assert.Len(t, results, 1) 169 169 - assert.Equal(t, "hand", results[0].Fields["grinderType"]) 170 170 - assert.Equal(t, "conical", results[0].Fields["burrType"]) 171 171 - } 172 172 - 173 173 - func TestSearchEntitySuggestions_BeanFields(t *testing.T) { 174 174 - idx := newTestFeedIndex(t) 175 175 - 176 176 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDBean, "b1", map[string]interface{}{ 177 177 - "name": "Ethiopian Yirgacheffe", 178 178 - "origin": "Ethiopia", 179 179 - "roastLevel": "Light", 180 180 - "process": "Washed", 181 181 - }) 182 182 - 183 183 - // Search by origin 184 184 - results, err := idx.SearchEntitySuggestions(atproto.NSIDBean, "ethiopia", 10) 185 185 - assert.NoError(t, err) 186 186 - assert.Len(t, results, 1) 187 187 - assert.Equal(t, "Ethiopian Yirgacheffe", results[0].Name) 188 188 - assert.Equal(t, "Light", results[0].Fields["roastLevel"]) 189 189 - } 190 190 - 191 191 - func TestSearchEntitySuggestions_BrewerFields(t *testing.T) { 192 192 - idx := newTestFeedIndex(t) 193 193 - 194 194 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDBrewer, "br1", map[string]interface{}{ 195 195 - "name": "Hario V60", 196 196 - "brewerType": "Pour-Over", 197 197 - }) 198 198 - 199 199 - results, err := idx.SearchEntitySuggestions(atproto.NSIDBrewer, "hario", 10) 200 200 - assert.NoError(t, err) 201 201 - assert.Len(t, results, 1) 202 202 - assert.Equal(t, "Pour-Over", results[0].Fields["brewerType"]) 203 203 - } 204 204 - 205 205 - func TestSearchEntitySuggestions_SortOrder(t *testing.T) { 206 206 - idx := newTestFeedIndex(t) 207 207 - 208 208 - // "Alpha Roasters" used by 3 people 209 209 - insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{"name": "Alpha Roasters"}) 210 210 - insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{"name": "Alpha Roasters"}) 211 211 - insertRecord(t, idx, "did:plc:charlie", atproto.NSIDRoaster, "r3", map[string]interface{}{"name": "Alpha Roasters"}) 212 212 - 213 213 - // "Alpha Beta" used by 1 person 214 214 - insertRecord(t, idx, "did:plc:dave", atproto.NSIDRoaster, "r4", map[string]interface{}{"name": "Alpha Beta"}) 215 215 - 216 216 - results, err := idx.SearchEntitySuggestions(atproto.NSIDRoaster, "alpha", 10) 217 217 - assert.NoError(t, err) 218 218 - assert.Len(t, results, 2) 219 219 - // More popular first 220 220 - assert.Equal(t, "Alpha Roasters", results[0].Name) 221 221 - assert.Equal(t, 3, results[0].Count) 222 222 - assert.Equal(t, "Alpha Beta", results[1].Name) 223 223 - assert.Equal(t, 1, results[1].Count) 224 224 - }

+2 -2

internal/handlers/handlers.go

··· 47 47 48 48 // Moderation dependencies (optional) 49 49 moderationService *moderation.Service 50 50 - moderationStore *boltstore.ModerationStore 50 50 + moderationStore moderation.Store 51 51 52 52 // Join request dependencies (optional) 53 53 emailSender *email.Sender ··· 83 83 } 84 84 85 85 // SetModeration configures the handler with moderation service and store 86 86 - func (h *Handler) SetModeration(svc *moderation.Service, store *boltstore.ModerationStore) { 86 86 + func (h *Handler) SetModeration(svc *moderation.Service, store moderation.Store) { 87 87 h.moderationService = svc 88 88 h.moderationStore = store 89 89 }

+5 -5

internal/handlers/suggestions.go

··· 6 6 "strconv" 7 7 8 8 "arabica/internal/atproto" 9 9 - "arabica/internal/firehose" 9 9 + "arabica/internal/suggestions" 10 10 11 11 "github.com/rs/zerolog/log" 12 12 ) ··· 57 57 return 58 58 } 59 59 60 60 - suggestions, err := h.feedIndex.SearchEntitySuggestions(nsid, query, limit) 60 60 + results, err := suggestions.Search(h.feedIndex, nsid, query, limit) 61 61 if err != nil { 62 62 log.Error().Err(err).Str("entity", entityType).Str("query", query).Msg("Failed to search suggestions") 63 63 http.Error(w, "Failed to search suggestions", http.StatusInternalServerError) 64 64 return 65 65 } 66 66 67 67 - if suggestions == nil { 68 68 - suggestions = []firehose.EntitySuggestion{} 67 67 + if results == nil { 68 68 + results = []suggestions.EntitySuggestion{} 69 69 } 70 70 71 71 w.Header().Set("Content-Type", "application/json") 72 72 - if err := json.NewEncoder(w).Encode(suggestions); err != nil { 72 72 + if err := json.NewEncoder(w).Encode(results); err != nil { 73 73 log.Error().Err(err).Msg("Failed to encode suggestions response") 74 74 } 75 75 }

+44

internal/moderation/store.go

··· 1 1 + package moderation 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "time" 6 6 + ) 7 7 + 8 8 + // Store defines the persistence interface for moderation data. 9 9 + // Implementations must be safe for concurrent use. 10 10 + type Store interface { 11 11 + // Hidden records 12 12 + HideRecord(ctx context.Context, entry HiddenRecord) error 13 13 + UnhideRecord(ctx context.Context, atURI string) error 14 14 + IsRecordHidden(ctx context.Context, atURI string) bool 15 15 + GetHiddenRecord(ctx context.Context, atURI string) (*HiddenRecord, error) 16 16 + ListHiddenRecords(ctx context.Context) ([]HiddenRecord, error) 17 17 + 18 18 + // Blacklist 19 19 + BlacklistUser(ctx context.Context, entry BlacklistedUser) error 20 20 + UnblacklistUser(ctx context.Context, did string) error 21 21 + IsBlacklisted(ctx context.Context, did string) bool 22 22 + GetBlacklistedUser(ctx context.Context, did string) (*BlacklistedUser, error) 23 23 + ListBlacklistedUsers(ctx context.Context) ([]BlacklistedUser, error) 24 24 + 25 25 + // Reports 26 26 + CreateReport(ctx context.Context, report Report) error 27 27 + GetReport(ctx context.Context, id string) (*Report, error) 28 28 + ListPendingReports(ctx context.Context) ([]Report, error) 29 29 + ListAllReports(ctx context.Context) ([]Report, error) 30 30 + ResolveReport(ctx context.Context, id string, status ReportStatus, resolvedBy string) error 31 31 + CountReportsForURI(ctx context.Context, atURI string) (int, error) 32 32 + CountReportsForDID(ctx context.Context, did string) (int, error) 33 33 + CountReportsForDIDSince(ctx context.Context, did string, since time.Time) (int, error) 34 34 + HasReportedURI(ctx context.Context, reporterDID, subjectURI string) (bool, error) 35 35 + CountReportsFromUserSince(ctx context.Context, reporterDID string, since time.Time) (int, error) 36 36 + 37 37 + // Audit log 38 38 + LogAction(ctx context.Context, entry AuditEntry) error 39 39 + ListAuditLog(ctx context.Context, limit int) ([]AuditEntry, error) 40 40 + 41 41 + // Auto-hide resets 42 42 + SetAutoHideReset(ctx context.Context, did string, resetAt time.Time) error 43 43 + GetAutoHideReset(ctx context.Context, did string) (time.Time, error) 44 44 + }

+319

internal/suggestions/suggestions.go

··· 1 1 + package suggestions 2 2 + 3 3 + import ( 4 4 + "encoding/json" 5 5 + "regexp" 6 6 + "sort" 7 7 + "strings" 8 8 + 9 9 + "arabica/internal/atproto" 10 10 + "arabica/internal/firehose" 11 11 + ) 12 12 + 13 13 + // EntitySuggestion represents a suggestion for auto-completing an entity 14 14 + type EntitySuggestion struct { 15 15 + Name string `json:"name"` 16 16 + SourceURI string `json:"source_uri"` 17 17 + Fields map[string]string `json:"fields"` 18 18 + Count int `json:"count"` 19 19 + } 20 20 + 21 21 + // RecordSource provides read access to indexed records. 22 22 + type RecordSource interface { 23 23 + ListRecordsByCollection(collection string) ([]firehose.IndexedRecord, error) 24 24 + } 25 25 + 26 26 + // entityFieldConfig defines which fields to extract and search for each entity type 27 27 + type entityFieldConfig struct { 28 28 + allFields []string 29 29 + searchFields []string 30 30 + nameField string 31 31 + dedupKey func(fields map[string]string) string 32 32 + } 33 33 + 34 34 + var entityConfigs = map[string]entityFieldConfig{ 35 35 + atproto.NSIDRoaster: { 36 36 + allFields: []string{"name", "location", "website"}, 37 37 + searchFields: []string{"name", "location", "website"}, 38 38 + nameField: "name", 39 39 + dedupKey: roasterDedupKey, 40 40 + }, 41 41 + atproto.NSIDGrinder: { 42 42 + allFields: []string{"name", "grinderType", "burrType"}, 43 43 + searchFields: []string{"name", "grinderType", "burrType"}, 44 44 + nameField: "name", 45 45 + dedupKey: grinderDedupKey, 46 46 + }, 47 47 + atproto.NSIDBrewer: { 48 48 + allFields: []string{"name", "brewerType", "description"}, 49 49 + searchFields: []string{"name", "brewerType"}, 50 50 + nameField: "name", 51 51 + dedupKey: brewerDedupKey, 52 52 + }, 53 53 + atproto.NSIDBean: { 54 54 + allFields: []string{"name", "origin", "roastLevel", "process"}, 55 55 + searchFields: []string{"name", "origin", "roastLevel"}, 56 56 + nameField: "name", 57 57 + dedupKey: beanDedupKey, 58 58 + }, 59 59 + } 60 60 + 61 61 + // --- Dedup key functions --- 62 62 + // Each returns a string that groups "same entity" records together. 63 63 + // Records with the same dedup key are merged; different keys stay separate. 64 64 + 65 65 + // roasterDedupKey: fuzzy name + normalized location. 66 66 + // "Counter Culture Coffee" in "Durham, NC" vs "Counter Culture" in "Durham" → same. 67 67 + // "Stumptown" in "Portland" vs "Stumptown" in "NYC" → different. 68 68 + // Website is not included because it's too sparse — many records lack one, 69 69 + // causing false splits. Website is still kept in Fields for display. 70 70 + func roasterDedupKey(fields map[string]string) string { 71 71 + parts := []string{fuzzyName(fields["name"])} 72 72 + if loc := normalize(fields["location"]); loc != "" { 73 73 + parts = append(parts, loc) 74 74 + } 75 75 + return strings.Join(parts, "|") 76 76 + } 77 77 + 78 78 + // grinderDedupKey: exact name + grinder type + burr type. 79 79 + // "1Zpresso JX Pro" hand/conical vs "1Zpresso JX Pro" electric/flat → different. 80 80 + func grinderDedupKey(fields map[string]string) string { 81 81 + parts := []string{normalize(fields["name"])} 82 82 + if gt := normalize(fields["grinderType"]); gt != "" { 83 83 + parts = append(parts, gt) 84 84 + } 85 85 + if bt := normalize(fields["burrType"]); bt != "" { 86 86 + parts = append(parts, bt) 87 87 + } 88 88 + return strings.Join(parts, "|") 89 89 + } 90 90 + 91 91 + // brewerDedupKey: exact name + brewer type. 92 92 + // "Hario V60" pour-over vs "Hario V60" dripper → different (if someone miscategorized). 93 93 + func brewerDedupKey(fields map[string]string) string { 94 94 + parts := []string{normalize(fields["name"])} 95 95 + if bt := normalize(fields["brewerType"]); bt != "" { 96 96 + parts = append(parts, bt) 97 97 + } 98 98 + return strings.Join(parts, "|") 99 99 + } 100 100 + 101 101 + // beanDedupKey: exact name + origin + process. 102 102 + // "Yirgacheffe" from Ethiopia/washed vs "Yirgacheffe" from Ethiopia/natural → different. 103 103 + func beanDedupKey(fields map[string]string) string { 104 104 + parts := []string{normalize(fields["name"])} 105 105 + if o := normalize(fields["origin"]); o != "" { 106 106 + parts = append(parts, o) 107 107 + } 108 108 + if p := normalize(fields["process"]); p != "" { 109 109 + parts = append(parts, p) 110 110 + } 111 111 + return strings.Join(parts, "|") 112 112 + } 113 113 + 114 114 + // --- Normalization helpers --- 115 115 + 116 116 + // normalize lowercases, trims whitespace, and collapses internal whitespace. 117 117 + func normalize(s string) string { 118 118 + return collapseSpaces(strings.ToLower(strings.TrimSpace(s))) 119 119 + } 120 120 + 121 121 + // Common suffixes stripped during fuzzy name normalization for roasters/brewers. 122 122 + // Order matters: longer suffixes first to avoid partial stripping. 123 123 + var commonSuffixes = []string{ 124 124 + "coffee roasters", 125 125 + "coffee roasting", 126 126 + "coffee company", 127 127 + "coffee co", 128 128 + "roasting company", 129 129 + "roasting co", 130 130 + "roasters", 131 131 + "roasting", 132 132 + "coffee", 133 133 + "co.", 134 134 + } 135 135 + 136 136 + // fuzzyName normalizes a name by lowercasing, stripping common coffee-industry 137 137 + // suffixes, punctuation, and extra whitespace. This lets "Counter Culture Coffee" 138 138 + // and "Counter Culture" merge, while still keeping genuinely different names apart. 139 139 + func fuzzyName(name string) string { 140 140 + s := strings.ToLower(strings.TrimSpace(name)) 141 141 + 142 142 + // Strip common suffixes 143 143 + for _, suffix := range commonSuffixes { 144 144 + if strings.HasSuffix(s, suffix) { 145 145 + s = strings.TrimSpace(s[:len(s)-len(suffix)]) 146 146 + break // only strip one suffix 147 147 + } 148 148 + } 149 149 + 150 150 + // Remove punctuation (keep letters, digits, spaces) 151 151 + s = stripPunctuation(s) 152 152 + 153 153 + return collapseSpaces(s) 154 154 + } 155 155 + 156 156 + var nonAlphanumSpace = regexp.MustCompile(`[^a-z0-9\s]`) 157 157 + 158 158 + func stripPunctuation(s string) string { 159 159 + return nonAlphanumSpace.ReplaceAllString(s, "") 160 160 + } 161 161 + 162 162 + var multiSpace = regexp.MustCompile(`\s+`) 163 163 + 164 164 + func collapseSpaces(s string) string { 165 165 + return strings.TrimSpace(multiSpace.ReplaceAllString(s, " ")) 166 166 + } 167 167 + 168 168 + // extractDomain pulls the domain from a URL for normalization. 169 169 + // "https://www.counterculturecoffee.com/shop" → "counterculturecoffee.com" 170 170 + func extractDomain(rawURL string) string { 171 171 + if rawURL == "" { 172 172 + return "" 173 173 + } 174 174 + s := strings.ToLower(strings.TrimSpace(rawURL)) 175 175 + // Strip scheme 176 176 + if i := strings.Index(s, "://"); i >= 0 { 177 177 + s = s[i+3:] 178 178 + } 179 179 + // Strip www. 180 180 + s = strings.TrimPrefix(s, "www.") 181 181 + // Strip path 182 182 + if i := strings.IndexByte(s, '/'); i >= 0 { 183 183 + s = s[:i] 184 184 + } 185 185 + // Strip port 186 186 + if i := strings.IndexByte(s, ':'); i >= 0 { 187 187 + s = s[:i] 188 188 + } 189 189 + return s 190 190 + } 191 191 + 192 192 + // Search searches indexed records for entity suggestions matching a query. 193 193 + // It matches against searchable fields, deduplicates using entity-specific 194 194 + // keys, and returns results sorted by popularity. 195 195 + func Search(source RecordSource, collection, query string, limit int) ([]EntitySuggestion, error) { 196 196 + if limit <= 0 { 197 197 + limit = 10 198 198 + } 199 199 + 200 200 + config, ok := entityConfigs[collection] 201 201 + if !ok { 202 202 + return nil, nil 203 203 + } 204 204 + 205 205 + queryLower := strings.ToLower(strings.TrimSpace(query)) 206 206 + if len(queryLower) < 2 { 207 207 + return nil, nil 208 208 + } 209 209 + 210 210 + records, err := source.ListRecordsByCollection(collection) 211 211 + if err != nil { 212 212 + return nil, err 213 213 + } 214 214 + 215 215 + // dedupKey -> aggregated suggestion 216 216 + type candidate struct { 217 217 + suggestion EntitySuggestion 218 218 + fieldCount int // number of non-empty fields (to pick best representative) 219 219 + dids map[string]struct{} 220 220 + } 221 221 + candidates := make(map[string]*candidate) 222 222 + 223 223 + for _, indexed := range records { 224 224 + var recordData map[string]any 225 225 + if err := json.Unmarshal(indexed.Record, &recordData); err != nil { 226 226 + continue 227 227 + } 228 228 + 229 229 + // Extract fields 230 230 + fields := make(map[string]string) 231 231 + for _, f := range config.allFields { 232 232 + if v, ok := recordData[f].(string); ok && v != "" { 233 233 + fields[f] = v 234 234 + } 235 235 + } 236 236 + 237 237 + name := fields[config.nameField] 238 238 + if name == "" { 239 239 + continue 240 240 + } 241 241 + 242 242 + // Check if any searchable field matches the query 243 243 + matched := false 244 244 + for _, sf := range config.searchFields { 245 245 + val := strings.ToLower(fields[sf]) 246 246 + if val == "" { 247 247 + continue 248 248 + } 249 249 + if strings.HasPrefix(val, queryLower) || strings.Contains(val, queryLower) { 250 250 + matched = true 251 251 + break 252 252 + } 253 253 + } 254 254 + if !matched { 255 255 + continue 256 256 + } 257 257 + 258 258 + // Deduplicate using entity-specific key 259 259 + key := config.dedupKey(fields) 260 260 + 261 261 + if existing, ok := candidates[key]; ok { 262 262 + existing.dids[indexed.DID] = struct{}{} 263 263 + // Keep the record with more complete fields 264 264 + nonEmpty := countNonEmpty(fields) 265 265 + if nonEmpty > existing.fieldCount { 266 266 + existing.suggestion.Name = name 267 267 + existing.suggestion.Fields = fields 268 268 + existing.suggestion.SourceURI = indexed.URI 269 269 + existing.fieldCount = nonEmpty 270 270 + } 271 271 + } else { 272 272 + candidates[key] = &candidate{ 273 273 + suggestion: EntitySuggestion{ 274 274 + Name: name, 275 275 + SourceURI: indexed.URI, 276 276 + Fields: fields, 277 277 + }, 278 278 + fieldCount: countNonEmpty(fields), 279 279 + dids: map[string]struct{}{indexed.DID: {}}, 280 280 + } 281 281 + } 282 282 + } 283 283 + 284 284 + // Build results with counts 285 285 + results := make([]EntitySuggestion, 0, len(candidates)) 286 286 + for _, c := range candidates { 287 287 + c.suggestion.Count = len(c.dids) 288 288 + results = append(results, c.suggestion) 289 289 + } 290 290 + 291 291 + // Sort: prefix matches first, then by count desc, then alphabetically 292 292 + sort.Slice(results, func(i, j int) bool { 293 293 + iPrefix := strings.HasPrefix(strings.ToLower(results[i].Name), queryLower) 294 294 + jPrefix := strings.HasPrefix(strings.ToLower(results[j].Name), queryLower) 295 295 + if iPrefix != jPrefix { 296 296 + return iPrefix 297 297 + } 298 298 + if results[i].Count != results[j].Count { 299 299 + return results[i].Count > results[j].Count 300 300 + } 301 301 + return strings.ToLower(results[i].Name) < strings.ToLower(results[j].Name) 302 302 + }) 303 303 + 304 304 + if len(results) > limit { 305 305 + results = results[:limit] 306 306 + } 307 307 + 308 308 + return results, nil 309 309 + } 310 310 + 311 311 + func countNonEmpty(fields map[string]string) int { 312 312 + n := 0 313 313 + for _, v := range fields { 314 314 + if v != "" { 315 315 + n++ 316 316 + } 317 317 + } 318 318 + return n 319 319 + }

+460

internal/suggestions/suggestions_test.go

··· 1 1 + package suggestions 2 2 + 3 3 + import ( 4 4 + "encoding/json" 5 5 + "os" 6 6 + "path/filepath" 7 7 + "testing" 8 8 + "time" 9 9 + 10 10 + "arabica/internal/atproto" 11 11 + "arabica/internal/firehose" 12 12 + 13 13 + "github.com/stretchr/testify/assert" 14 14 + ) 15 15 + 16 16 + func newTestFeedIndex(t *testing.T) *firehose.FeedIndex { 17 17 + t.Helper() 18 18 + dir := t.TempDir() 19 19 + path := filepath.Join(dir, "test-index.db") 20 20 + idx, err := firehose.NewFeedIndex(path, 1*time.Hour) 21 21 + assert.NoError(t, err) 22 22 + t.Cleanup(func() { 23 23 + idx.Close() 24 24 + os.Remove(path) 25 25 + }) 26 26 + return idx 27 27 + } 28 28 + 29 29 + func insertRecord(t *testing.T, idx *firehose.FeedIndex, did, collection, rkey string, fields map[string]interface{}) { 30 30 + t.Helper() 31 31 + fields["$type"] = collection 32 32 + fields["createdAt"] = time.Now().Format(time.RFC3339) 33 33 + data, err := json.Marshal(fields) 34 34 + assert.NoError(t, err) 35 35 + err = idx.UpsertRecord(did, collection, rkey, "cid-"+rkey, data, 0) 36 36 + assert.NoError(t, err) 37 37 + } 38 38 + 39 39 + // --- Helper unit tests --- 40 40 + 41 41 + func TestFuzzyName(t *testing.T) { 42 42 + tests := []struct { 43 43 + input string 44 44 + want string 45 45 + }{ 46 46 + {"Counter Culture Coffee", "counter culture"}, 47 47 + {"Counter Culture", "counter culture"}, 48 48 + {"counter culture coffee roasters", "counter culture"}, 49 49 + {"Stumptown Coffee Roasters", "stumptown"}, 50 50 + {"Stumptown", "stumptown"}, 51 51 + {"Black & White Coffee", "black white"}, 52 52 + {" Some Roasting Company ", "some"}, 53 53 + {"Heart Coffee Roasters", "heart"}, 54 54 + {"Heart Roasters", "heart"}, 55 55 + {"Heart", "heart"}, 56 56 + } 57 57 + for _, tt := range tests { 58 58 + assert.Equal(t, tt.want, fuzzyName(tt.input), "fuzzyName(%q)", tt.input) 59 59 + } 60 60 + } 61 61 + 62 62 + func TestExtractDomain(t *testing.T) { 63 63 + tests := []struct { 64 64 + input string 65 65 + want string 66 66 + }{ 67 67 + {"https://www.counterculturecoffee.com/shop", "counterculturecoffee.com"}, 68 68 + {"http://example.com", "example.com"}, 69 69 + {"https://example.com:8080/path", "example.com"}, 70 70 + {"www.example.com", "example.com"}, 71 71 + {"example.com", "example.com"}, 72 72 + {"", ""}, 73 73 + } 74 74 + for _, tt := range tests { 75 75 + assert.Equal(t, tt.want, extractDomain(tt.input), "extractDomain(%q)", tt.input) 76 76 + } 77 77 + } 78 78 + 79 79 + func TestNormalize(t *testing.T) { 80 80 + assert.Equal(t, "durham, nc", normalize(" Durham, NC ")) 81 81 + assert.Equal(t, "oakland ca", normalize("Oakland CA")) 82 82 + assert.Equal(t, "", normalize("")) 83 83 + } 84 84 + 85 85 + // --- Roaster dedup tests --- 86 86 + 87 87 + func TestRoasterDedup_SameNameDifferentLocation(t *testing.T) { 88 88 + idx := newTestFeedIndex(t) 89 89 + 90 90 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 91 91 + "name": "Stumptown Coffee", 92 92 + "location": "Portland, OR", 93 93 + }) 94 94 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{ 95 95 + "name": "Stumptown Coffee", 96 96 + "location": "New York, NY", 97 97 + }) 98 98 + 99 99 + results, err := Search(idx, atproto.NSIDRoaster, "stumptown", 10) 100 100 + assert.NoError(t, err) 101 101 + assert.Len(t, results, 2, "different locations should produce separate suggestions") 102 102 + } 103 103 + 104 104 + func TestRoasterDedup_FuzzyNameMerge(t *testing.T) { 105 105 + idx := newTestFeedIndex(t) 106 106 + 107 107 + // Same roaster, different name variations, same location 108 108 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 109 109 + "name": "Counter Culture Coffee", 110 110 + "location": "Durham, NC", 111 111 + "website": "https://counterculturecoffee.com", 112 112 + }) 113 113 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{ 114 114 + "name": "Counter Culture Coffee Roasters", 115 115 + "location": "Durham, NC", 116 116 + }) 117 117 + 118 118 + results, err := Search(idx, atproto.NSIDRoaster, "counter", 10) 119 119 + assert.NoError(t, err) 120 120 + assert.Len(t, results, 1, "fuzzy name + same location should merge") 121 121 + assert.Equal(t, 2, results[0].Count) 122 122 + } 123 123 + 124 124 + func TestRoasterDedup_NoLocationMerges(t *testing.T) { 125 125 + idx := newTestFeedIndex(t) 126 126 + 127 127 + // Both have no location — should merge on fuzzy name alone 128 128 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 129 129 + "name": "Blue Bottle Coffee", 130 130 + }) 131 131 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{ 132 132 + "name": "Blue Bottle", 133 133 + }) 134 134 + 135 135 + results, err := Search(idx, atproto.NSIDRoaster, "blue", 10) 136 136 + assert.NoError(t, err) 137 137 + assert.Len(t, results, 1, "same fuzzy name with no location should merge") 138 138 + assert.Equal(t, 2, results[0].Count) 139 139 + } 140 140 + 141 141 + // --- Grinder dedup tests --- 142 142 + 143 143 + func TestGrinderDedup_SameNameDifferentType(t *testing.T) { 144 144 + idx := newTestFeedIndex(t) 145 145 + 146 146 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDGrinder, "g1", map[string]interface{}{ 147 147 + "name": "Baratza Encore", 148 148 + "grinderType": "electric", 149 149 + "burrType": "conical", 150 150 + }) 151 151 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDGrinder, "g2", map[string]interface{}{ 152 152 + "name": "Baratza Encore", 153 153 + "grinderType": "electric", 154 154 + "burrType": "flat", 155 155 + }) 156 156 + 157 157 + results, err := Search(idx, atproto.NSIDGrinder, "baratza", 10) 158 158 + assert.NoError(t, err) 159 159 + assert.Len(t, results, 2, "different burr types should produce separate suggestions") 160 160 + } 161 161 + 162 162 + func TestGrinderDedup_SameEverythingMerges(t *testing.T) { 163 163 + idx := newTestFeedIndex(t) 164 164 + 165 165 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDGrinder, "g1", map[string]interface{}{ 166 166 + "name": "1Zpresso JX Pro", 167 167 + "grinderType": "hand", 168 168 + "burrType": "conical", 169 169 + }) 170 170 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDGrinder, "g2", map[string]interface{}{ 171 171 + "name": "1Zpresso JX Pro", 172 172 + "grinderType": "hand", 173 173 + "burrType": "conical", 174 174 + }) 175 175 + 176 176 + results, err := Search(idx, atproto.NSIDGrinder, "1zp", 10) 177 177 + assert.NoError(t, err) 178 178 + assert.Len(t, results, 1) 179 179 + assert.Equal(t, 2, results[0].Count) 180 180 + } 181 181 + 182 182 + // --- Brewer dedup tests --- 183 183 + 184 184 + func TestBrewerDedup_SameNameDifferentType(t *testing.T) { 185 185 + idx := newTestFeedIndex(t) 186 186 + 187 187 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDBrewer, "br1", map[string]interface{}{ 188 188 + "name": "Hario V60", 189 189 + "brewerType": "pour-over", 190 190 + }) 191 191 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDBrewer, "br2", map[string]interface{}{ 192 192 + "name": "Hario V60", 193 193 + "brewerType": "dripper", 194 194 + }) 195 195 + 196 196 + results, err := Search(idx, atproto.NSIDBrewer, "hario", 10) 197 197 + assert.NoError(t, err) 198 198 + assert.Len(t, results, 2, "different brewer types should produce separate suggestions") 199 199 + } 200 200 + 201 201 + func TestBrewerDedup_SameNameSameTypeMerges(t *testing.T) { 202 202 + idx := newTestFeedIndex(t) 203 203 + 204 204 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDBrewer, "br1", map[string]interface{}{ 205 205 + "name": "AeroPress", 206 206 + "brewerType": "immersion", 207 207 + }) 208 208 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDBrewer, "br2", map[string]interface{}{ 209 209 + "name": "AeroPress", 210 210 + "brewerType": "immersion", 211 211 + }) 212 212 + 213 213 + results, err := Search(idx, atproto.NSIDBrewer, "aero", 10) 214 214 + assert.NoError(t, err) 215 215 + assert.Len(t, results, 1) 216 216 + assert.Equal(t, 2, results[0].Count) 217 217 + } 218 218 + 219 219 + // --- Bean dedup tests --- 220 220 + 221 221 + func TestBeanDedup_SameNameDifferentProcess(t *testing.T) { 222 222 + idx := newTestFeedIndex(t) 223 223 + 224 224 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDBean, "b1", map[string]interface{}{ 225 225 + "name": "Yirgacheffe", 226 226 + "origin": "Ethiopia", 227 227 + "process": "Washed", 228 228 + }) 229 229 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDBean, "b2", map[string]interface{}{ 230 230 + "name": "Yirgacheffe", 231 231 + "origin": "Ethiopia", 232 232 + "process": "Natural", 233 233 + }) 234 234 + 235 235 + results, err := Search(idx, atproto.NSIDBean, "yirga", 10) 236 236 + assert.NoError(t, err) 237 237 + assert.Len(t, results, 2, "different processes should produce separate suggestions") 238 238 + } 239 239 + 240 240 + func TestBeanDedup_SameNameDifferentOrigin(t *testing.T) { 241 241 + idx := newTestFeedIndex(t) 242 242 + 243 243 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDBean, "b1", map[string]interface{}{ 244 244 + "name": "Gesha", 245 245 + "origin": "Panama", 246 246 + }) 247 247 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDBean, "b2", map[string]interface{}{ 248 248 + "name": "Gesha", 249 249 + "origin": "Ethiopia", 250 250 + }) 251 251 + 252 252 + results, err := Search(idx, atproto.NSIDBean, "gesha", 10) 253 253 + assert.NoError(t, err) 254 254 + assert.Len(t, results, 2, "different origins should produce separate suggestions") 255 255 + } 256 256 + 257 257 + func TestBeanDedup_SameEverythingMerges(t *testing.T) { 258 258 + idx := newTestFeedIndex(t) 259 259 + 260 260 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDBean, "b1", map[string]interface{}{ 261 261 + "name": "Ethiopian Yirgacheffe", 262 262 + "origin": "Ethiopia", 263 263 + "roastLevel": "Light", 264 264 + "process": "Washed", 265 265 + }) 266 266 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDBean, "b2", map[string]interface{}{ 267 267 + "name": "Ethiopian Yirgacheffe", 268 268 + "origin": "Ethiopia", 269 269 + "process": "Washed", 270 270 + }) 271 271 + 272 272 + results, err := Search(idx, atproto.NSIDBean, "ethiopia", 10) 273 273 + assert.NoError(t, err) 274 274 + assert.Len(t, results, 1) 275 275 + assert.Equal(t, 2, results[0].Count) 276 276 + } 277 277 + 278 278 + // --- General search tests --- 279 279 + 280 280 + func TestSearch_PrefixMatch(t *testing.T) { 281 281 + idx := newTestFeedIndex(t) 282 282 + 283 283 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 284 284 + "name": "Black & White Coffee", 285 285 + "location": "Raleigh, NC", 286 286 + }) 287 287 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{ 288 288 + "name": "Blue Bottle", 289 289 + "location": "Oakland, CA", 290 290 + }) 291 291 + 292 292 + results, err := Search(idx, atproto.NSIDRoaster, "bl", 10) 293 293 + assert.NoError(t, err) 294 294 + assert.Len(t, results, 2) 295 295 + assert.Equal(t, "Black & White Coffee", results[0].Name) 296 296 + assert.Equal(t, "Blue Bottle", results[1].Name) 297 297 + } 298 298 + 299 299 + func TestSearch_CaseInsensitive(t *testing.T) { 300 300 + idx := newTestFeedIndex(t) 301 301 + 302 302 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 303 303 + "name": "Stumptown Coffee", 304 304 + }) 305 305 + 306 306 + results, err := Search(idx, atproto.NSIDRoaster, "STUMP", 10) 307 307 + assert.NoError(t, err) 308 308 + assert.Len(t, results, 1) 309 309 + assert.Equal(t, "Stumptown Coffee", results[0].Name) 310 310 + } 311 311 + 312 312 + func TestSearch_SubstringMatch(t *testing.T) { 313 313 + idx := newTestFeedIndex(t) 314 314 + 315 315 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 316 316 + "name": "Red Rooster Coffee", 317 317 + "location": "Floyd, VA", 318 318 + }) 319 319 + 320 320 + results, err := Search(idx, atproto.NSIDRoaster, "floyd", 10) 321 321 + assert.NoError(t, err) 322 322 + assert.Len(t, results, 1) 323 323 + assert.Equal(t, "Red Rooster Coffee", results[0].Name) 324 324 + } 325 325 + 326 326 + func TestSearch_Deduplication(t *testing.T) { 327 327 + idx := newTestFeedIndex(t) 328 328 + 329 329 + // Two users have the same roaster, same location (one with website, one without) 330 330 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 331 331 + "name": "Counter Culture Coffee", 332 332 + "location": "Durham, NC", 333 333 + "website": "https://counterculturecoffee.com", 334 334 + }) 335 335 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{ 336 336 + "name": "Counter Culture", 337 337 + "location": "Durham, NC", 338 338 + }) 339 339 + 340 340 + results, err := Search(idx, atproto.NSIDRoaster, "counter", 10) 341 341 + assert.NoError(t, err) 342 342 + assert.Len(t, results, 1) 343 343 + assert.Equal(t, 2, results[0].Count) 344 344 + assert.Equal(t, "Durham, NC", results[0].Fields["location"]) 345 345 + } 346 346 + 347 347 + func TestSearch_Limit(t *testing.T) { 348 348 + idx := newTestFeedIndex(t) 349 349 + 350 350 + for i := 0; i < 5; i++ { 351 351 + rkey := "r" + string(rune('0'+i)) 352 352 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDGrinder, rkey, map[string]interface{}{ 353 353 + "name": "Grinder " + string(rune('A'+i)), 354 354 + "grinderType": "hand", 355 355 + }) 356 356 + } 357 357 + 358 358 + results, err := Search(idx, atproto.NSIDGrinder, "grinder", 3) 359 359 + assert.NoError(t, err) 360 360 + assert.Len(t, results, 3) 361 361 + } 362 362 + 363 363 + func TestSearch_ShortQuery(t *testing.T) { 364 364 + idx := newTestFeedIndex(t) 365 365 + 366 366 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{ 367 367 + "name": "ABC", 368 368 + }) 369 369 + 370 370 + results, err := Search(idx, atproto.NSIDRoaster, "a", 10) 371 371 + assert.NoError(t, err) 372 372 + assert.Empty(t, results) 373 373 + 374 374 + results, err = Search(idx, atproto.NSIDRoaster, "ab", 10) 375 375 + assert.NoError(t, err) 376 376 + assert.Len(t, results, 1) 377 377 + } 378 378 + 379 379 + func TestSearch_EmptyQuery(t *testing.T) { 380 380 + idx := newTestFeedIndex(t) 381 381 + 382 382 + results, err := Search(idx, atproto.NSIDRoaster, "", 10) 383 383 + assert.NoError(t, err) 384 384 + assert.Empty(t, results) 385 385 + } 386 386 + 387 387 + func TestSearch_UnknownCollection(t *testing.T) { 388 388 + idx := newTestFeedIndex(t) 389 389 + 390 390 + results, err := Search(idx, "unknown.collection", "test", 10) 391 391 + assert.NoError(t, err) 392 392 + assert.Empty(t, results) 393 393 + } 394 394 + 395 395 + func TestSearch_GrinderFields(t *testing.T) { 396 396 + idx := newTestFeedIndex(t) 397 397 + 398 398 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDGrinder, "g1", map[string]interface{}{ 399 399 + "name": "1Zpresso JX Pro", 400 400 + "grinderType": "hand", 401 401 + "burrType": "conical", 402 402 + }) 403 403 + 404 404 + results, err := Search(idx, atproto.NSIDGrinder, "1zp", 10) 405 405 + assert.NoError(t, err) 406 406 + assert.Len(t, results, 1) 407 407 + assert.Equal(t, "hand", results[0].Fields["grinderType"]) 408 408 + assert.Equal(t, "conical", results[0].Fields["burrType"]) 409 409 + } 410 410 + 411 411 + func TestSearch_BeanFields(t *testing.T) { 412 412 + idx := newTestFeedIndex(t) 413 413 + 414 414 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDBean, "b1", map[string]interface{}{ 415 415 + "name": "Ethiopian Yirgacheffe", 416 416 + "origin": "Ethiopia", 417 417 + "roastLevel": "Light", 418 418 + "process": "Washed", 419 419 + }) 420 420 + 421 421 + results, err := Search(idx, atproto.NSIDBean, "ethiopia", 10) 422 422 + assert.NoError(t, err) 423 423 + assert.Len(t, results, 1) 424 424 + assert.Equal(t, "Ethiopian Yirgacheffe", results[0].Name) 425 425 + assert.Equal(t, "Light", results[0].Fields["roastLevel"]) 426 426 + } 427 427 + 428 428 + func TestSearch_BrewerFields(t *testing.T) { 429 429 + idx := newTestFeedIndex(t) 430 430 + 431 431 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDBrewer, "br1", map[string]interface{}{ 432 432 + "name": "Hario V60", 433 433 + "brewerType": "Pour-Over", 434 434 + }) 435 435 + 436 436 + results, err := Search(idx, atproto.NSIDBrewer, "hario", 10) 437 437 + assert.NoError(t, err) 438 438 + assert.Len(t, results, 1) 439 439 + assert.Equal(t, "Pour-Over", results[0].Fields["brewerType"]) 440 440 + } 441 441 + 442 442 + func TestSearch_SortOrder(t *testing.T) { 443 443 + idx := newTestFeedIndex(t) 444 444 + 445 445 + // "Alpha Roasters" used by 3 people (same location so they merge) 446 446 + insertRecord(t, idx, "did:plc:alice", atproto.NSIDRoaster, "r1", map[string]interface{}{"name": "Alpha Roasters"}) 447 447 + insertRecord(t, idx, "did:plc:bob", atproto.NSIDRoaster, "r2", map[string]interface{}{"name": "Alpha Roasters"}) 448 448 + insertRecord(t, idx, "did:plc:charlie", atproto.NSIDRoaster, "r3", map[string]interface{}{"name": "Alpha Roasters"}) 449 449 + 450 450 + // "Alpha Beta" used by 1 person 451 451 + insertRecord(t, idx, "did:plc:dave", atproto.NSIDRoaster, "r4", map[string]interface{}{"name": "Alpha Beta"}) 452 452 + 453 453 + results, err := Search(idx, atproto.NSIDRoaster, "alpha", 10) 454 454 + assert.NoError(t, err) 455 455 + assert.Len(t, results, 2) 456 456 + assert.Equal(t, "Alpha Roasters", results[0].Name) 457 457 + assert.Equal(t, 3, results[0].Count) 458 458 + assert.Equal(t, "Alpha Beta", results[1].Name) 459 459 + assert.Equal(t, 1, results[1].Count) 460 460 + }