+29 -33

README.md

··· 3 3 A small service that receives events from the AT firehose and produces them to Kafka. Supports standard JSON outputs as well as [Osprey](https://github.com/roostorg/osprey) 4 4 formatted events. 5 5 6 6 + Additionally, at-kafka supports subscribing to [Tap](https://github.com/bluesky-social/indigo/tree/main/cmd/tap) if youare attempting to perform a network backfill. 7 7 + 6 8 ## Usage 7 9 8 10 ### Docker Compose ··· 11 13 12 14 ```yaml 13 15 environment: 14 14 - ATKAFKA_RELAY_HOST: "wss://bsky.network" 15 15 - ATKAFKA_BOOTSTRAP_SERVERS: "kafka:29092" 16 16 - ATKAFKA_OUTPUT_TOPIC: "atproto-events" 17 17 - ATKAFKA_OSPREY_COMPATIBLE: "false" 18 18 - ``` 16 16 + # For relay mode 17 17 + ATKAFKA_RELAY_HOST: "wss://bsky.network" # ATProto relay to subscribe to for events 19 18 20 20 - Then start: 19 19 + # For tap mode 20 20 + ATKAFKA_TAP_HOST: "ws://localhost:2480" # Tap websocket host to subscribe to for events 21 21 + ATKAFKA_DISABLE_ACKS: false # Whether to disable sending of acks to Tap 21 22 22 22 - ```bash 23 23 - docker compose up -d # Start services 24 24 - ``` 25 25 - 26 26 - ### Docker Run 23 23 + # Kafka configuration 24 24 + ATKAFKA_BOOTSTRAP_SERVERS: "kafka:29092" # Kafka bootstrap servers, comma separated 25 25 + ATKAFKA_OUTPUT_TOPIC: "atproto-events" # The output topic for events 26 26 + ATKAFKA_OSPREY_COMPATIBLE: false # Whether to produce Osprey-compatible events 27 27 28 28 - For standard mode: 28 28 + # Match only Blacksky PDS users 29 29 + ATKAFKA_MATCHED_SERVICES: "blacksky.app" # A comma-separated list of PDSes to emit events for 30 30 + # OR ignore anyone on Bluesky PBC PDSes 31 31 + ATKAFKA_IGNORED_SERVICES: "*.bsky.network" # OR a comma-separated list of PDSes to _not_ emit events for 29 32 30 30 - ```bash 31 31 - docker run -d \ 32 32 - -e ATKAFKA_BOOTSTRAP_SERVERS=kafka:9092 \ 33 33 - -e ATKAFKA_OUTPUT_TOPIC=atproto-events \ 34 34 - -p 2112:2112 \ 35 35 - ghcr.io/haileyok/at-kafka:latest 33 33 + # Match only Teal.fm records 34 34 + ATKAFKA_MATCHED_COLLECTIONS: "fm.teal.*" # A comma-separated list of collections to emit events for 35 35 + # OR ignore all Bluesky records 36 36 + ATKAFKA_IGNORED_COLLECTIONS: "app.bsky.*" # OR a comma-separated list of collections to ignore events for 36 37 ``` 37 38 38 38 - For Osprey-compatible mode: 39 39 + Then start: 39 40 40 41 ```bash 41 41 - docker run -d \ 42 42 - -e ATKAFKA_BOOTSTRAP_SERVERS=kafka:9092 \ 43 43 - -e ATKAFKA_OUTPUT_TOPIC=atproto-events \ 44 44 - -e ATKAFKA_OSPREY_COMPATIBLE=true \ 45 45 - -p 2112:2112 \ 46 46 - ghcr.io/haileyok/at-kafka:latest 47 47 - ``` 42 42 + # For normal mode 43 43 + docker compose up -d 48 44 49 49 - ## Configuration 45 45 + # For tap mode 46 46 + docker compose -f docker-compose.tap.yml up -d 50 47 51 51 - | Flag | Environment Variable | Default | Description | 52 52 - |------|---------------------|---------|-------------| 53 53 - | `--relay-host` | `ATKAFKA_RELAY_HOST` | `wss://bsky.network` | AT Protocol relay host to connect to | 54 54 - | `--bootstrap-servers` | `ATKAFKA_BOOTSTRAP_SERVERS` | (required) | Comma-separated list of Kafka bootstrap servers | 55 55 - | `--output-topic` | `ATKAFKA_OUTPUT_TOPIC` | (required) | Kafka topic to publish events to | 56 56 - | `--osprey-compatible` | `ATKAFKA_OSPREY_COMPATIBLE` | `false` | Enable Osprey-compatible event format | 48 48 + ``` 57 49 58 50 ## Event Structure 59 51 ··· 143 135 144 136 - `atkafka_handled_events` - Total events that are received on the firehose and handled 145 137 - `atkafka_produced_events` - Total messages that are output on the bus 138 138 + - `atkafka_plc_requests` - Total number of PLC requests that were made, if applicable, and whether they were cached 139 139 + - `atkafka_api_requests` - Total number of API requests that were made, if applicable, and whether they were cached 140 140 + - `atkafka_cache_size` - The size of the PLC and API caches 141 141 + - `atkafka_acks_sent` - Total acks that were sent to Tap, if applicable

+10 -3

atkafka/atkafka.go

··· 28 28 29 29 type Server struct { 30 30 relayHost string 31 31 + tapHost string 32 32 + disableAcks bool 31 33 bootstrapServers []string 32 34 outputTopic string 33 35 ospreyCompat bool ··· 42 44 plcClient *PlcClient 43 45 apiClient *ApiClient 44 46 logger *slog.Logger 47 47 + ws *websocket.Conn 45 48 } 46 49 47 50 type ServerArgs struct { 48 51 // network params 49 49 - RelayHost string 50 50 - PlcHost string 51 51 - ApiHost string 52 52 + RelayHost string 53 53 + TapHost string 54 54 + DisableAcks bool 55 55 + PlcHost string 56 56 + ApiHost string 52 57 53 58 // for watched and ignoed services or collections, only one list may be supplied 54 59 // for both services and collections, wildcards are acceptable. for example: ··· 113 118 114 119 s := &Server{ 115 120 relayHost: args.RelayHost, 121 121 + tapHost: args.TapHost, 122 122 + disableAcks: args.DisableAcks, 116 123 plcClient: plcClient, 117 124 apiClient: apiClient, 118 125 bootstrapServers: args.BootstrapServers,

+5

atkafka/metrics.go

··· 30 30 Namespace: "atkafka", 31 31 Name: "api_requests", 32 32 }, []string{"kind", "status", "cached"}) 33 33 + 34 34 + acksSent = promauto.NewCounterVec(prometheus.CounterOpts{ 35 35 + Namespace: "atkafka", 36 36 + Name: "acks_sent", 37 37 + }, []string{"status"}) 33 38 )

+303

atkafka/tap.go

··· 1 1 + package atkafka 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "encoding/json" 6 6 + "fmt" 7 7 + "net/http" 8 8 + "net/url" 9 9 + "os" 10 10 + "os/signal" 11 11 + "strings" 12 12 + "syscall" 13 13 + "time" 14 14 + 15 15 + "github.com/araddon/dateparse" 16 16 + "github.com/bluesky-social/indigo/atproto/syntax" 17 17 + "github.com/gorilla/websocket" 18 18 + "github.com/twmb/franz-go/pkg/kgo" 19 19 + "golang.org/x/sync/semaphore" 20 20 + ) 21 21 + 22 22 + func (s *Server) RunTapMode(ctx context.Context) error { 23 23 + sema := semaphore.NewWeighted(1_000) 24 24 + 25 25 + s.logger.Info("starting tap consumer", "tap-host", s.tapHost, "bootstrap-servers", s.bootstrapServers, "output-topic", s.outputTopic) 26 26 + 27 27 + createCtx, cancel := context.WithTimeout(ctx, 10*time.Second) 28 28 + defer cancel() 29 29 + 30 30 + producerLogger := s.logger.With("component", "producer") 31 31 + kafProducer, err := NewProducer(createCtx, producerLogger, s.bootstrapServers, s.outputTopic, 32 32 + WithEnsureTopic(true), 33 33 + WithTopicPartitions(200), 34 34 + ) 35 35 + if err != nil { 36 36 + return fmt.Errorf("failed to create producer: %w", err) 37 37 + } 38 38 + defer kafProducer.Close() 39 39 + s.producer = kafProducer 40 40 + s.logger.Info("created producer") 41 41 + 42 42 + wsDialer := websocket.DefaultDialer 43 43 + u, err := url.Parse(s.tapHost) 44 44 + if err != nil { 45 45 + return fmt.Errorf("invalid tapHost: %w", err) 46 46 + } 47 47 + u.Path = "/channel" 48 48 + s.logger.Info("created dialer") 49 49 + 50 50 + wsErr := make(chan error, 1) 51 51 + shutdownWs := make(chan struct{}, 1) 52 52 + go func() { 53 53 + logger := s.logger.With("component", "websocket") 54 54 + 55 55 + logger.Info("subscribing to tap stream", "upstream", s.tapHost) 56 56 + 57 57 + conn, _, err := wsDialer.Dial(u.String(), http.Header{ 58 58 + "User-Agent": []string{"at-kafka/0.0.0"}, 59 59 + }) 60 60 + if err != nil { 61 61 + wsErr <- err 62 62 + return 63 63 + } 64 64 + s.ws = conn 65 65 + 66 66 + for { 67 67 + var evt TapEvent 68 68 + err := conn.ReadJSON(&evt) 69 69 + if err != nil { 70 70 + logger.Error("error reading json from websocket", "err", err) 71 71 + break 72 72 + } 73 73 + 74 74 + if err := sema.Acquire(ctx, 1); err != nil { 75 75 + logger.Error("error acquring sema", "err", err) 76 76 + break 77 77 + } 78 78 + 79 79 + go func() { 80 80 + defer sema.Release(1) 81 81 + s.handleTapEvent(ctx, &evt) 82 82 + }() 83 83 + } 84 84 + 85 85 + <-shutdownWs 86 86 + 87 87 + wsErr <- nil 88 88 + }() 89 89 + s.logger.Info("created tap consumer") 90 90 + 91 91 + signals := make(chan os.Signal, 1) 92 92 + signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT) 93 93 + 94 94 + select { 95 95 + case sig := <-signals: 96 96 + s.logger.Info("shutting down on signal", "signal", sig) 97 97 + case err := <-wsErr: 98 98 + if err != nil { 99 99 + s.logger.Error("websocket error", "err", err) 100 100 + } else { 101 101 + s.logger.Info("websocket shutdown unexpectedly") 102 102 + } 103 103 + } 104 104 + 105 105 + close(shutdownWs) 106 106 + 107 107 + return nil 108 108 + } 109 109 + 110 110 + func (s *Server) handleTapEvent(ctx context.Context, evt *TapEvent) error { 111 111 + logger := s.logger.With("component", "handleEvent") 112 112 + 113 113 + var collection string 114 114 + var actionName string 115 115 + 116 116 + var evtKey string 117 117 + var evtsToProduce [][]byte 118 118 + 119 119 + if evt.Record != nil { 120 120 + // key events by DID 121 121 + evtKey = evt.Record.Did 122 122 + did := evt.Record.Did 123 123 + kind := evt.Record.Action 124 124 + collection = evt.Record.Collection 125 125 + rkey := evt.Record.Rkey 126 126 + atUri := fmt.Sprintf("at://%s/%s/%s", did, collection, rkey) 127 127 + 128 128 + skip := false 129 129 + if len(s.watchedCollections) > 0 { 130 130 + skip = true 131 131 + for _, watchedCollection := range s.watchedCollections { 132 132 + if watchedCollection == collection || strings.HasPrefix(collection, watchedCollection+".") { 133 133 + skip = false 134 134 + break 135 135 + } 136 136 + } 137 137 + } else if len(s.ignoredCollections) > 0 { 138 138 + for _, ignoredCollection := range s.ignoredCollections { 139 139 + if ignoredCollection == collection || strings.HasPrefix(collection, ignoredCollection+".") { 140 140 + skip = true 141 141 + break 142 142 + } 143 143 + } 144 144 + } 145 145 + 146 146 + if skip { 147 147 + logger.Debug("skipping event based on collection", "collection", collection) 148 148 + return nil 149 149 + } 150 150 + 151 151 + actionName = "operation#" + kind 152 152 + 153 153 + handledEvents.WithLabelValues(actionName, collection).Inc() 154 154 + 155 155 + // create the formatted operation 156 156 + atkOp := AtKafkaOp{ 157 157 + Action: evt.Record.Action, 158 158 + Collection: collection, 159 159 + Rkey: rkey, 160 160 + Uri: atUri, 161 161 + Cid: evt.Record.Cid, 162 162 + Path: fmt.Sprintf("%s/%s", collection, rkey), 163 163 + } 164 164 + 165 165 + if evt.Record.Record != nil { 166 166 + atkOp.Record = *evt.Record.Record 167 167 + } 168 168 + 169 169 + kafkaEvt := AtKafkaEvent{ 170 170 + Did: did, 171 171 + Operation: &atkOp, 172 172 + } 173 173 + 174 174 + if evt.Record.Record != nil { 175 175 + timestamp, err := parseTimeFromRecord(collection, *evt.Record.Record, rkey) 176 176 + if err != nil { 177 177 + return fmt.Errorf("error getting timestamp from record: %w", err) 178 178 + } 179 179 + kafkaEvt.Timestamp = timestamp.Format(time.RFC3339Nano) 180 180 + } 181 181 + 182 182 + evtBytes, err := json.Marshal(&kafkaEvt) 183 183 + if err != nil { 184 184 + return fmt.Errorf("failed to marshal kafka event: %w", err) 185 185 + } 186 186 + 187 187 + evtsToProduce = append(evtsToProduce, evtBytes) 188 188 + } 189 189 + 190 190 + for _, evtBytes := range evtsToProduce { 191 191 + if err := s.produceAsyncTap(ctx, evtKey, evtBytes, evt.Id); err != nil { 192 192 + return err 193 193 + } 194 194 + } 195 195 + 196 196 + return nil 197 197 + } 198 198 + 199 199 + func (s *Server) produceAsyncTap(ctx context.Context, key string, msg []byte, id int64) error { 200 200 + logger := s.logger.With("name", "produceAsyncTap", "key", key, "id", id) 201 201 + callback := func(r *kgo.Record, err error) { 202 202 + status := "ok" 203 203 + ackStatus := "ok" 204 204 + 205 205 + go func() { 206 206 + producedEvents.WithLabelValues(status).Inc() 207 207 + if s.ws != nil && !s.disableAcks { 208 208 + acksSent.WithLabelValues(ackStatus).Inc() 209 209 + } 210 210 + }() 211 211 + 212 212 + if err != nil { 213 213 + logger.Error("error producing message", "err", err) 214 214 + status = "error" 215 215 + } else if s.ws != nil && !s.disableAcks { 216 216 + if err := s.ws.WriteJSON(TapAck{ 217 217 + Type: "ack", 218 218 + Id: id, 219 219 + }); err != nil { 220 220 + logger.Error("error sending ack", "err", err) 221 221 + ackStatus = "error" 222 222 + } 223 223 + } 224 224 + } 225 225 + 226 226 + if err := s.producer.ProduceAsync(ctx, key, msg, callback); err != nil { 227 227 + return fmt.Errorf("failed to produce message: %w", err) 228 228 + } 229 229 + 230 230 + return nil 231 231 + } 232 232 + 233 233 + func parseTimeFromRecord(collection string, rec map[string]any, rkey string) (*time.Time, error) { 234 234 + var rkeyTime time.Time 235 235 + if rkey != "self" { 236 236 + rt, err := syntax.ParseTID(rkey) 237 237 + if err == nil { 238 238 + rkeyTime = rt.Time() 239 239 + } 240 240 + } 241 241 + 242 242 + switch collection { 243 243 + case "app.bsky.feed.post": 244 244 + cat, ok := rec["createdAt"].(string) 245 245 + if ok { 246 246 + t, err := dateparse.ParseAny(cat) 247 247 + if err == nil { 248 248 + return &t, nil 249 249 + } 250 250 + 251 251 + if rkeyTime.IsZero() { 252 252 + return timePtr(time.Now()), nil 253 253 + } 254 254 + } 255 255 + 256 256 + return &rkeyTime, nil 257 257 + case "app.bsky.feed.repost": 258 258 + cat, ok := rec["createdAt"].(string) 259 259 + if ok { 260 260 + t, err := dateparse.ParseAny(cat) 261 261 + if err == nil { 262 262 + return &t, nil 263 263 + } 264 264 + 265 265 + if rkeyTime.IsZero() { 266 266 + return nil, fmt.Errorf("failed to get a useful timestamp from record") 267 267 + } 268 268 + } 269 269 + 270 270 + return &rkeyTime, nil 271 271 + case "app.bsky.feed.like": 272 272 + cat, ok := rec["createdAt"].(string) 273 273 + if ok { 274 274 + t, err := dateparse.ParseAny(cat) 275 275 + if err == nil { 276 276 + return &t, nil 277 277 + } 278 278 + 279 279 + if rkeyTime.IsZero() { 280 280 + return nil, fmt.Errorf("failed to get a useful timestamp from record") 281 281 + } 282 282 + } 283 283 + 284 284 + return &rkeyTime, nil 285 285 + case "app.bsky.actor.profile": 286 286 + // We can't really trust the createdat in the profile record anyway, and its very possible its missing. just use iat for this one 287 287 + return timePtr(time.Now()), nil 288 288 + case "app.bsky.feed.generator": 289 289 + if !rkeyTime.IsZero() { 290 290 + return &rkeyTime, nil 291 291 + } 292 292 + return timePtr(time.Now()), nil 293 293 + default: 294 294 + if !rkeyTime.IsZero() { 295 295 + return &rkeyTime, nil 296 296 + } 297 297 + return timePtr(time.Now()), nil 298 298 + } 299 299 + } 300 300 + 301 301 + func timePtr(t time.Time) *time.Time { 302 302 + return &t 303 303 + }

+30

atkafka/types.go

··· 65 65 AccountAge int64 `json:"accountAge"` 66 66 Profile *bsky.ActorDefs_ProfileViewDetailed `json:"profile"` 67 67 } 68 68 + 69 69 + type TapEvent struct { 70 70 + Id int64 `json:"id"` 71 71 + Type string `json:"type"` 72 72 + Record *TapEventRecord `json:"record,omitempty"` 73 73 + Identity *TapEventIdentity `json:"identity,omitempty"` 74 74 + } 75 75 + 76 76 + type TapEventRecord struct { 77 77 + Live bool `json:"live"` 78 78 + Rev string `json:"rev"` 79 79 + Did string `json:"did"` 80 80 + Collection string `json:"collection"` 81 81 + Rkey string `json:"rkey"` 82 82 + Action string `json:"action"` 83 83 + Cid string `json:"cid"` 84 84 + Record *map[string]any `json:"record,omitempty"` 85 85 + } 86 86 + 87 87 + type TapEventIdentity struct { 88 88 + Did string `json:"did"` 89 89 + Handle string `json:"handle"` 90 90 + IsActive bool `json:"isActive"` 91 91 + Status string `json:"status"` 92 92 + } 93 93 + 94 94 + type TapAck struct { 95 95 + Type string `json:"type"` 96 96 + Id int64 `json:"id"` 97 97 + }

+44

cmd/atkafka/main.go

··· 102 102 103 103 return nil 104 104 }, 105 105 + Commands: cli.Commands{ 106 106 + &cli.Command{ 107 107 + Name: "tap-mode", 108 108 + Flags: []cli.Flag{ 109 109 + &cli.StringFlag{ 110 110 + Name: "tap-host", 111 111 + Usage: "Tap host to subscribe to for events", 112 112 + Value: "ws://localhost:2480", 113 113 + EnvVars: []string{"ATKAFKA_TAP_HOST"}, 114 114 + }, 115 115 + &cli.BoolFlag{ 116 116 + Name: "disable-acks", 117 117 + Usage: "Set to `true` to disable sending of event acks to Tap. May result in the loss of events.", 118 118 + EnvVars: []string{"ATKAFKA_DISABLE_ACKS"}, 119 119 + }, 120 120 + }, 121 121 + Action: func(cmd *cli.Context) error { 122 122 + 123 123 + ctx := context.Background() 124 124 + 125 125 + telemetry.StartMetrics(cmd) 126 126 + logger := telemetry.StartLogger(cmd) 127 127 + 128 128 + s, err := atkafka.NewServer(&atkafka.ServerArgs{ 129 129 + TapHost: cmd.String("tap-host"), 130 130 + DisableAcks: cmd.Bool("disable-acks"), 131 131 + BootstrapServers: cmd.StringSlice("bootstrap-servers"), 132 132 + OutputTopic: cmd.String("output-topic"), 133 133 + WatchedCollections: cmd.StringSlice("watched-collections"), 134 134 + IgnoredCollections: cmd.StringSlice("ignored-collections"), 135 135 + Logger: logger, 136 136 + }) 137 137 + if err != nil { 138 138 + return fmt.Errorf("failed to create new server: %w", err) 139 139 + } 140 140 + 141 141 + if err := s.RunTapMode(ctx); err != nil { 142 142 + return fmt.Errorf("error running server: %w", err) 143 143 + } 144 144 + 145 145 + return nil 146 146 + }, 147 147 + }, 148 148 + }, 105 149 } 106 150 107 151 if err := app.Run(os.Args); err != nil {

+80

docker-compose.tap.yml

··· 1 1 + services: 2 2 + zookeeper: 3 3 + image: confluentinc/cp-zookeeper:7.6.0 4 4 + hostname: zookeeper 5 5 + container_name: zookeeper 6 6 + ports: 7 7 + - "2181:2181" 8 8 + environment: 9 9 + ZOOKEEPER_CLIENT_PORT: 2181 10 10 + ZOOKEEPER_TICK_TIME: 2000 11 11 + volumes: 12 12 + - zookeeper-data:/var/lib/zookeeper/data 13 13 + - zookeeper-logs:/var/lib/zookeeper/log 14 14 + 15 15 + kafka: 16 16 + image: confluentinc/cp-kafka:7.6.0 17 17 + hostname: kafka 18 18 + container_name: kafka 19 19 + depends_on: 20 20 + - zookeeper 21 21 + ports: 22 22 + - "9092:9092" 23 23 + - "9101:9101" 24 24 + environment: 25 25 + KAFKA_BROKER_ID: 1 26 26 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 27 27 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 28 28 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 29 29 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 30 30 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 31 31 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 32 32 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 33 33 + KAFKA_JMX_PORT: 9101 34 34 + KAFKA_JMX_HOSTNAME: localhost 35 35 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' 36 36 + volumes: 37 37 + - kafka-data:/var/lib/kafka/data 38 38 + 39 39 + 40 40 + tap: 41 41 + image: ghcr.io/bluesky-social/indigo/tap:latest 42 42 + hostname: tap 43 43 + container_name: tap 44 44 + depends_on: 45 45 + - kafka 46 46 + ports: 47 47 + - "2480:2480" 48 48 + environment: 49 49 + TAP_BIND: ":2480" 50 50 + TAP_FULL_NETWORK: true 51 51 + TAP_DISABLE_ACKS: false 52 52 + TAP_COLLECTION_FILTERS: "app.bsky.graph.follow" 53 53 + volumes: 54 54 + - tap-data:/data 55 55 + 56 56 + atkafka: 57 57 + build: 58 58 + context: . 59 59 + dockerfile: Dockerfile 60 60 + container_name: atkafka 61 61 + depends_on: 62 62 + - kafka 63 63 + - tap 64 64 + ports: 65 65 + # metrics port 66 66 + - "6010:6009" 67 67 + command: ["tap-mode"] 68 68 + environment: 69 69 + ATKAFKA_TAP_HOST: "ws://tap:2480" 70 70 + ATKAFKA_DISABLE_ACKS: false 71 71 + ATKAFKA_BOOTSTRAP_SERVERS: "kafka:29092" 72 72 + ATKAFKA_OUTPUT_TOPIC: "tap-events" 73 73 + 74 74 + restart: unless-stopped 75 75 + 76 76 + volumes: 77 77 + zookeeper-data: 78 78 + zookeeper-logs: 79 79 + kafka-data: 80 80 + tap-data:

+2 -1

go.mod

··· 3 3 go 1.25.4 4 4 5 5 require ( 6 6 + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de 6 7 github.com/bluesky-social/go-util v0.0.0-20251012040650-2ebbf57f5934 7 8 github.com/bluesky-social/indigo v0.0.0-20251125184450-35c1e15d2e5f 8 9 github.com/gorilla/websocket v1.5.3 ··· 11 12 github.com/prometheus/client_golang v1.23.2 12 13 github.com/twmb/franz-go v1.19.5 13 14 github.com/urfave/cli/v2 v2.25.7 15 15 + golang.org/x/sync v0.16.0 14 16 golang.org/x/time v0.6.0 15 17 ) 16 18 ··· 95 97 go.yaml.in/yaml/v2 v2.4.2 // indirect 96 98 golang.org/x/crypto v0.40.0 // indirect 97 99 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect 98 98 - golang.org/x/sync v0.16.0 // indirect 99 100 golang.org/x/sys v0.35.0 // indirect 100 101 golang.org/x/text v0.28.0 // indirect 101 102 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect

+5

go.sum

··· 3 3 github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b/go.mod h1:4+EPqMRApwwE/6yo6CxiHoSnBzjRr3jsqer7frxP8y4= 4 4 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= 5 5 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= 6 6 + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= 7 7 + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= 6 8 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 7 9 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= 8 10 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= ··· 200 202 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 201 203 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 202 204 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 205 205 + github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 203 206 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= 204 207 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 205 208 github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= ··· 249 252 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= 250 253 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= 251 254 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= 255 255 + github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 252 256 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 253 257 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 254 258 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 255 259 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 256 260 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 257 261 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 262 262 + github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= 258 263 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 259 264 github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= 260 265 github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=