···18- Support for the
19 [Tap](https://github.com/bluesky-social/indigo/blob/main/cmd/tap/README.md)
20 sync and backfill utility service, via `Drinkup.Tap`.
002122### Changed
23
···18- Support for the
19 [Tap](https://github.com/bluesky-social/indigo/blob/main/cmd/tap/README.md)
20 sync and backfill utility service, via `Drinkup.Tap`.
21+- Support for [Jetstream](https://github.com/bluesky-social/jetstream), a
22+ simplified JSON event stream for ATProto, via `Drinkup.Jetstream`.
2324### Changed
25
···53 HTTP/HTTPS URL of the ATProto Firehose relay.
5455 Defaults to `"https://bsky.network"` which is the public Bluesky relay.
0056 """
57 @type host() :: String.t()
58
···53 HTTP/HTTPS URL of the ATProto Firehose relay.
5455 Defaults to `"https://bsky.network"` which is the public Bluesky relay.
56+57+ You can find a list of third-party relays at https://compare.hose.cam/.
58 """
59 @type host() :: String.t()
60
···1+defmodule Drinkup.Jetstream.Consumer do
2+ @moduledoc """
3+ Consumer behaviour for handling Jetstream events.
4+5+ Implement this behaviour to process events from a Jetstream instance.
6+ Events are dispatched asynchronously via `Task.Supervisor`.
7+8+ Unlike Tap, Jetstream does not require event acknowledgments. Events are
9+ processed in a fire-and-forget manner.
10+11+ ## Example
12+13+ defmodule MyJetstreamConsumer do
14+ @behaviour Drinkup.Jetstream.Consumer
15+16+ def handle_event(%Drinkup.Jetstream.Event.Commit{operation: :create} = event) do
17+ # Handle new record creation
18+ IO.inspect(event, label: "New record")
19+ :ok
20+ end
21+22+ def handle_event(%Drinkup.Jetstream.Event.Commit{operation: :delete} = event) do
23+ # Handle record deletion
24+ IO.inspect(event, label: "Deleted record")
25+ :ok
26+ end
27+28+ def handle_event(%Drinkup.Jetstream.Event.Identity{} = event) do
29+ # Handle identity changes
30+ IO.inspect(event, label: "Identity update")
31+ :ok
32+ end
33+34+ def handle_event(%Drinkup.Jetstream.Event.Account{active: false} = event) do
35+ # Handle account deactivation
36+ IO.inspect(event, label: "Account inactive")
37+ :ok
38+ end
39+40+ def handle_event(_event), do: :ok
41+ end
42+43+ ## Event Types
44+45+ The consumer will receive one of three event types:
46+47+ - `Drinkup.Jetstream.Event.Commit` - Repository commits (create, update, delete)
48+ - `Drinkup.Jetstream.Event.Identity` - Identity updates (handle changes, etc.)
49+ - `Drinkup.Jetstream.Event.Account` - Account status changes (active, taken down, etc.)
50+51+ ## Error Handling
52+53+ If your `handle_event/1` implementation raises an exception, it will be logged
54+ but will not affect the stream. The error is caught and logged by the event
55+ dispatcher.
56+ """
57+58+ alias Drinkup.Jetstream.Event
59+60+ @callback handle_event(Event.t()) :: any()
61+end
···1+defmodule Drinkup.Jetstream.Event.Account do
2+ @moduledoc """
3+ Struct for account events from Jetstream.
4+5+ Represents a change to an account's status on a host (e.g., PDS or Relay).
6+ The semantics of this event are that the status is at the host which emitted
7+ the event, not necessarily that at the currently active PDS.
8+9+ For example, a Relay takedown would emit a takedown with `active: false`,
10+ even if the PDS is still active.
11+ """
12+13+ use TypedStruct
14+15+ typedstruct enforce: true do
16+ @typedoc """
17+ The status of an inactive account.
18+19+ Known values from the ATProto lexicon:
20+ - `:takendown` - Account has been taken down
21+ - `:suspended` - Account is suspended
22+ - `:deleted` - Account has been deleted
23+ - `:deactivated` - Account has been deactivated by the user
24+ - `:desynchronized` - Account is out of sync
25+ - `:throttled` - Account is throttled
26+27+ The status can also be any other string value for future compatibility.
28+ """
29+ @type status() ::
30+ :takendown
31+ | :suspended
32+ | :deleted
33+ | :deactivated
34+ | :desynchronized
35+ | :throttled
36+ | String.t()
37+38+ field :did, String.t()
39+ field :time_us, integer()
40+ field :kind, :account, default: :account
41+ field :active, boolean()
42+ field :seq, integer()
43+ field :time, NaiveDateTime.t()
44+ field :status, status() | nil
45+ end
46+47+ @doc """
48+ Parses a Jetstream account payload into an Account struct.
49+50+ ## Example Payload (Active)
51+52+ %{
53+ "active" => true,
54+ "did" => "did:plc:ufbl4k27gp6kzas5glhz7fim",
55+ "seq" => 1409753013,
56+ "time" => "2024-09-05T06:11:04.870Z"
57+ }
58+59+ ## Example Payload (Inactive)
60+61+ %{
62+ "active" => false,
63+ "did" => "did:plc:abc123",
64+ "seq" => 1409753014,
65+ "time" => "2024-09-05T06:12:00.000Z",
66+ "status" => "takendown"
67+ }
68+ """
69+ @spec from(String.t(), integer(), map()) :: t()
70+ def from(
71+ did,
72+ time_us,
73+ %{
74+ "active" => active,
75+ "seq" => seq,
76+ "time" => time
77+ } = account
78+ ) do
79+ %__MODULE__{
80+ did: did,
81+ time_us: time_us,
82+ active: active,
83+ seq: seq,
84+ time: parse_datetime(time),
85+ status: parse_status(Map.get(account, "status"))
86+ }
87+ end
88+89+ @spec parse_datetime(String.t()) :: NaiveDateTime.t()
90+ defp parse_datetime(time_str) do
91+ case NaiveDateTime.from_iso8601(time_str) do
92+ {:ok, datetime} -> datetime
93+ {:error, _} -> raise "Invalid datetime format: #{time_str}"
94+ end
95+ end
96+97+ @spec parse_status(String.t() | nil) :: status() | nil
98+ defp parse_status(nil), do: nil
99+ defp parse_status("takendown"), do: :takendown
100+ defp parse_status("suspended"), do: :suspended
101+ defp parse_status("deleted"), do: :deleted
102+ defp parse_status("deactivated"), do: :deactivated
103+ defp parse_status("desynchronized"), do: :desynchronized
104+ defp parse_status("throttled"), do: :throttled
105+ defp parse_status(status) when is_binary(status), do: status
106+end
···1+defmodule Drinkup.Jetstream.Options do
2+ @moduledoc """
3+ Configuration options for Jetstream event stream connection.
4+5+ Jetstream is a simplified JSON event stream that converts the CBOR-encoded
6+ ATProto Firehose into lightweight, friendly JSON. It provides zstd compression
7+ and filtering capabilities for collections and DIDs.
8+9+ ## Options
10+11+ - `:consumer` (required) - Module implementing `Drinkup.Jetstream.Consumer` behaviour
12+ - `:name` - Unique name for this Jetstream instance in the supervision tree (default: `Drinkup.Jetstream`)
13+ - `:host` - Jetstream service URL (default: `"wss://jetstream2.us-east.bsky.network"`)
14+ - `:wanted_collections` - List of collection NSIDs or prefixes to filter (default: `[]` = all collections)
15+ - `:wanted_dids` - List of DIDs to filter (default: `[]` = all repos)
16+ - `:cursor` - Unix microseconds timestamp to resume from (default: `nil` = live-tail)
17+ - `:require_hello` - Pause replay until first options update is sent (default: `false`)
18+ - `:max_message_size_bytes` - Maximum message size to receive (default: `nil` = no limit)
19+20+ ## Example
21+22+ %{
23+ consumer: MyJetstreamConsumer,
24+ name: MyJetstream,
25+ host: "wss://jetstream2.us-east.bsky.network",
26+ wanted_collections: ["app.bsky.feed.post", "app.bsky.feed.like"],
27+ wanted_dids: ["did:plc:abc123"],
28+ cursor: 1725519626134432
29+ }
30+31+ ## Collection Filters
32+33+ The `wanted_collections` option supports:
34+ - Full NSIDs: `"app.bsky.feed.post"`
35+ - NSID prefixes: `"app.bsky.graph.*"`, `"app.bsky.*"`
36+37+ You can specify up to 100 collection filters.
38+39+ ## DID Filters
40+41+ The `wanted_dids` option accepts a list of DID strings.
42+ You can specify up to 10,000 DIDs.
43+44+ ## Compression
45+46+ Jetstream always uses zstd compression with a custom dictionary.
47+ This is handled automatically by the socket implementation.
48+ """
49+50+ use TypedStruct
51+52+ @default_host "wss://jetstream2.us-east.bsky.network"
53+54+ @typedoc """
55+ Map of configuration options accepted by `Drinkup.Jetstream.child_spec/1`.
56+ """
57+ @type options() :: %{
58+ required(:consumer) => consumer(),
59+ optional(:name) => name(),
60+ optional(:host) => host(),
61+ optional(:wanted_collections) => wanted_collections(),
62+ optional(:wanted_dids) => wanted_dids(),
63+ optional(:cursor) => cursor(),
64+ optional(:require_hello) => require_hello(),
65+ optional(:max_message_size_bytes) => max_message_size_bytes()
66+ }
67+68+ @typedoc """
69+ Module implementing the `Drinkup.Jetstream.Consumer` behaviour.
70+ """
71+ @type consumer() :: module()
72+73+ @typedoc """
74+ Unique identifier for this Jetstream instance in the supervision tree.
75+76+ Used for Registry lookups and naming child processes.
77+ """
78+ @type name() :: atom()
79+80+ @typedoc """
81+ WebSocket URL of the Jetstream service.
82+83+ Defaults to `"wss://jetstream2.us-east.bsky.network"` which is a public Bluesky instance.
84+ """
85+ @type host() :: String.t()
86+87+ @typedoc """
88+ List of collection NSIDs or NSID prefixes to filter.
89+90+ Examples:
91+ - `["app.bsky.feed.post"]` - Only posts
92+ - `["app.bsky.graph.*"]` - All graph collections
93+ - `["app.bsky.*"]` - All Bluesky app collections
94+95+ You can specify up to 100 collection filters.
96+ Defaults to `[]` (all collections).
97+ """
98+ @type wanted_collections() :: [String.t()]
99+100+ @typedoc """
101+ List of DIDs to filter events by.
102+103+ You can specify up to 10,000 DIDs.
104+ Defaults to `[]` (all repos).
105+ """
106+ @type wanted_dids() :: [String.t()]
107+108+ @typedoc """
109+ Unix microseconds timestamp to resume streaming from.
110+111+ When provided, Jetstream will replay events starting from this timestamp.
112+ Useful for resuming after a restart without missing events. The cursor is
113+ automatically tracked and updated as events are received.
114+115+ Defaults to `nil` (live-tail from current time).
116+ """
117+ @type cursor() :: pos_integer() | nil
118+119+ @typedoc """
120+ Whether to pause replay/live-tail until the first options update is sent.
121+122+ When `true`, the connection will wait for a `Drinkup.Jetstream.update_options/2`
123+ call before starting to receive events.
124+125+ Defaults to `false`.
126+ """
127+ @type require_hello() :: boolean()
128+129+ @typedoc """
130+ Maximum message size in bytes that the client would like to receive.
131+132+ Zero or `nil` means no limit. Negative values are treated as zero.
133+ Defaults to `nil` (no maximum size).
134+ """
135+ @type max_message_size_bytes() :: integer() | nil
136+137+ typedstruct do
138+ field :consumer, consumer(), enforce: true
139+ field :name, name(), default: Drinkup.Jetstream
140+ field :host, host(), default: @default_host
141+ # TODO: Add NSID prefix validation once available in atex
142+ field :wanted_collections, wanted_collections(), default: []
143+ field :wanted_dids, wanted_dids(), default: []
144+ field :cursor, cursor()
145+ field :require_hello, require_hello(), default: false
146+ field :max_message_size_bytes, max_message_size_bytes()
147+ end
148+149+ @spec from(options()) :: t()
150+ def from(%{consumer: _} = options), do: struct(__MODULE__, options)
151+end