···11module Skyfall
22+33+ #
44+ # This module defines constants for known Bluesky record collection types, and a mapping of those
55+ # names to symbol short codes which can be used as shorthand when processing events or in
66+ # Jetstream filters.
77+ #
88+29 module Collection
310 BSKY_PROFILE = "app.bsky.actor.profile"
411 BSKY_ACTOR_STATUS = "app.bsky.actor.status"
···19262027 BSKY_NOTIF_DECLARATION = "app.bsky.notification.declaration"
2128 BSKY_CHAT_DECLARATION = "chat.bsky.actor.declaration"
2929+3030+ # Mapping of NSID collection names to symbol short codes
22312332 SHORT_CODES = {
2433 BSKY_ACTOR_STATUS => :bsky_actor_status,
···4150 BSKY_NOTIF_DECLARATION => :bsky_notif_declaration
4251 }
43525353+ # Returns a symbol short code for a given collection NSID, or `:unknown`
5454+ # if NSID is not on the list.
5555+ # @param collection [String] collection NSID
5656+ # @return [Symbol] short code or :unknown
5757+4458 def self.short_code(collection)
4559 SHORT_CODES[collection] || :unknown
4660 end
6161+6262+ # Returns a collection NSID assigned to a given short code symbol, if one is defined.
6363+ # @param code [Symbol] one of the symbols listed in {SHORT_CODES}
6464+ # @return [String, nil] assigned NSID string, or nil when code is not known
47654866 def self.from_short_code(code)
4967 SHORT_CODES.detect { |k, v| v == code }&.first
+35-1
lib/skyfall/errors.rb
···11module Skyfall
22+ #
33+ # Wrapper base class for Skyfall error classes.
44+ #
25 class Error < StandardError
36 end
4788+ #
99+ # Raised when some part of the message being decoded has invalid format.
1010+ #
511 class DecodeError < Error
612 end
7131414+ #
1515+ # Raised when the server sends a message which is formatted correctly, but written in a version
1616+ # that's not supported by this library.
1717+ #
818 class UnsupportedError < Error
919 end
10202121+ #
2222+ # Raised when {Stream#connect} is called and there's already another instance of {Stream} or its
2323+ # subclass like {Firehose} that's connected to another websocket.
2424+ #
2525+ # This is currently not supported in Skyfall, because it uses EventMachine behind the scenes, which
2626+ # runs everything on a single "reactor" thread, and there can be only one such reactor thread in
2727+ # a given process. In theory, it should be possible for two connections to run inside a single
2828+ # shared EventMachine event loop, but it would require some more coordination and it might have
2929+ # unexpected side effects - e.g. synchronous work (including I/O and network requests) done during
3030+ # processing of an event from one connection would be blocking the other connection.
3131+ #
1132 class ReactorActiveError < Error
1233 def initialize
1334 super(
···1738 end
1839 end
19404141+ #
4242+ # Raised when the server sends a message which is formatted correctly, but describes some kind of
4343+ # error condition that the server has detected.
4444+ #
2045 class SubscriptionError < Error
2121- attr_reader :error_type, :error_message
22464747+ # @return [String] a short machine-readable error code
4848+ attr_reader :error_type
4949+5050+ # @return [String] a human-readable error message
5151+ attr_reader :error_message
5252+5353+ #
5454+ # @param error_type [String] a short machine-readable error code
5555+ # @param error_message [String, nil] a human-readable error message
5656+ #
2357 def initialize(error_type, error_message = nil)
2458 @error_type = error_type
2559 @error_message = error_message
···22require 'uri'
3344module Skyfall
55+66+ #
77+ # Client of a standard AT Protocol firehose websocket.
88+ #
99+ # This is the main Skyfall class to use to connect to a CBOR-based firehose
1010+ # websocket endpoint like `subscribeRepos` (on a PDS or a relay).
1111+ #
1212+ # To connect to the firehose, you need to:
1313+ #
1414+ # * create an instance of {Firehose}, passing it the hostname/URL of the server,
1515+ # name of the endpoint (normally `:subscribe_repos`) and optionally a cursor
1616+ # * set up callbacks to be run when connecting, disconnecting, when a message
1717+ # is received etc. (you need to set at least a message handler)
1818+ # * call {#connect} to start the connection
1919+ # * handle the received messages (instances of a {Skyfall::Firehose::Message}
2020+ # subclass)
2121+ #
2222+ # @example
2323+ # client = Skyfall::Firehose.new('bsky.network', :subscribe_repos, last_cursor)
2424+ #
2525+ # client.on_message do |msg|
2626+ # next unless msg.type == :commit
2727+ #
2828+ # msg.operations.each do |op|
2929+ # if op.type == :bsky_post && op.action == :create
3030+ # puts "[#{msg.time}] #{msg.repo}: #{op.raw_record['text']}"
3131+ # end
3232+ # end
3333+ # end
3434+ #
3535+ # client.connect
3636+ #
3737+ # # You might also want to set some or all of these lifecycle callback handlers:
3838+ #
3939+ # client.on_connecting { |url| puts "Connecting to #{url}..." }
4040+ # client.on_connect { puts "Connected" }
4141+ # client.on_disconnect { puts "Disconnected" }
4242+ # client.on_reconnect { puts "Connection lost, trying to reconnect..." }
4343+ # client.on_timeout { puts "Connection stalled, triggering a reconnect..." }
4444+ # client.on_error { |e| puts "ERROR: #{e}" }
4545+ #
4646+ # @note Most of the methods of this class that you might want to use are defined in {Skyfall::Stream}.
4747+ #
4848+549 class Firehose < Stream
5050+5151+ # the main firehose endpoint on a PDS or relay
652 SUBSCRIBE_REPOS = "com.atproto.sync.subscribeRepos"
5353+5454+ # only used with moderation services (labellers)
755 SUBSCRIBE_LABELS = "com.atproto.label.subscribeLabels"
856957 NAMED_ENDPOINTS = {
···1159 :subscribe_labels => SUBSCRIBE_LABELS
1260 }
13616262+ # Current cursor (seq of the last seen message)
6363+ # @return [Integer, nil]
1464 attr_accessor :cursor
15656666+ #
6767+ # @param server [String] Address of the server to connect to.
6868+ # Expects a string with either just a hostname, or a ws:// or wss:// URL with no path.
6969+ #
7070+ # @param endpoint [Symbol, String] XRPC method name.
7171+ # Pass either a full NSID, or a symbol shorthand from {NAMED_ENDPOINTS}
7272+ #
7373+ # @param cursor [Integer, String, nil] sequence number from which to resume
7474+ #
7575+ # @raise [ArgumentError] if any of the parameters is invalid
7676+ #
7777+1678 def initialize(server, endpoint, cursor = nil)
1779 require_relative 'firehose/message'
1880 super(server)
···248625872688 protected
8989+9090+ # Returns the full URL of the websocket endpoint to connect to.
9191+ # @return [String]
27922893 def build_websocket_url
2994 @root_url + "/xrpc/" + @endpoint + (@cursor ? "?cursor=#{@cursor}" : "")
3095 end
9696+9797+ # Processes a single message received from the websocket. Passes the received data to the
9898+ # {#on_raw_message} handler, builds a {Skyfall::Firehose::Message} object, and passes it to
9999+ # the {#on_message} handler (if defined). Also updates the {#cursor} to this message's sequence
100100+ # number (note: this is skipped if {#on_message} is not set).
101101+ #
102102+ # @param msg
103103+ # {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/API/MessageEvent Faye::WebSocket::API::MessageEvent}
104104+ # @return [nil]
3110532106 def handle_message(msg)
33107 data = msg.data
+19
lib/skyfall/firehose/account_message.rb
···11require_relative '../firehose'
2233module Skyfall
44+55+ #
66+ # Firehose message sent when the status of an account changes. This can be:
77+ #
88+ # - an account being created, sending its initial state (should be active)
99+ # - an account being deactivated or suspended
1010+ # - an account being restored back to an active state from deactivation/suspension
1111+ # - an account being deleted (the status returning `:deleted`)
1212+ #
1313+414 class Firehose::AccountMessage < Firehose::Message
1515+1616+ #
1717+ # @private
1818+ # @param type_object [Hash] first decoded CBOR frame with metadata
1919+ # @param data_object [Hash] second decoded CBOR frame with payload
2020+ # @raise [DecodeError] if the message doesn't include required data
2121+ #
522 def initialize(type_object, data_object)
623 super
724 raise DecodeError.new("Missing event details") if @data_object['active'].nil?
···1027 @status = @data_object['status']&.to_sym
1128 end
12293030+ # @return [Boolean] true if the account is active, false if it's deactivated/suspended etc.
1331 def active?
1432 @active
1533 end
16343535+ # @return [Symbol, nil] for inactive accounts, specifies the exact state; nil for active accounts
1736 attr_reader :status
1837 end
1938end
+16
lib/skyfall/firehose/commit_message.rb
···44require_relative 'operation'
5566module Skyfall
77+88+ #
99+ # Firehose message which includes one or more operations on records in the repo (a record was
1010+ # created, updated or deleted). In most cases this is a single record operation.
1111+ #
1212+ # Most of the messages received from the firehose are of this type, and this is the type you
1313+ # will usually be most interested in.
1414+ #
1515+716 class Firehose::CommitMessage < Firehose::Message
1717+1818+ # @return [CID] CID (Content Identifier) of the commit
819 def commit
920 @commit ||= @data_object['commit'] && CID.from_cbor_tag(@data_object['commit'])
1021 end
11222323+ # @return [Skyfall::CarArchive] commit data in the form of a parsed CAR archive
1224 def blocks
1325 @blocks ||= CarArchive.new(@data_object['blocks'])
1426 end
15272828+ # @return [Array<Firehose::Operation>] record operations (usually one) included in the commit
1629 def operations
1730 @operations ||= @data_object['ops'].map { |op| Firehose::Operation.new(self, op) }
1831 end
19323333+ # Looks up record data assigned to a given operation in the commit's CAR archive.
3434+ # @param op [Firehose::Operation]
3535+ # @return [Hash, nil]
2036 def raw_record_for_operation(op)
2137 op.cid && blocks.section_with_cid(op.cid)
2238 end
+14
lib/skyfall/firehose/identity_message.rb
···11require_relative '../firehose'
2233module Skyfall
44+55+ #
66+ # Firehose message sent when a new DID is created or when the details of someone's DID document
77+ # are changed (usually either a handle change or a migration to a different PDS). The message
88+ # should include currently assigned handle (though the field is not required).
99+ #
1010+ # Note: the message is originally emitted from the account's PDS and is passed as is by relays,
1111+ # which means you can't fully trust that the handle is actually correctly assigned to the DID
1212+ # and verified by DNS or well-known. To confirm that, use `DID.resolve_handle` from
1313+ # [DIDKit](https://ruby.sdk.blue/didkit/).
1414+ #
1515+416 class Firehose::IdentityMessage < Firehose::Message
1717+1818+ # @return [String, nil] current handle assigned to the DID
519 def handle
620 @data_object['handle']
721 end
+28-1
lib/skyfall/firehose/info_message.rb
···11require_relative '../firehose'
2233module Skyfall
44+55+ #
66+ # An informational firehose message from the websocket service itself, unrelated to any repos.
77+ #
88+ # Currently there is only one type of message defined, `"OutdatedCursor"`, which is sent when
99+ # the client connects with a cursor that is older than the oldest event currently kept in the
1010+ # backfill buffer. This message means that you're likely missing some events that were sent
1111+ # since the last time the client was connected but which were already deleted from the buffer.
1212+ #
1313+ # Note: the {#did}, {#seq} and {#time} properties are always `nil` for `#info` messages.
1414+ #
1515+416 class Firehose::InfoMessage < Firehose::Message
55- attr_reader :name, :message
1717+1818+ # @return [String] short machine-readable code of the info message
1919+ attr_reader :name
2020+2121+ # @return [String, nil] a human-readable description
2222+ attr_reader :message
6232424+ # Message which means that the cursor passed when connecting is older than the oldest event
2525+ # currently kept in the backfill buffer, and that you've likely missed some events that have
2626+ # already been deleted
727 OUTDATED_CURSOR = "OutdatedCursor"
8282929+ #
3030+ # @private
3131+ # @param type_object [Hash] first decoded CBOR frame with metadata
3232+ # @param data_object [Hash] second decoded CBOR frame with payload
3333+ #
934 def initialize(type_object, data_object)
1035 super
1136···1338 @message = @data_object['message']
1439 end
15404141+ # @return [String] a formatted summary
1642 def to_s
1743 (@name || "InfoMessage") + (@message ? ": #{@message}" : "")
1844 end
19452046 protected
21474848+ # @return [Array<Symbol>] list of instance variables to be printed in the {#inspect} output
2249 def inspectable_variables
2350 super - [:@did, :@seq]
2451 end
+16
lib/skyfall/firehose/labels_message.rb
···22require_relative '../label'
3344module Skyfall
55+66+ #
77+ # A message which includes one or more labels (as {Skyfall::Label}). This type of message
88+ # is only sent from a `:subscribe_labels` firehose from a labeller service.
99+ #
1010+ # Note: the {#did} and {#time} properties are always `nil` for `#labels` messages.
1111+ #
1212+513 class Firehose::LabelsMessage < Firehose::Message
6141515+ # @return [Array<Skyfall::Label>] labels included in the batch
716 attr_reader :labels
8171818+ #
1919+ # @private
2020+ # @param type_object [Hash] first decoded CBOR frame with metadata
2121+ # @param data_object [Hash] second decoded CBOR frame with payload
2222+ # @raise [DecodeError] if the message doesn't include required data
2323+ #
924 def initialize(type_object, data_object)
1025 super
1126 raise DecodeError.new("Missing event details") unless @data_object['labels'].is_a?(Array)
···15301631 protected
17323333+ # @return [Array<Symbol>] list of instance variables to be printed in the {#inspect} output
1834 def inspectable_variables
1935 super - [:@did]
2036 end
+73-3
lib/skyfall/firehose/message.rb
···66require 'time'
7788module Skyfall
99+1010+ # @abstract
1111+ # Abstract base class representing a CBOR firehose message.
1212+ #
1313+ # Actual messages are returned as instances of one of the subclasses of this class,
1414+ # depending on the type of message, most commonly as {Skyfall::Firehose::CommitMessage}.
1515+ #
1616+ # The {new} method is overridden here so that it can be called with a binary data message
1717+ # from the websocket, and it parses the type from the appropriate frame and builds an
1818+ # instance of a matching subclass.
1919+ #
2020+ # You normally don't need to call this class directly, unless you're building a custom
2121+ # subclass of {Skyfall::Stream}, or reading raw data packets from the websocket through
2222+ # the {Skyfall::Stream#on_raw_message} event handler.
2323+924 class Firehose::Message
1025 using Skyfall::Extensions
1126···1732 require_relative 'sync_message'
1833 require_relative 'unknown_message'
19342020- attr_reader :type, :did, :seq
3535+ # Type of the message (e.g. `:commit`, `:identity` etc.)
3636+ # @return [Symbol]
3737+ attr_reader :type
3838+3939+ # DID of the account (repo) that the event is sent by.
4040+ # @return [String, nil]
4141+ attr_reader :did
4242+4343+ # Sequential number of the message, to be used as a cursor when reconnecting.
4444+ # @return [Integer, nil]
4545+ attr_reader :seq
4646+2147 alias repo did
22482323- # :nodoc: - consider this as semi-private API
2424- attr_reader :type_object, :data_object
4949+ # First of the two CBOR objects forming the message payload, which mostly just includes the type field.
5050+ # @api private
5151+ # @return [Hash]
5252+ attr_reader :type_object
25535454+ # Second of the two CBOR objects forming the message payload, which contains the rest of the data.
5555+ # @api private
5656+ # @return [Hash]
5757+ attr_reader :data_object
5858+5959+ #
6060+ # Parses the CBOR objects from the binary data and returns an instance of an appropriate subclass.
6161+ #
6262+ # {Skyfall::Firehose::UnknownMessage} is returned if the message type is not recognized.
6363+ #
6464+ # @param data [String] binary payload of a firehose websocket message
6565+ # @return [Skyfall::Firehose::Message]
6666+ # @raise [Skyfall::DecodeError] if the structure of the message is invalid
6767+ # @raise [Skyfall::UnsupportedError] if the message has an unknown future version
6868+ # @raise [Skyfall::SubscriptionError] if the data contains an error message from the server
6969+ #
2670 def self.new(data)
2771 type_object, data_object = decode_cbor_objects(data)
2872···4185 message
4286 end
43878888+ #
8989+ # @private
9090+ # @param type_object [Hash] first decoded CBOR frame with metadata
9191+ # @param data_object [Hash] second decoded CBOR frame with payload
9292+ #
4493 def initialize(type_object, data_object)
4594 @type_object = type_object
4695 @data_object = data_object
···5099 @seq = @data_object['seq']
51100 end
52101102102+ #
103103+ # List of operations on records included in the message. Only `#commit` messages include
104104+ # operations, but for convenience the method is declared here and returns an empty array
105105+ # in other messages.
106106+ # @return [Array<Firehose::Operation>]
107107+ #
53108 def operations
54109 []
55110 end
56111112112+ #
113113+ # @return [Boolean] true if the message is {Firehose::UnknownMessage} (of unrecognized type)
114114+ #
57115 def unknown?
58116 self.is_a?(Firehose::UnknownMessage)
59117 end
60118119119+ #
120120+ # Timestamp decoded from the message.
121121+ #
122122+ # Note: this represents the time when the message was emitted from the original PDS, which
123123+ # might differ a lot from the `created_at` time saved in the record data, e.g. if user's local
124124+ # time is set incorrectly, or if an archive of existing posts was imported from another platform.
125125+ #
126126+ # @return [Time, nil]
127127+ #
61128 def time
62129 @time ||= @data_object['time'] && Time.parse(@data_object['time'])
63130 end
64131132132+ # Returns a string with a representation of the object for debugging purposes.
133133+ # @return [String]
65134 def inspect
66135 vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
67136 "#<#{self.class}:0x#{object_id} #{vars}>"
···7013971140 protected
72141142142+ # @return [Array<Symbol>] list of instance variables to be printed in the {#inspect} output
73143 def inspectable_variables
74144 instance_variables - [:@type_object, :@data_object, :@blocks]
75145 end
+40
lib/skyfall/firehose/operation.rb
···22require_relative '../firehose'
3344module Skyfall
55+66+ #
77+ # A single record operation from a firehose commit event. An operation is a new record being
88+ # created, or an existing record modified or deleted. It includes the URI and other details of
99+ # the record in question, type of the action taken, and record data for "created" and "update"
1010+ # actions.
1111+ #
1212+ # Note: when a record is deleted, the previous record data is *not* included in the commit, only
1313+ # its URI. This means that if you're tracking records which are referencing other records, e.g.
1414+ # follow, block, or like records, you need to store information about this referencing record
1515+ # including an URI or rkey, because in case of a delete, you will not get information about which
1616+ # post was unliked or which account was unfollowed, only which like/follow record was deleted.
1717+ #
1818+ # At the moment, Skyfall doesn't parse the record data into any rich models specific for a given
1919+ # record type with a convenient API, but simply returns them as `Hash` objects (see {#raw_record}).
2020+ # In the future, a separate `#record` method might be added which returns a parsed record model.
2121+ #
2222+523 class Firehose::Operation
2424+2525+ #
2626+ # @param message [Skyfall::Firehose::Message] commit message the operation is included in
2727+ # @param json [Hash] operation data
2828+ #
629 def initialize(message, json)
730 @message = message
831 @json = json
932 end
10333434+ # @return [String] DID of the account/repository in which the operation happened
1135 def repo
1236 @message.repo
1337 end
14381539 alias did repo
16404141+ # @return [String] path part of the record URI (collection + rkey)
1742 def path
1843 @json['path']
1944 end
20454646+ # @return [Symbol] type of the operation (`:create`, `:update` or `:delete`)
2147 def action
2248 @json['action'].to_sym
2349 end
24505151+ # @return [String] record collection NSID
2552 def collection
2653 @json['path'].split('/')[0]
2754 end
28555656+ # @return [String] record rkey
2957 def rkey
3058 @json['path'].split('/')[1]
3159 end
32606161+ # @return [String] full AT URI of the record
3362 def uri
3463 "at://#{repo}/#{path}"
3564 end
36656666+ # @return [CID, nil] CID (Content Identifier) of the record (nil for delete operations)
3767 def cid
3868 @cid ||= @json['cid'] && CID.from_cbor_tag(@json['cid'])
3969 end
40707171+ # @return [Hash, nil] record data as a plain Ruby Hash (nil for delete operations)
4172 def raw_record
4273 @raw_record ||= @message.raw_record_for_operation(self)
4374 end
44757676+ # Symbol short code of the collection, like `:bsky_post`. If the collection NSID is not
7777+ # recognized, the type is `:unknown`. The full NSID is always available through the
7878+ # `#collection` property.
7979+ #
8080+ # @return [Symbol]
8181+ # @see Skyfall::Collection
8282+ #
4583 def type
4684 Collection.short_code(collection)
4785 end
48868787+ # Returns a string with a representation of the object for debugging purposes.
8888+ # @return [String]
4989 def inspect
5090 vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
5191 "#<#{self.class}:0x#{object_id} #{vars}>"
+5
lib/skyfall/firehose/unknown_message.rb
···11require_relative '../firehose'
2233module Skyfall
44+55+ #
66+ # Firehose message of an unrecognized type.
77+ #
88+49 class Firehose::UnknownMessage < Firehose::Message
510 end
611end
+81
lib/skyfall/jetstream.rb
···55require 'uri'
6677module Skyfall
88+99+ #
1010+ # Client of a Jetstream service (JSON-based firehose).
1111+ #
1212+ # This is an equivalent of {Skyfall::Firehose} for Jetstream sources, mirroring its API.
1313+ # It returns messages as instances of subclasses of {Skyfall::Jetstream::Message}, which
1414+ # are generally equivalent to the respective {Skyfall::Firehose::Message} variants as much
1515+ # as possible.
1616+ #
1717+ # To connect to a Jetstream websocket, you need to:
1818+ #
1919+ # * create an instance of Jetstream, passing it the hostname/URL of the server, and optionally
2020+ # parameters such as cursor or collection/DID filters
2121+ # * set up callbacks to be run when connecting, disconnecting, when a message is received etc.
2222+ # (you need to set at least a message handler)
2323+ # * call {#connect} to start the connection
2424+ # * handle the received messages
2525+ #
2626+ # @example
2727+ # client = Skyfall::Jetstream.new('jetstream2.us-east.bsky.network', {
2828+ # wanted_collections: 'app.bsky.feed.post',
2929+ # wanted_dids: @dids
3030+ # })
3131+ #
3232+ # client.on_message do |msg|
3333+ # next unless msg.type == :commit
3434+ #
3535+ # op = msg.operation
3636+ #
3737+ # if op.type == :bsky_post && op.action == :create
3838+ # puts "[#{msg.time}] #{msg.repo}: #{op.raw_record['text']}"
3939+ # end
4040+ # end
4141+ #
4242+ # client.connect
4343+ #
4444+ # # You might also want to set some or all of these lifecycle callback handlers:
4545+ #
4646+ # client.on_connecting { |url| puts "Connecting to #{url}..." }
4747+ # client.on_connect { puts "Connected" }
4848+ # client.on_disconnect { puts "Disconnected" }
4949+ # client.on_reconnect { puts "Connection lost, trying to reconnect..." }
5050+ # client.on_timeout { puts "Connection stalled, triggering a reconnect..." }
5151+ # client.on_error { |e| puts "ERROR: #{e}" }
5252+ #
5353+ # @note Most of the methods of this class that you might want to use are defined in {Skyfall::Stream}.
5454+ #
5555+856 class Jetstream < Stream
5757+5858+ # Current cursor (time of the last seen message)
5959+ # @return [Integer, nil]
960 attr_accessor :cursor
10616262+ #
6363+ # @param server [String] Address of the server to connect to.
6464+ # Expects a string with either just a hostname, or a ws:// or wss:// URL with no path.
6565+ # @param params [Hash] options, see below:
6666+ #
6767+ # @option params [Integer] :cursor
6868+ # cursor from which to resume
6969+ #
7070+ # @option params [Array<String>] :wanted_dids
7171+ # DID filter to pass to the server (`:wantedDids` is also accepted);
7272+ # value should be a DID string or an array of those
7373+ #
7474+ # @option params [Array<String, Symbol>] :wanted_collections
7575+ # collection filter to pass to the server (`:wantedCollections` is also accepted);
7676+ # value should be an NSID string or a symbol shorthand, or an array of those
7777+ #
7878+ # @raise [ArgumentError] if the server parameter or the options are invalid
7979+ #
1180 def initialize(server, params = {})
1281 require_relative 'jetstream/message'
1382 super(server)
···20892190 protected
22919292+ # Returns the full URL of the websocket endpoint to connect to.
9393+ # @return [String]
9494+2395 def build_websocket_url
2496 params = @cursor ? @params.merge(cursor: @cursor) : @params
2597 query = URI.encode_www_form(params)
26982799 @root_url + "/subscribe" + (query.length > 0 ? "?#{query}" : '')
28100 end
101101+102102+ # Processes a single message received from the websocket. Passes the received data to the
103103+ # {#on_raw_message} handler, builds a {Skyfall::Jetstream::Message} object, and passes it to
104104+ # the {#on_message} handler (if defined). Also updates the {#cursor} to this message's
105105+ # microsecond timestamp (note: this is skipped if {#on_message} is not set).
106106+ #
107107+ # @param msg
108108+ # {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/API/MessageEvent Faye::WebSocket::API::MessageEvent}
109109+ # @return [nil]
2911030111 def handle_message(msg)
31112 data = msg.data
+17
lib/skyfall/jetstream/account_message.rb
···22require_relative '../jetstream'
3344module Skyfall
55+66+ #
77+ # Jetstream message sent when the status of an account changes. This can be:
88+ #
99+ # - an account being created, sending its initial state (should be active)
1010+ # - an account being deactivated or suspended
1111+ # - an account being restored back to an active state from deactivation/suspension
1212+ # - an account being deleted (the status returning `:deleted`)
1313+ #
1414+515 class Jetstream::AccountMessage < Jetstream::Message
1616+1717+ #
1818+ # @param json [Hash] message JSON decoded from the websocket message
1919+ # @raise [DecodeError] if the message doesn't include required data
2020+ #
621 def initialize(json)
722 raise DecodeError.new("Missing event details") if json['account'].nil?
823 super
924 end
10252626+ # @return [Boolean] true if the account is active, false if it's deactivated/suspended etc.
1127 def active?
1228 @json['account']['active']
1329 end
14303131+ # @return [Symbol, nil] for inactive accounts, specifies the exact state; nil for active accounts
1532 def status
1633 @json['account']['status']&.to_sym
1734 end
+21
lib/skyfall/jetstream/commit_message.rb
···33require_relative 'operation'
4455module Skyfall
66+77+ #
88+ # Jetstream message which includes a single operation on a record in the repo (a record was
99+ # created, updated or deleted). Most of the messages received from Jetstream are of this type,
1010+ # and this is the type you will usually be most interested in.
1111+ #
1212+613 class Jetstream::CommitMessage < Jetstream::Message
1414+1515+ #
1616+ # @param json [Hash] message JSON decoded from the websocket message
1717+ # @raise [DecodeError] if the message doesn't include required data
1818+ #
719 def initialize(json)
820 raise DecodeError.new("Missing event details") if json['commit'].nil?
921 super
1022 end
11232424+ # Returns the record operation included in the commit.
2525+ # @return [Jetstream::Operation]
2626+ #
1227 def operation
1328 @operation ||= Jetstream::Operation.new(self, json['commit'])
1429 end
15301631 alias op operation
17323333+ # Returns record operations included in the commit. Currently a `:commit` message from
3434+ # Jetstream always includes exactly one operation, but for compatibility with
3535+ # {Skyfall::Firehose}'s API it's also returned in an array here.
3636+ #
3737+ # @return [Array<Jetstream::Operation>]
3838+ #
1839 def operations
1940 [operation]
2041 end
+18
lib/skyfall/jetstream/identity_message.rb
···22require_relative '../jetstream'
3344module Skyfall
55+66+ #
77+ # Jetstream message sent when a new DID is created or when the details of someone's DID document
88+ # are changed (usually either a handle change or a migration to a different PDS). The message
99+ # should include currently assigned handle (though the field is not required).
1010+ #
1111+ # Note: the message is originally emitted from the account's PDS and is passed as is by relays,
1212+ # which means you can't fully trust that the handle is actually correctly assigned to the DID
1313+ # and verified by DNS or well-known. To confirm that, use `DID.resolve_handle` from
1414+ # [DIDKit](https://ruby.sdk.blue/didkit/).
1515+ #
1616+517 class Jetstream::IdentityMessage < Jetstream::Message
1818+1919+ #
2020+ # @param json [Hash] message JSON decoded from the websocket message
2121+ # @raise [DecodeError] if the message doesn't include required data
2222+ #
623 def initialize(json)
724 raise DecodeError.new("Missing event details") if json['identity'].nil?
825 super
926 end
10272828+ # @return [String, nil] current handle assigned to the DID
1129 def handle
1230 @json['identity']['handle']
1331 end
+69-2
lib/skyfall/jetstream/message.rb
···44require 'time'
5566module Skyfall
77+88+ # @abstract
99+ # Abstract base class representing a Jetstream message.
1010+ #
1111+ # Actual messages are returned as instances of one of the subclasses of this class,
1212+ # depending on the type of message, most commonly as {Skyfall::Jetstream::CommitMessage}.
1313+ #
1414+ # The {new} method is overridden here so that it can be called with a JSON message from
1515+ # the websocket, and it parses the type from the JSON and builds an instance of a matching
1616+ # subclass.
1717+ #
1818+ # You normally don't need to call this class directly, unless you're building a custom
1919+ # subclass of {Skyfall::Stream} or reading raw data packets from the websocket through
2020+ # the {Skyfall::Stream#on_raw_message} event handler.
2121+722 class Jetstream::Message
823 require_relative 'account_message'
924 require_relative 'commit_message'
1025 require_relative 'identity_message'
1126 require_relative 'unknown_message'
12271313- attr_reader :did, :type, :time_us
2828+ # Type of the message (e.g. `:commit`, `:identity` etc.)
2929+ # @return [Symbol]
3030+ attr_reader :type
3131+3232+ # DID of the account (repo) that the event is sent by
3333+ # @return [String]
3434+ attr_reader :did
3535+3636+ # Server timestamp of the message (in Unix time microseconds), which serves as a cursor
3737+ # when reconnecting; an equivalent of {Skyfall::Firehose::Message#seq} in CBOR firehose
3838+ # messages.
3939+ # @return [Integer]
4040+ attr_reader :time_us
4141+1442 alias repo did
1543 alias seq time_us
16441717- # :nodoc: - consider this as semi-private API
4545+ # The raw JSON of the message as parsed from the websocket packet.
1846 attr_reader :json
19474848+ #
4949+ # Parses the JSON data from a websocket message and returns an instance of an appropriate subclass.
5050+ #
5151+ # {Skyfall::Jetstream::UnknownMessage} is returned if the message type is not recognized.
5252+ #
5353+ # @param data [String] plain text payload of a Jetstream websocket message
5454+ # @return [Skyfall::Jetstream::Message]
5555+ # @raise [DecodeError] if the message doesn't include required data
5656+ #
2057 def self.new(data)
2158 json = JSON.parse(data)
2259···3269 message
3370 end
34717272+ #
7373+ # @param json [Hash] message JSON decoded from the websocket message
7474+ #
3575 def initialize(json)
3676 @json = json
3777 @type = @json['kind'].to_sym
···3979 @time_us = @json['time_us']
4080 end
41818282+ #
8383+ # @return [Boolean] true if the message is {Jetstream::UnknownMessage} (of unrecognized type)
8484+ #
4285 def unknown?
4386 self.is_a?(Jetstream::UnknownMessage)
4487 end
45888989+ # Returns a record operation included in the message. Only `:commit` messages include
9090+ # operations, but for convenience the method is declared here and returns nil in other messages.
9191+ #
9292+ # @return [nil]
9393+ #
4694 def operation
4795 nil
4896 end
49975098 alias op operation
5199100100+ # List of operations on records included in the message. Only `:commit` messages include
101101+ # operations, but for convenience the method is declared here and returns an empty array
102102+ # in other messages.
103103+ #
104104+ # @return [Array<Jetstream::Operation>]
105105+ #
52106 def operations
53107 []
54108 end
55109110110+ #
111111+ # Timestamp decoded from the message.
112112+ #
113113+ # Note: the time is read from the {#time_us} field, which stores the event time as an integer in
114114+ # Unix time microseconds, and which is used as an equivalent of {Skyfall::Firehose::Message#seq}
115115+ # in CBOR firehose messages. This timestamp represents the time when the message was received
116116+ # and stored by Jetstream, which might differ a lot from the `created_at` time saved in the
117117+ # record data, e.g. if user's local time is set incorrectly or if an archive of existing posts
118118+ # was imported from another platform. It will also differ (usually only slightly) from the
119119+ # timestamp of the original CBOR message emitted from the PDS and passed through the relay.
120120+ #
121121+ # @return [Time]
122122+ #
56123 def time
57124 @time ||= @json['time_us'] && Time.at(@json['time_us'] / 1_000_000.0)
58125 end
+40
lib/skyfall/jetstream/operation.rb
···22require_relative '../jetstream'
3344module Skyfall
55+66+ #
77+ # A single record operation from a Jetstream commit event. An operation is a new record being
88+ # created, or an existing record modified or deleted. It includes the URI and other details of
99+ # the record in question, type of the action taken, and record data for "created" and "update"
1010+ # actions.
1111+ #
1212+ # Note: when a record is deleted, the previous record data is *not* included in the commit, only
1313+ # its URI. This means that if you're tracking records which are referencing other records, e.g.
1414+ # follow, block, or like records, you need to store information about this referencing record
1515+ # including an URI or rkey, because in case of a delete, you will not get information about which
1616+ # post was unliked or which account was unfollowed, only which like/follow record was deleted.
1717+ #
1818+ # At the moment, Skyfall doesn't parse the record data into any rich models specific for a given
1919+ # record type with a convenient API, but simply returns them as `Hash` objects (see {#raw_record}).
2020+ # In the future, a second `#record` method might be added which returns a parsed record model.
2121+ #
2222+523 class Jetstream::Operation
2424+2525+ #
2626+ # @param message [Skyfall::Jetstream::Message] commit message the operation is parsed from
2727+ # @param json [Hash] operation data
2828+ #
629 def initialize(message, json)
730 @message = message
831 @json = json
932 end
10333434+ # @return [String] DID of the account/repository in which the operation happened
1135 def repo
1236 @message.repo
1337 end
14381539 alias did repo
16404141+ # @return [String] path part of the record URI (collection + rkey)
1742 def path
1843 @json['collection'] + '/' + @json['rkey']
1944 end
20454646+ # @return [Symbol] type of the operation (`:create`, `:update` or `:delete`)
2147 def action
2248 @json['operation'].to_sym
2349 end
24505151+ # @return [String] record collection NSID
2552 def collection
2653 @json['collection']
2754 end
28555656+ # @return [String] record rkey
2957 def rkey
3058 @json['rkey']
3159 end
32606161+ # @return [String] full AT URI of the record
3362 def uri
3463 "at://#{repo}/#{collection}/#{rkey}"
3564 end
36656666+ # @return [CID, nil] CID (Content Identifier) of the record (nil for delete operations)
3767 def cid
3868 @cid ||= @json['cid'] && CID.from_json(@json['cid'])
3969 end
40707171+ # @return [Hash, nil] record data as a plain Ruby Hash (nil for delete operations)
4172 def raw_record
4273 @json['record']
4374 end
44757676+ # Symbol short code of the collection, like `:bsky_post`. If the collection NSID is not
7777+ # recognized, the type is `:unknown`. The full NSID is always available through the
7878+ # `#collection` property.
7979+ #
8080+ # @return [Symbol]
8181+ # @see Skyfall::Collection
8282+ #
4583 def type
4684 Collection.short_code(collection)
4785 end
48868787+ # Returns a string with a representation of the object for debugging purposes.
8888+ # @return [String]
4989 def inspect
5090 vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
5191 "#<#{self.class}:0x#{object_id} #{vars}>"
+5
lib/skyfall/jetstream/unknown_message.rb
···11require_relative '../jetstream'
2233module Skyfall
44+55+ #
66+ # Jetstream message of an unrecognized type.
77+ #
88+49 class Jetstream::UnknownMessage < Jetstream::Message
510 end
611end
+31
lib/skyfall/label.rb
···22require 'time'
3344module Skyfall
55+66+ #
77+ # A single label emitted from the "subscribeLabels" firehose of a labeller service.
88+ #
99+ # The label assigns some specific value - from a list of available values defined by this
1010+ # labeller - to a specific target (at:// URI or a DID). In general, this will usually be either
1111+ # a "badge" that a user requested to be assigned to themselves from a fun/informative labeller,
1212+ # or some kind of (likely negative) label assigned to a user or post by a moderation labeller.
1313+ #
1414+ # You generally don't need to create instances of this class manually, but will receive them
1515+ # from {Skyfall::Firehose} that's connected to `:subscribe_labels` in the {Stream#on_message}
1616+ # callback handler (wrapped in a {Skyfall::Firehose::LabelsMessage}).
1717+ #
1818+519 class Label
2020+2121+ # @return [Hash] the label's JSON data
622 attr_reader :data
7232424+ #
2525+ # @param data [Hash] raw label JSON
2626+ # @raise [Skyfall::DecodeError] if the data has an invalid format
2727+ # @raise [Skyfall::UnsupportedError] if the label is in an unsupported future version
2828+ #
829 def initialize(data)
930 @data = data
1031···2041 raise DecodeError.new("Invalid uri: #{uri}") unless uri.start_with?('at://') || uri.start_with?('did:')
2142 end
22434444+ # @return [Integer] label format version number
2345 def version
2446 @data['ver']
2547 end
26484949+ # DID of the labelling authority (the labeller service).
5050+ # @return [String]
2751 def authority
2852 @data['src']
2953 end
30545555+ # AT URI or DID of the labelled subject (e.g. a user or post).
5656+ # @return [String]
3157 def subject
3258 @data['uri']
3359 end
34606161+ # @return [CID, nil] CID of the specific version of the subject that this label applies to
3562 def cid
3663 @cid ||= @data['cid'] && CID.from_json(@data['cid'])
3764 end
38656666+ # @return [String] label value
3967 def value
4068 @data['val']
4169 end
42707171+ # @return [Boolean] if true, then this is a negation (delete) of an existing label
4372 def negation?
4473 !!@data['neg']
4574 end
46757676+ # @return [Time] timestamp when the label was created
4777 def created_at
4878 @created_at ||= Time.parse(@data['cts'])
4979 end
50808181+ # @return [Time, nil] optional timestamp when the label expires
5182 def expires_at
5283 @expires_at ||= @data['exp'] && Time.parse(@data['exp'])
5384 end
+140-2
lib/skyfall/stream.rb
···55require_relative 'version'
6677module Skyfall
88+99+ # Base class of a websocket client. It provides basic websocket client functionality such as
1010+ # connecting to the service, keeping the connection alive and running lifecycle callbacks.
1111+ #
1212+ # In most cases, you will not create instances of this class directly, but rather use either
1313+ # {Firehose} or {Jetstream}. Use this class as a superclass if you need to implement some
1414+ # custom client for a websocket API that isn't supported yet.
1515+816 class Stream
917 EVENTS = %w(message raw_message connecting connect disconnect reconnect error timeout)
1018 MAX_RECONNECT_INTERVAL = 300
11191212- attr_accessor :auto_reconnect, :last_update, :user_agent
1313- attr_accessor :heartbeat_timeout, :heartbeat_interval, :check_heartbeat
2020+ # If enabled, the client will try to reconnect if the connection is closed unexpectedly.
2121+ # (Default: true)
2222+ #
2323+ # When the reconnect attempt fails, it will wait with an exponential backoff delay before
2424+ # retrying again, up to {MAX_RECONNECT_INTERVAL} seconds.
2525+ #
2626+ # @return [Boolean]
2727+ attr_accessor :auto_reconnect
2828+2929+ # User agent sent in the header when connecting.
3030+ #
3131+ # Default value is {#default_user_agent} = {#version_string} `(Skyfall/x.y)`. It's recommended
3232+ # to set it or extend it with some information that indicates what service this is and who is
3333+ # running it (e.g. a Bluesky handle).
3434+ #
3535+ # @return [String]
3636+ # @example
3737+ # client.user_agent = "my.service (@my.handle) #{client.version_string}"
3838+ attr_accessor :user_agent
3939+4040+ # If enabled, runs a timer which does periodical "heatbeat checks".
4141+ #
4242+ # The heartbeat timer is started when the client connects to the service, and checks if the stream
4343+ # hasn't stalled and is still regularly sending new messages. If no messages are detected for some
4444+ # period of time, the client forces a reconnect.
4545+ #
4646+ # This is **not** enabled by default, because depending on the service you're connecting to, it
4747+ # might be normal to not receive any messages for a while.
4848+ #
4949+ # @see #heartbeat_timeout
5050+ # @see #heartbeat_interval
5151+ # @return [Boolean]
5252+ attr_accessor :check_heartbeat
5353+5454+ # Interval in seconds between heartbeat checks (default: 10). Only used if {#check_heartbeat} is set.
5555+ # @return [Numeric]
5656+ attr_accessor :heartbeat_interval
5757+5858+ # Number of seconds without messages after which reconnect is triggered (default: 300).
5959+ # Only used if {#check_heartbeat} is set.
6060+ # @return [Numeric]
6161+ attr_accessor :heartbeat_timeout
6262+6363+ # Time when the most recent message was received from the websocket.
6464+ #
6565+ # Note: this is _local time_ when the message was received; this is different from the timestamp
6666+ # of the message, which is the server time of the original source (PDS) when emitting the message,
6767+ # and different from a potential `created_at` saved in the record.
6868+ #
6969+ # @return [Time, nil]
7070+ attr_accessor :last_update
14717272+ #
7373+ # @param server [String] Address of the server to connect to.
7474+ # Expects a string with either just a hostname, or a ws:// or wss:// URL.
7575+ #
7676+ # @raise [ArgumentError] if the server parameter is invalid
7777+ #
1578 def initialize(server)
1679 @root_url = build_root_url(server)
1780···2790 @handlers[:error] = proc { |e| puts "ERROR: #{e}" }
2891 end
29929393+ #
9494+ # Opens a connection to the configured websocket.
9595+ #
9696+ # This method starts an EventMachine reactor on the current thread, and will only return
9797+ # once the connection is closed.
9898+ #
9999+ # @return [nil]
100100+ # @raise [ReactorActiveError] if another stream is already running
101101+ #
30102 def connect
31103 return if @ws
32104···86158 end
87159 end
88160161161+ #
162162+ # Forces a reconnect, closing the connection and calling {#connect} again.
163163+ # @return [nil]
164164+ #
89165 def reconnect
90166 @reconnecting = true
91167 @connection_attempts = 0
···93169 @ws ? @ws.close : connect
94170 end
95171172172+ #
173173+ # Closes the connection and stops the EventMachine reactor thread.
174174+ # @return [nil]
175175+ #
96176 def disconnect
97177 return unless EM.reactor_running?
98178···103183104184 alias close disconnect
105185186186+ #
187187+ # Default user agent sent when connecting to the service. (Currently `"#{version_string}"`)
188188+ # @return [String]
189189+ #
106190 def default_user_agent
107191 version_string
108192 end
109193194194+ #
195195+ # Skyfall version string for use in user agent strings (`"Skyfall/x.y"`).
196196+ # @return [String]
197197+ #
110198 def version_string
111199 "Skyfall/#{Skyfall::VERSION}"
112200 end
···132220 end
133221134222223223+ # Returns a string with a representation of the object for debugging purposes.
224224+ # @return [String]
135225 def inspect
136226 vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
137227 "#<#{self.class}:0x#{object_id} #{vars}>"
···140230141231 protected
142232233233+ # @note This method is designed to be overridden in subclasses.
234234+ #
235235+ # Returns the full URL of the websocket endpoint to connect to, with path and query parameters
236236+ # if needed. The base implementation simply returns the base URL passed to the initializer.
237237+ #
238238+ # Override this method in subclasses to point to the specific endpoint and add necessary
239239+ # parameters like cursor or filters, depending on the arguments passed to the constructor.
240240+ #
241241+ # @return [String]
242242+143243 def build_websocket_url
144244 @root_url
145245 end
146246247247+ # Builds and configures a websocket client object that is used to connect to the requested service.
248248+ #
249249+ # @return [Faye::WebSocket::Client]
250250+ # see {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/Client Faye::WebSocket::Client}
251251+147252 def build_websocket_client(url)
148253 Faye::WebSocket::Client.new(url, nil, { headers: { 'User-Agent' => user_agent }.merge(request_headers) })
149254 end
150255256256+ # @note This method is designed to be overridden in subclasses.
257257+ #
258258+ # Processes a single message received from the websocket. The implementation is expected to
259259+ # parse the message from a plain text or binary form, build an appropriate message object,
260260+ # and call the `:message` and/or `:raw_message` callback handlers, passing the right parameters.
261261+ #
262262+ # The base implementation simply takes the message data and passes it as is to `:raw_message`,
263263+ # and does not call `:message` at all.
264264+ #
265265+ # @param msg
266266+ # {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/API/MessageEvent Faye::WebSocket::API::MessageEvent}
267267+ # @return [nil]
268268+151269 def handle_message(msg)
152270 data = msg.data
153271 @handlers[:raw_message]&.call(data)
154272 end
155273274274+ # Additional headers to pass with the request when connecting to the websocket endpoint.
275275+ # The user agent header (built from {#user_agent}) is added separately.
276276+ #
277277+ # The base implementation returns an empty hash.
278278+ #
279279+ # @return [Hash] a hash of `{ header_name => header_value }`
280280+156281 def request_headers
157282 {}
158283 end
159284285285+ # Returns the underlying websocket client object. It can be used e.g. to send messages back
286286+ # to the server (but see also: {#send_data}).
287287+ #
288288+ # @return [Faye::WebSocket::Client]
289289+ # see {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/Client Faye::WebSocket::Client}
290290+160291 def socket
161292 @ws
162293 end
163294295295+ # Sends a message back to the server.
296296+ #
297297+ # @param data [String, Array] the message to send -
298298+ # a string for text websockets, a binary string or byte array for binary websockets
299299+ # @return [Boolean] true if the message was sent successfully
300300+164301 def send_data(data)
165302 @ws.send(data)
166303 end
167304305305+ # @return [Array<Symbol>] list of instance variables to be printed in the {#inspect} output
168306 def inspectable_variables
169307 instance_variables - [:@handlers, :@ws]
170308 end