mackuba.eu
Don't forget to lycansubscribe
added skeleton of a firehose client from bluefeeds
+1
Gemfile
+1
Gemfile
+17
Gemfile.lock
+17
Gemfile.lock
···
25
25
minitest (>= 5.1)
26
26
securerandom (>= 0.3)
27
27
tzinfo (~> 2.0, >= 2.0.5)
28
+
base32 (0.3.4)
28
29
base58 (0.2.3)
29
30
base64 (0.3.0)
30
31
bcrypt_pbkdf (1.1.1)
···
36
37
net-sftp (>= 2.0.0)
37
38
net-ssh (>= 2.0.14)
38
39
net-ssh-gateway (>= 1.1.0)
40
+
cbor (0.5.10.1)
39
41
concurrent-ruby (1.3.5)
40
42
connection_pool (2.5.3)
41
43
date (3.4.1)
42
44
drb (2.2.3)
43
45
ed25519 (1.4.0)
44
46
erb (5.0.2)
47
+
eventmachine (1.2.7)
48
+
faye-websocket (0.12.0)
49
+
eventmachine (>= 0.12.0)
50
+
websocket-driver (>= 0.8.0)
45
51
highline (3.1.2)
46
52
reline
47
53
i18n (1.14.7)
···
111
117
sinatra-activerecord (2.0.28)
112
118
activerecord (>= 4.1)
113
119
sinatra (>= 1.0)
120
+
skyfall (0.6.0)
121
+
base32 (~> 0.3, >= 0.3.4)
122
+
base64 (~> 0.1)
123
+
cbor (~> 0.5, >= 0.5.9.6)
124
+
eventmachine (~> 1.2, >= 1.2.7)
125
+
faye-websocket (~> 0.12)
114
126
stringio (3.1.7)
115
127
tilt (2.6.1)
116
128
timeout (0.4.3)
117
129
tzinfo (2.0.6)
118
130
concurrent-ruby (~> 1.0)
131
+
websocket-driver (0.8.0)
132
+
base64
133
+
websocket-extensions (>= 0.1.0)
134
+
websocket-extensions (0.1.5)
119
135
120
136
PLATFORMS
121
137
aarch64-linux
···
143
159
rake
144
160
sinatra
145
161
sinatra-activerecord (~> 2.0)
162
+
skyfall (~> 0.6)
146
163
147
164
BUNDLED WITH
148
165
2.7.0
+135
app/firehose_client.rb
+135
app/firehose_client.rb
···
1
+
require 'skyfall'
2
+
3
+
require_relative 'init'
4
+
require_relative 'models/subscription'
5
+
6
+
class FirehoseClient
7
+
attr_accessor :start_cursor, :service
8
+
9
+
DEFAULT_RELAY = 'bsky.network'
10
+
11
+
def initialize
12
+
@env = (ENV['APP_ENV'] || ENV['RACK_ENV'] || :development).to_sym
13
+
@service = DEFAULT_RELAY
14
+
end
15
+
16
+
def start
17
+
return if @sky
18
+
19
+
log "Starting firehose process (YJIT = #{RubyVM::YJIT.enabled? ? 'on' : 'off'})"
20
+
21
+
last_cursor = load_or_init_cursor
22
+
cursor = @start_cursor || last_cursor
23
+
24
+
@sky = Skyfall::Firehose.new(@service, :subscribe_repos, cursor)
25
+
@sky.user_agent = "Lycan (https://tangled.sh/@mackuba.eu/lycan) #{@sky.version_string}"
26
+
@sky.check_heartbeat = true
27
+
28
+
@sky.on_message do |m|
29
+
start_time = Time.now
30
+
diff = start_time - @last_update
31
+
32
+
if diff > 30
33
+
log "Receiving messages again after #{sprintf('%.1f', diff)}s, starting from #{m.time.getlocal}"
34
+
end
35
+
36
+
@last_update = start_time
37
+
process_message(m)
38
+
end
39
+
40
+
@sky.on_connecting { |u| log "Connecting to #{u}..." }
41
+
@sky.on_connect {
42
+
log "Connected ✓"
43
+
44
+
@replaying = true
45
+
@last_update = Time.now
46
+
47
+
@timer ||= EM::PeriodicTimer.new(20) do
48
+
now = Time.now
49
+
diff = now - @last_update
50
+
51
+
if diff > 30
52
+
log "Timer: last update #{sprintf('%.1f', diff)}s ago"
53
+
end
54
+
end
55
+
}
56
+
57
+
@sky.on_disconnect {
58
+
log "Disconnected."
59
+
}
60
+
61
+
@sky.on_reconnect {
62
+
log "Connection lost, reconnecting..."
63
+
64
+
@timer&.cancel
65
+
@timer = nil
66
+
}
67
+
68
+
@sky.on_timeout {
69
+
log "Trying to reconnect..."
70
+
}
71
+
72
+
@sky.on_error { |e| log "ERROR: #{e.class} #{e.message}" }
73
+
74
+
@sky.connect
75
+
end
76
+
77
+
def stop
78
+
save_cursor(@sky.cursor) unless @sky.nil?
79
+
80
+
@sky&.disconnect
81
+
@sky = nil
82
+
end
83
+
84
+
def load_or_init_cursor
85
+
if sub = Subscription.find_by(service: @service)
86
+
sub.cursor
87
+
else
88
+
Subscription.create!(service: @service, cursor: 0)
89
+
nil
90
+
end
91
+
end
92
+
93
+
def save_cursor(cursor)
94
+
Subscription.where(service: @service).update_all(cursor: cursor)
95
+
end
96
+
97
+
def process_message(msg)
98
+
save_cursor(msg.seq) if msg.seq % 1000 == 0
99
+
100
+
case msg.type
101
+
when :info
102
+
log "InfoMessage: #{msg}"
103
+
when :account
104
+
process_account_event(msg)
105
+
when :commit
106
+
if @replaying
107
+
log "Replaying events since #{msg.time.getlocal} -->"
108
+
@replaying = false
109
+
end
110
+
111
+
msg.operations.each do |op|
112
+
case op.type
113
+
when :bsky_post
114
+
# ...
115
+
end
116
+
end
117
+
end
118
+
end
119
+
120
+
def process_account_event(msg)
121
+
if msg.status == :deleted
122
+
# ...
123
+
end
124
+
end
125
+
126
+
def log(text)
127
+
puts "[#{Time.now}] #{text}"
128
+
end
129
+
130
+
def inspect
131
+
vars = instance_variables - [:@timer]
132
+
values = vars.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
133
+
"#<#{self.class}:0x#{object_id} #{values}>"
134
+
end
135
+
end
+5
app/models/subscription.rb
+5
app/models/subscription.rb
+56
bin/firehose
+56
bin/firehose
···
1
+
#!/usr/bin/env ruby
2
+
3
+
$LOAD_PATH.unshift(File.expand_path('..', __dir__))
4
+
5
+
require 'bundler/setup'
6
+
require 'app/firehose_client'
7
+
8
+
$stdout.sync = true
9
+
10
+
if ENV['ARLOG'] == '1'
11
+
ActiveRecord::Base.logger = Logger.new(STDOUT)
12
+
else
13
+
ActiveRecord::Base.logger = nil
14
+
end
15
+
16
+
def print_help
17
+
puts "Usage: #{$0} [options...]"
18
+
puts "Options:"
19
+
puts " -r12345 = start from cursor 12345"
20
+
end
21
+
22
+
firehose = FirehoseClient.new
23
+
24
+
args = ARGV.dup
25
+
26
+
while arg = args.shift
27
+
case arg
28
+
when /^\-r(\d+)$/
29
+
firehose.start_cursor = $1.to_i
30
+
when '-h', '--help'
31
+
print_help
32
+
exit 0
33
+
else
34
+
puts "Unrecognized option: #{arg}"
35
+
print_help
36
+
exit 1
37
+
end
38
+
end
39
+
40
+
trap("SIGINT") {
41
+
firehose.log "Stopping..."
42
+
43
+
EM.add_timer(0) {
44
+
firehose.stop
45
+
}
46
+
}
47
+
48
+
trap("SIGTERM") {
49
+
firehose.log "Shutting down the service..."
50
+
51
+
EM.add_timer(0) {
52
+
firehose.stop
53
+
}
54
+
}
55
+
56
+
firehose.start
+8
db/migrate/20250918024627_add_subscriptions.rb
+8
db/migrate/20250918024627_add_subscriptions.rb
+6
-1
db/schema.rb
+6
-1
db/schema.rb
···
10
10
#
11
11
# It's strongly recommended that you check this file into your version control system.
12
12
13
-
ActiveRecord::Schema[7.2].define(version: 2025_09_06_233017) do
13
+
ActiveRecord::Schema[7.2].define(version: 2025_09_18_024627) do
14
14
# These are extensions that must be enabled in order to support this database
15
15
enable_extension "plpgsql"
16
16
···
73
73
t.string "post_uri"
74
74
t.index ["actor_id", "rkey"], name: "index_reposts_on_actor_id_and_rkey", unique: true
75
75
t.index ["actor_id", "time", "id"], name: "index_reposts_on_actor_id_and_time_and_id", order: { time: :desc, id: :desc }
76
+
end
77
+
78
+
create_table "subscriptions", force: :cascade do |t|
79
+
t.string "service", null: false
80
+
t.bigint "cursor", null: false
76
81
end
77
82
78
83
create_table "users", id: :serial, force: :cascade do |t|