mackuba.eu
Don't forget to lycansubscribe
added skeleton of a firehose client from bluefeeds
+228
-1
+1
Gemfile
+1
Gemfile
+17
Gemfile.lock
+17
Gemfile.lock
···
25
minitest (>= 5.1)
26
securerandom (>= 0.3)
27
tzinfo (~> 2.0, >= 2.0.5)
28
base58 (0.2.3)
29
base64 (0.3.0)
30
bcrypt_pbkdf (1.1.1)
···
36
net-sftp (>= 2.0.0)
37
net-ssh (>= 2.0.14)
38
net-ssh-gateway (>= 1.1.0)
39
concurrent-ruby (1.3.5)
40
connection_pool (2.5.3)
41
date (3.4.1)
42
drb (2.2.3)
43
ed25519 (1.4.0)
44
erb (5.0.2)
45
highline (3.1.2)
46
reline
47
i18n (1.14.7)
···
111
sinatra-activerecord (2.0.28)
112
activerecord (>= 4.1)
113
sinatra (>= 1.0)
114
stringio (3.1.7)
115
tilt (2.6.1)
116
timeout (0.4.3)
117
tzinfo (2.0.6)
118
concurrent-ruby (~> 1.0)
119
120
PLATFORMS
121
aarch64-linux
···
143
rake
144
sinatra
145
sinatra-activerecord (~> 2.0)
146
147
BUNDLED WITH
148
2.7.0
···
25
minitest (>= 5.1)
26
securerandom (>= 0.3)
27
tzinfo (~> 2.0, >= 2.0.5)
28
+
base32 (0.3.4)
29
base58 (0.2.3)
30
base64 (0.3.0)
31
bcrypt_pbkdf (1.1.1)
···
37
net-sftp (>= 2.0.0)
38
net-ssh (>= 2.0.14)
39
net-ssh-gateway (>= 1.1.0)
40
+
cbor (0.5.10.1)
41
concurrent-ruby (1.3.5)
42
connection_pool (2.5.3)
43
date (3.4.1)
44
drb (2.2.3)
45
ed25519 (1.4.0)
46
erb (5.0.2)
47
+
eventmachine (1.2.7)
48
+
faye-websocket (0.12.0)
49
+
eventmachine (>= 0.12.0)
50
+
websocket-driver (>= 0.8.0)
51
highline (3.1.2)
52
reline
53
i18n (1.14.7)
···
117
sinatra-activerecord (2.0.28)
118
activerecord (>= 4.1)
119
sinatra (>= 1.0)
120
+
skyfall (0.6.0)
121
+
base32 (~> 0.3, >= 0.3.4)
122
+
base64 (~> 0.1)
123
+
cbor (~> 0.5, >= 0.5.9.6)
124
+
eventmachine (~> 1.2, >= 1.2.7)
125
+
faye-websocket (~> 0.12)
126
stringio (3.1.7)
127
tilt (2.6.1)
128
timeout (0.4.3)
129
tzinfo (2.0.6)
130
concurrent-ruby (~> 1.0)
131
+
websocket-driver (0.8.0)
132
+
base64
133
+
websocket-extensions (>= 0.1.0)
134
+
websocket-extensions (0.1.5)
135
136
PLATFORMS
137
aarch64-linux
···
159
rake
160
sinatra
161
sinatra-activerecord (~> 2.0)
162
+
skyfall (~> 0.6)
163
164
BUNDLED WITH
165
2.7.0
+135
app/firehose_client.rb
+135
app/firehose_client.rb
···
···
1
+
require 'skyfall'
2
+
3
+
require_relative 'init'
4
+
require_relative 'models/subscription'
5
+
6
+
class FirehoseClient
7
+
attr_accessor :start_cursor, :service
8
+
9
+
DEFAULT_RELAY = 'bsky.network'
10
+
11
+
def initialize
12
+
@env = (ENV['APP_ENV'] || ENV['RACK_ENV'] || :development).to_sym
13
+
@service = DEFAULT_RELAY
14
+
end
15
+
16
+
def start
17
+
return if @sky
18
+
19
+
log "Starting firehose process (YJIT = #{RubyVM::YJIT.enabled? ? 'on' : 'off'})"
20
+
21
+
last_cursor = load_or_init_cursor
22
+
cursor = @start_cursor || last_cursor
23
+
24
+
@sky = Skyfall::Firehose.new(@service, :subscribe_repos, cursor)
25
+
@sky.user_agent = "Lycan (https://tangled.sh/@mackuba.eu/lycan) #{@sky.version_string}"
26
+
@sky.check_heartbeat = true
27
+
28
+
@sky.on_message do |m|
29
+
start_time = Time.now
30
+
diff = start_time - @last_update
31
+
32
+
if diff > 30
33
+
log "Receiving messages again after #{sprintf('%.1f', diff)}s, starting from #{m.time.getlocal}"
34
+
end
35
+
36
+
@last_update = start_time
37
+
process_message(m)
38
+
end
39
+
40
+
@sky.on_connecting { |u| log "Connecting to #{u}..." }
41
+
@sky.on_connect {
42
+
log "Connected ✓"
43
+
44
+
@replaying = true
45
+
@last_update = Time.now
46
+
47
+
@timer ||= EM::PeriodicTimer.new(20) do
48
+
now = Time.now
49
+
diff = now - @last_update
50
+
51
+
if diff > 30
52
+
log "Timer: last update #{sprintf('%.1f', diff)}s ago"
53
+
end
54
+
end
55
+
}
56
+
57
+
@sky.on_disconnect {
58
+
log "Disconnected."
59
+
}
60
+
61
+
@sky.on_reconnect {
62
+
log "Connection lost, reconnecting..."
63
+
64
+
@timer&.cancel
65
+
@timer = nil
66
+
}
67
+
68
+
@sky.on_timeout {
69
+
log "Trying to reconnect..."
70
+
}
71
+
72
+
@sky.on_error { |e| log "ERROR: #{e.class} #{e.message}" }
73
+
74
+
@sky.connect
75
+
end
76
+
77
+
def stop
78
+
save_cursor(@sky.cursor) unless @sky.nil?
79
+
80
+
@sky&.disconnect
81
+
@sky = nil
82
+
end
83
+
84
+
def load_or_init_cursor
85
+
if sub = Subscription.find_by(service: @service)
86
+
sub.cursor
87
+
else
88
+
Subscription.create!(service: @service, cursor: 0)
89
+
nil
90
+
end
91
+
end
92
+
93
+
def save_cursor(cursor)
94
+
Subscription.where(service: @service).update_all(cursor: cursor)
95
+
end
96
+
97
+
def process_message(msg)
98
+
save_cursor(msg.seq) if msg.seq % 1000 == 0
99
+
100
+
case msg.type
101
+
when :info
102
+
log "InfoMessage: #{msg}"
103
+
when :account
104
+
process_account_event(msg)
105
+
when :commit
106
+
if @replaying
107
+
log "Replaying events since #{msg.time.getlocal} -->"
108
+
@replaying = false
109
+
end
110
+
111
+
msg.operations.each do |op|
112
+
case op.type
113
+
when :bsky_post
114
+
# ...
115
+
end
116
+
end
117
+
end
118
+
end
119
+
120
+
def process_account_event(msg)
121
+
if msg.status == :deleted
122
+
# ...
123
+
end
124
+
end
125
+
126
+
def log(text)
127
+
puts "[#{Time.now}] #{text}"
128
+
end
129
+
130
+
def inspect
131
+
vars = instance_variables - [:@timer]
132
+
values = vars.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
133
+
"#<#{self.class}:0x#{object_id} #{values}>"
134
+
end
135
+
end
+5
app/models/subscription.rb
+5
app/models/subscription.rb
+56
bin/firehose
+56
bin/firehose
···
···
1
+
#!/usr/bin/env ruby
2
+
3
+
$LOAD_PATH.unshift(File.expand_path('..', __dir__))
4
+
5
+
require 'bundler/setup'
6
+
require 'app/firehose_client'
7
+
8
+
$stdout.sync = true
9
+
10
+
if ENV['ARLOG'] == '1'
11
+
ActiveRecord::Base.logger = Logger.new(STDOUT)
12
+
else
13
+
ActiveRecord::Base.logger = nil
14
+
end
15
+
16
+
def print_help
17
+
puts "Usage: #{$0} [options...]"
18
+
puts "Options:"
19
+
puts " -r12345 = start from cursor 12345"
20
+
end
21
+
22
+
firehose = FirehoseClient.new
23
+
24
+
args = ARGV.dup
25
+
26
+
while arg = args.shift
27
+
case arg
28
+
when /^\-r(\d+)$/
29
+
firehose.start_cursor = $1.to_i
30
+
when '-h', '--help'
31
+
print_help
32
+
exit 0
33
+
else
34
+
puts "Unrecognized option: #{arg}"
35
+
print_help
36
+
exit 1
37
+
end
38
+
end
39
+
40
+
trap("SIGINT") {
41
+
firehose.log "Stopping..."
42
+
43
+
EM.add_timer(0) {
44
+
firehose.stop
45
+
}
46
+
}
47
+
48
+
trap("SIGTERM") {
49
+
firehose.log "Shutting down the service..."
50
+
51
+
EM.add_timer(0) {
52
+
firehose.stop
53
+
}
54
+
}
55
+
56
+
firehose.start
+8
db/migrate/20250918024627_add_subscriptions.rb
+8
db/migrate/20250918024627_add_subscriptions.rb
+6
-1
db/schema.rb
+6
-1
db/schema.rb
···
10
#
11
# It's strongly recommended that you check this file into your version control system.
12
13
-
ActiveRecord::Schema[7.2].define(version: 2025_09_06_233017) do
14
# These are extensions that must be enabled in order to support this database
15
enable_extension "plpgsql"
16
···
73
t.string "post_uri"
74
t.index ["actor_id", "rkey"], name: "index_reposts_on_actor_id_and_rkey", unique: true
75
t.index ["actor_id", "time", "id"], name: "index_reposts_on_actor_id_and_time_and_id", order: { time: :desc, id: :desc }
76
end
77
78
create_table "users", id: :serial, force: :cascade do |t|
···
10
#
11
# It's strongly recommended that you check this file into your version control system.
12
13
+
ActiveRecord::Schema[7.2].define(version: 2025_09_18_024627) do
14
# These are extensions that must be enabled in order to support this database
15
enable_extension "plpgsql"
16
···
73
t.string "post_uri"
74
t.index ["actor_id", "rkey"], name: "index_reposts_on_actor_id_and_rkey", unique: true
75
t.index ["actor_id", "time", "id"], name: "index_reposts_on_actor_id_and_time_and_id", order: { time: :desc, id: :desc }
76
+
end
77
+
78
+
create_table "subscriptions", force: :cascade do |t|
79
+
t.string "service", null: false
80
+
t.bigint "cursor", null: false
81
end
82
83
create_table "users", id: :serial, force: :cascade do |t|