mackuba.eu
Don't forget to lycansubscribe
added queue field to items to mark where they're from
+4
-4
app/firehose_client.rb
+4
-4
app/firehose_client.rb
···
153
153
return unless @current_user
154
154
155
155
if op.action == :create
156
-
@current_user.likes.import_from_record(op.uri, op.raw_record)
156
+
@current_user.likes.import_from_record(op.uri, op.raw_record, queue: :firehose)
157
157
elsif op.action == :delete
158
158
@current_user.likes.where(rkey: op.rkey).delete_all
159
159
end
···
163
163
return unless @current_user
164
164
165
165
if op.action == :create
166
-
@current_user.reposts.import_from_record(op.uri, op.raw_record)
166
+
@current_user.reposts.import_from_record(op.uri, op.raw_record, queue: :firehose)
167
167
elsif op.action == :delete
168
168
@current_user.reposts.where(rkey: op.rkey).delete_all
169
169
end
···
172
172
def process_post(msg, op)
173
173
if op.action == :create
174
174
if @current_user
175
-
@current_user.quotes.import_from_record(op.uri, op.raw_record)
176
-
@current_user.pins.import_from_record(op.uri, op.raw_record)
175
+
@current_user.quotes.import_from_record(op.uri, op.raw_record, queue: :firehose)
176
+
@current_user.pins.import_from_record(op.uri, op.raw_record, queue: :firehose)
177
177
end
178
178
elsif op.action == :delete
179
179
if @current_user
+5
-5
app/import_manager.rb
+5
-5
app/import_manager.rb
···
11
11
@user = user
12
12
end
13
13
14
-
def start(sets, include_pending)
14
+
def start(sets)
15
15
queued_items = []
16
16
importers = []
17
17
sets = [sets] unless sets.is_a?(Array)
···
19
19
sets.each do |set|
20
20
case set
21
21
when 'likes'
22
-
queued_items += @user.likes.pending.to_a if include_pending
22
+
queued_items += @user.likes.in_queue(:import).to_a
23
23
importers << LikesImporter.new(@user)
24
24
when 'reposts'
25
-
queued_items += @user.reposts.pending.to_a if include_pending
25
+
queued_items += @user.reposts.in_queue(:import).to_a
26
26
importers << RepostsImporter.new(@user)
27
27
when 'posts'
28
-
queued_items += @user.quotes.pending.to_a + @user.pins.pending.to_a if include_pending
28
+
queued_items += @user.quotes.in_queue(:import).to_a + @user.pins.in_queue(:import).to_a
29
29
importers << PostsImporter.new(@user)
30
30
when 'all'
31
-
queued_items += @user.all_pending_items if include_pending
31
+
queued_items += @user.all_items_in_queue(:import)
32
32
importers += [
33
33
LikesImporter.new(@user),
34
34
RepostsImporter.new(@user),
+1
-1
app/import_worker.rb
+1
-1
app/import_worker.rb
+1
-1
app/importers/likes_importer.rb
+1
-1
app/importers/likes_importer.rb
+2
-2
app/importers/posts_importer.rb
+2
-2
app/importers/posts_importer.rb
···
18
18
19
19
records.each do |record|
20
20
begin
21
-
quote = @user.quotes.import_from_record(record['uri'], record['value'])
22
-
pin = @user.pins.import_from_record(record['uri'], record['value'])
21
+
quote = @user.quotes.import_from_record(record['uri'], record['value'], queue: :import)
22
+
pin = @user.pins.import_from_record(record['uri'], record['value'], queue: :import)
23
23
24
24
if @item_queue
25
25
if quote && quote.pending?
+1
-1
app/importers/reposts_importer.rb
+1
-1
app/importers/reposts_importer.rb
···
18
18
19
19
records.each do |record|
20
20
begin
21
-
repost = @user.reposts.import_from_record(record['uri'], record['value'])
21
+
repost = @user.reposts.import_from_record(record['uri'], record['value'], queue: :import)
22
22
23
23
if repost && repost.pending? && @item_queue
24
24
@item_queue.push(repost)
+11
app/models/importable.rb
+11
app/models/importable.rb
···
8
8
9
9
included do
10
10
scope :pending, -> { where(post: nil) }
11
+
scope :in_queue, ->(q) { where(queue: q) }
12
+
13
+
enum :queue, { firehose: 0, import: 1 }
14
+
15
+
validates_presence_of :post_uri, if: -> { post_id.nil? }
16
+
validate :check_queue
11
17
12
18
def pending?
13
19
post_uri != nil
14
20
end
15
21
22
+
def check_queue
23
+
errors.add(:queue, 'must be nil if already processed') if queue && post
24
+
end
25
+
16
26
def import_item!
17
27
post_uri = AT_URI(self.post_uri)
18
28
return nil if !post_uri.is_post?
···
20
30
if post = Post.find_by_at_uri(post_uri)
21
31
self.post = post
22
32
self.post_uri = nil
33
+
self.queue = nil
23
34
end
24
35
25
36
self.save!
-2
app/models/like.rb
-2
app/models/like.rb
-2
app/models/pin.rb
-2
app/models/pin.rb
-2
app/models/quote.rb
-2
app/models/quote.rb
-2
app/models/repost.rb
-2
app/models/repost.rb
+12
-4
app/models/user.rb
+12
-4
app/models/user.rb
···
19
19
before_destroy :delete_posts_cascading
20
20
21
21
has_many :likes, foreign_key: 'actor_id', dependent: :delete_all do
22
-
def import_from_record(like_uri, record)
22
+
def import_from_record(like_uri, record, **args)
23
23
like = self.new_from_record(like_uri, record)
24
24
return nil if like.nil? || self.where(rkey: like.rkey).exists?
25
25
26
+
like.assign_attributes(args)
26
27
like.import_item!
27
28
end
28
29
end
29
30
30
31
has_many :reposts, foreign_key: 'actor_id', dependent: :delete_all do
31
-
def import_from_record(repost_uri, record)
32
+
def import_from_record(repost_uri, record, **args)
32
33
repost = self.new_from_record(repost_uri, record)
33
34
return nil if repost.nil? || self.where(rkey: repost.rkey).exists?
34
35
36
+
repost.assign_attributes(args)
35
37
repost.import_item!
36
38
end
37
39
end
38
40
39
41
has_many :quotes, foreign_key: 'actor_id', dependent: :delete_all do
40
-
def import_from_record(post_uri, record)
42
+
def import_from_record(post_uri, record, **args)
41
43
quote = self.new_from_record(post_uri, record)
42
44
return nil if quote.nil? || self.where(rkey: quote.rkey).exists?
43
45
46
+
quote.assign_attributes(args)
44
47
quote.import_item!
45
48
end
46
49
end
47
50
48
51
has_many :pins, foreign_key: 'actor_id', dependent: :delete_all do
49
-
def import_from_record(post_uri, record)
52
+
def import_from_record(post_uri, record, **args)
50
53
pin = self.new_from_record(post_uri, record)
51
54
return nil if pin.nil? || self.where(rkey: pin.rkey).exists?
52
55
56
+
pin.assign_attributes(args)
53
57
pin.import_item!
54
58
end
55
59
end
···
68
72
69
73
def all_pending_items
70
74
[:likes, :reposts, :quotes, :pins].map { |x| self.send(x).pending.to_a }.reduce(&:+)
75
+
end
76
+
77
+
def all_items_in_queue(queue)
78
+
[:likes, :reposts, :quotes, :pins].map { |x| self.send(x).in_queue(queue).to_a }.reduce(&:+)
71
79
end
72
80
73
81
def delete_posts_cascading
+5
-1
app/post_downloader.rb
+5
-1
app/post_downloader.rb
···
88
88
end
89
89
90
90
def update_item(item, post)
91
-
item.update!(post: post, post_uri: nil)
91
+
item.update!(post: post, post_uri: nil, queue: nil)
92
92
93
93
@total_count += 1
94
94
@oldest_imported = [@oldest_imported, item.time].min
···
147
147
# delete reference if the account's PDS is the old bsky.social (so it must have been deleted pre Nov 2023)
148
148
item.destroy if hostname == 'bsky.social'
149
149
end
150
+
end
151
+
152
+
if !item.destroyed?
153
+
item.update!(queue: nil)
150
154
end
151
155
end
152
156
end
+7
db/migrate/20250923014702_add_queued_field.rb
+7
db/migrate/20250923014702_add_queued_field.rb
+5
-1
db/schema.rb
+5
-1
db/schema.rb
···
10
10
#
11
11
# It's strongly recommended that you check this file into your version control system.
12
12
13
-
ActiveRecord::Schema[7.2].define(version: 2025_09_20_182018) do
13
+
ActiveRecord::Schema[7.2].define(version: 2025_09_23_014702) do
14
14
# These are extensions that must be enabled in order to support this database
15
15
enable_extension "plpgsql"
16
16
···
34
34
t.datetime "time", null: false
35
35
t.bigint "post_id"
36
36
t.string "post_uri"
37
+
t.integer "queue", limit: 2
37
38
t.index ["actor_id", "rkey"], name: "index_likes_on_actor_id_and_rkey", unique: true
38
39
t.index ["actor_id", "time", "id"], name: "index_likes_on_actor_id_and_time_and_id", order: { time: :desc, id: :desc }
39
40
end
···
45
46
t.text "pin_text", null: false
46
47
t.bigint "post_id"
47
48
t.string "post_uri"
49
+
t.integer "queue", limit: 2
48
50
t.index ["actor_id", "rkey"], name: "index_pins_on_actor_id_and_rkey", unique: true
49
51
t.index ["actor_id", "time", "id"], name: "index_pins_on_actor_id_and_time_and_id", order: { time: :desc, id: :desc }
50
52
end
···
66
68
t.text "quote_text", null: false
67
69
t.bigint "post_id"
68
70
t.string "post_uri"
71
+
t.integer "queue", limit: 2
69
72
t.index ["actor_id", "rkey"], name: "index_quotes_on_actor_id_and_rkey", unique: true
70
73
t.index ["actor_id", "time", "id"], name: "index_quotes_on_actor_id_and_time_and_id", order: { time: :desc, id: :desc }
71
74
end
···
76
79
t.datetime "time", null: false
77
80
t.bigint "post_id"
78
81
t.string "post_uri"
82
+
t.integer "queue", limit: 2
79
83
t.index ["actor_id", "rkey"], name: "index_reposts_on_actor_id_and_rkey", unique: true
80
84
t.index ["actor_id", "time", "id"], name: "index_reposts_on_actor_id_and_time_and_id", order: { time: :desc, id: :desc }
81
85
end
+1
-2
lib/tasks/import.rake
+1
-2
lib/tasks/import.rake
···
27
27
end
28
28
29
29
user = User.find_or_create_by!(did: ENV['DID'])
30
-
pending = !ENV['SKIP_PENDING']
31
30
32
31
unless ENV['COLLECTION']
33
32
raise "Required COLLECTION parameter missing"
···
42
41
exit
43
42
}
44
43
45
-
import.start(ENV['COLLECTION'], pending)
44
+
import.start(ENV['COLLECTION'])
46
45
47
46
puts "\n\n\n\n\n"
48
47
end