danabra.mov
forked from
slices.network/slices
Highly ambitious ATProtocol AppView service and sdks
batch insert actors in chunks
chadtmiller.com
-17
api/.sqlx/query-0534a4ffc0309c583a4e9b1fe70f934f8724eb82077d2e5d246876ca8294a588.json
-17
api/.sqlx/query-0534a4ffc0309c583a4e9b1fe70f934f8724eb82077d2e5d246876ca8294a588.json
···
1
-
{
2
-
"db_name": "PostgreSQL",
3
-
"query": "INSERT INTO \"actor\" (\"did\", \"handle\", \"slice_uri\", \"indexed_at\")\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (\"did\", \"slice_uri\")\n DO UPDATE SET\n \"handle\" = EXCLUDED.\"handle\",\n \"indexed_at\" = EXCLUDED.\"indexed_at\"",
4
-
"describe": {
5
-
"columns": [],
6
-
"parameters": {
7
-
"Left": [
8
-
"Text",
9
-
"Text",
10
-
"Text",
11
-
"Text"
12
-
]
13
-
},
14
-
"nullable": []
15
-
},
16
-
"hash": "0534a4ffc0309c583a4e9b1fe70f934f8724eb82077d2e5d246876ca8294a588"
17
-
}
+17
api/.sqlx/query-e0ebe6db3bd5b69ee918b4315f4d0d359da8ab44b0244f084ed502bcde72abc6.json
+17
api/.sqlx/query-e0ebe6db3bd5b69ee918b4315f4d0d359da8ab44b0244f084ed502bcde72abc6.json
···
1
+
{
2
+
"db_name": "PostgreSQL",
3
+
"query": "INSERT INTO \"actor\" (\"did\", \"handle\", \"slice_uri\", \"indexed_at\")\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (\"did\", \"slice_uri\")\n DO UPDATE SET\n \"handle\" = EXCLUDED.\"handle\",\n \"indexed_at\" = EXCLUDED.\"indexed_at\"",
4
+
"describe": {
5
+
"columns": [],
6
+
"parameters": {
7
+
"Left": [
8
+
"Text",
9
+
"Text",
10
+
"Text",
11
+
"Text"
12
+
]
13
+
},
14
+
"nullable": []
15
+
},
16
+
"hash": "e0ebe6db3bd5b69ee918b4315f4d0d359da8ab44b0244f084ed502bcde72abc6"
17
+
}
+24
-15
api/src/database.rs
+24
-15
api/src/database.rs
···
448
448
449
449
450
450
pub async fn batch_insert_actors(&self, actors: &[Actor]) -> Result<(), DatabaseError> {
451
+
if actors.is_empty() {
452
+
return Ok(());
453
+
}
454
+
451
455
let mut tx = self.pool.begin().await?;
452
456
453
-
for actor in actors {
454
-
sqlx::query!(
455
-
r#"INSERT INTO "actor" ("did", "handle", "slice_uri", "indexed_at")
456
-
VALUES ($1, $2, $3, $4)
457
-
ON CONFLICT ("did", "slice_uri")
458
-
DO UPDATE SET
459
-
"handle" = EXCLUDED."handle",
460
-
"indexed_at" = EXCLUDED."indexed_at""#,
461
-
actor.did,
462
-
actor.handle,
463
-
actor.slice_uri,
464
-
actor.indexed_at
465
-
)
466
-
.execute(&mut *tx)
467
-
.await?;
457
+
// Process actors in chunks to avoid hitting parameter limits
458
+
const CHUNK_SIZE: usize = 1000;
459
+
460
+
for chunk in actors.chunks(CHUNK_SIZE) {
461
+
for actor in chunk {
462
+
sqlx::query!(
463
+
r#"INSERT INTO "actor" ("did", "handle", "slice_uri", "indexed_at")
464
+
VALUES ($1, $2, $3, $4)
465
+
ON CONFLICT ("did", "slice_uri")
466
+
DO UPDATE SET
467
+
"handle" = EXCLUDED."handle",
468
+
"indexed_at" = EXCLUDED."indexed_at""#,
469
+
actor.did,
470
+
actor.handle,
471
+
actor.slice_uri,
472
+
actor.indexed_at
473
+
)
474
+
.execute(&mut *tx)
475
+
.await?;
476
+
}
468
477
}
469
478
470
479
tx.commit().await?;