microcosm.blue
Server tools to backfill, tail, mirror, and verify PLC logs
racing backfills
bad-example.com
+5
-3
src/bin/bundle-weekly.rs
+5
-3
src/bin/bundle-weekly.rs
···
1
-
use allegedly::{bin_init, pages_to_weeks, poll_upstream};
2
use clap::Parser;
3
use std::path::PathBuf;
4
use url::Url;
···
23
///
24
/// Must be a week-truncated unix timestamp
25
#[arg(long, env)]
26
-
start_at: Option<u64>, // TODO!!
27
}
28
29
#[tokio::main]
···
34
let mut url = args.upstream;
35
url.set_path("/export");
36
37
log::trace!("ensure weekly output directory exists");
38
std::fs::create_dir_all(&args.dir)?;
39
40
let (tx, rx) = flume::bounded(PAGE_QUEUE_SIZE);
41
42
tokio::task::spawn(async move {
43
-
if let Err(e) = poll_upstream(None /*todo*/, url, tx).await {
44
log::error!("polling failed: {e}");
45
} else {
46
log::warn!("poller finished ok (weird?)");
···
1
+
use allegedly::{Week, bin_init, pages_to_weeks, poll_upstream};
2
use clap::Parser;
3
use std::path::PathBuf;
4
use url::Url;
···
23
///
24
/// Must be a week-truncated unix timestamp
25
#[arg(long, env)]
26
+
start_at: Option<i64>,
27
}
28
29
#[tokio::main]
···
34
let mut url = args.upstream;
35
url.set_path("/export");
36
37
+
let after = args.start_at.map(|n| Week::from_n(n).into());
38
+
39
log::trace!("ensure weekly output directory exists");
40
std::fs::create_dir_all(&args.dir)?;
41
42
let (tx, rx) = flume::bounded(PAGE_QUEUE_SIZE);
43
44
tokio::task::spawn(async move {
45
+
if let Err(e) = poll_upstream(after, url, tx).await {
46
log::error!("polling failed: {e}");
47
} else {
48
log::warn!("poller finished ok (weird?)");
+38
-23
src/bin/main.rs
+38
-23
src/bin/main.rs
···
77
rx: flume::Receiver<ExportPage>,
78
mut pg_client: tokio_postgres::Client,
79
) -> Result<(), anyhow::Error> {
80
-
let upsert_did = &pg_client
81
-
.prepare(
82
-
r#"
83
-
INSERT INTO dids (did) VALUES ($1)
84
-
ON CONFLICT DO NOTHING"#,
85
-
)
86
-
.await
87
-
.unwrap();
88
89
let insert_op = &pg_client
90
.prepare(
91
r#"
92
INSERT INTO operations (did, operation, cid, nullified, "createdAt")
93
-
VALUES ($1, $2, $3, $4, $5)"#,
94
-
) // TODO: check that it hasn't changed
95
.await
96
.unwrap();
97
98
while let Ok(page) = rx.recv_async().await {
99
log::trace!("got a page...");
100
101
-
let mut tx = pg_client.transaction().await.unwrap();
102
103
// TODO: probably figure out postgres COPY IN
104
// for now just write everything into a transaction
···
122
log::error!("ayeeeee just ignoring this error for now......");
123
continue;
124
};
125
-
let client = &tx;
126
127
-
client.execute(upsert_did, &[&op.did]).await.unwrap();
128
129
-
let sp = tx.savepoint("op").await.unwrap();
130
-
if let Err(e) = sp
131
.execute(
132
insert_op,
133
&[
···
139
],
140
)
141
.await
142
-
{
143
-
if e.code() != Some(&tokio_postgres::error::SqlState::UNIQUE_VIOLATION) {
144
-
anyhow::bail!(e);
145
-
}
146
-
// TODO: assert that the row has not changed
147
-
log::warn!("ignoring dup");
148
-
} else {
149
-
sp.commit().await.unwrap();
150
}
151
}
152
153
tx.commit().await.unwrap();
···
77
rx: flume::Receiver<ExportPage>,
78
mut pg_client: tokio_postgres::Client,
79
) -> Result<(), anyhow::Error> {
80
+
// TODO: one big upsert at the end from select distinct on the other table
81
+
82
+
// let upsert_did = &pg_client
83
+
// .prepare(
84
+
// r#"
85
+
// INSERT INTO dids (did) VALUES ($1)
86
+
// ON CONFLICT DO NOTHING"#,
87
+
// )
88
+
// .await
89
+
// .unwrap();
90
91
let insert_op = &pg_client
92
.prepare(
93
r#"
94
INSERT INTO operations (did, operation, cid, nullified, "createdAt")
95
+
VALUES ($1, $2, $3, $4, $5)
96
+
ON CONFLICT (did, cid) DO UPDATE
97
+
SET nullified = excluded.nullified,
98
+
"createdAt" = excluded."createdAt"
99
+
WHERE operations.nullified = excluded.nullified
100
+
OR operations."createdAt" = excluded."createdAt""#,
101
+
) // idea: op is provable via cid, so leave it out. after did/cid (pk) that leaves nullified and createdAt
102
+
// that we want to notice changing.
103
+
// normal insert: no conflict, rows changed = 1
104
+
// conflict (exact match): where clause passes, rows changed = 1
105
+
// conflict (mismatch): where clause fails, rows changed = 0 (detect this and warn!)
106
.await
107
.unwrap();
108
109
while let Ok(page) = rx.recv_async().await {
110
log::trace!("got a page...");
111
112
+
let tx = pg_client.transaction().await.unwrap();
113
114
// TODO: probably figure out postgres COPY IN
115
// for now just write everything into a transaction
···
133
log::error!("ayeeeee just ignoring this error for now......");
134
continue;
135
};
136
+
// let client = &tx;
137
138
+
// client.execute(upsert_did, &[&op.did]).await.unwrap();
139
140
+
// let sp = tx.savepoint("op").await.unwrap();
141
+
let inserted = tx
142
.execute(
143
insert_op,
144
&[
···
150
],
151
)
152
.await
153
+
.unwrap();
154
+
if inserted != 1 {
155
+
log::warn!(
156
+
"possible log modification: {inserted} rows changed after upserting {op:?}"
157
+
);
158
}
159
+
// {
160
+
// if e.code() != Some(&tokio_postgres::error::SqlState::UNIQUE_VIOLATION) {
161
+
// anyhow::bail!(e);
162
+
// }
163
+
// // TODO: assert that the row has not changed
164
+
// log::warn!("ignoring dup");
165
+
// }
166
}
167
168
tx.commit().await.unwrap();
+15
-4
src/weekly.rs
+15
-4
src/weekly.rs
···
8
#[derive(Debug, Clone, Copy, PartialEq)]
9
pub struct Week(i64);
10
11
impl From<Dt> for Week {
12
fn from(dt: Dt) -> Self {
13
let ts = dt.timestamp();
···
35
let total_t0 = Instant::now();
36
let mut week_ops = 0;
37
let mut week_t0 = total_t0;
38
-
let mut week = 0;
39
40
while let Ok(page) = rx.recv_async().await {
41
for mut s in page.ops {
···
50
let now = Instant::now();
51
52
log::info!(
53
-
"done week {week:3 } ({:10 }): {week_ops:7 } ({:5.0 }/s) ops, {:5 }k total ({:5.0 }/s)",
54
current_week.unwrap_or(Week(0)).0,
55
(week_ops as f64) / (now - week_t0).as_secs_f64(),
56
total_ops / 1000,
···
62
current_week = Some(op_week);
63
week_ops = 0;
64
week_t0 = now;
65
-
week += 1;
66
}
67
s.push('\n'); // hack
68
log::trace!("writing: {s}");
···
76
encoder.shutdown().await?;
77
let now = Instant::now();
78
log::info!(
79
-
"done week {week:3 } ({:10 }): {week_ops:7 } ({:5.0 }/s) ops, {:5 }k total ({:5.0 }/s)",
80
current_week.unwrap_or(Week(0)).0,
81
(week_ops as f64) / (now - week_t0).as_secs_f64(),
82
total_ops / 1000,
···
8
#[derive(Debug, Clone, Copy, PartialEq)]
9
pub struct Week(i64);
10
11
+
impl Week {
12
+
pub fn from_n(n: i64) -> Self {
13
+
Self(n)
14
+
}
15
+
pub fn n_ago(&self) -> i64 {
16
+
let Self(us) = self;
17
+
let Self(cur) = chrono::Utc::now().into();
18
+
(cur - us) / WEEK_IN_SECONDS
19
+
}
20
+
}
21
+
22
impl From<Dt> for Week {
23
fn from(dt: Dt) -> Self {
24
let ts = dt.timestamp();
···
46
let total_t0 = Instant::now();
47
let mut week_ops = 0;
48
let mut week_t0 = total_t0;
49
50
while let Ok(page) = rx.recv_async().await {
51
for mut s in page.ops {
···
60
let now = Instant::now();
61
62
log::info!(
63
+
"done week {:3 } ({:10 }): {week_ops:7 } ({:5.0 }/s) ops, {:5 }k total ({:5.0 }/s)",
64
+
current_week.map(|w| -w.n_ago()).unwrap_or(0),
65
current_week.unwrap_or(Week(0)).0,
66
(week_ops as f64) / (now - week_t0).as_secs_f64(),
67
total_ops / 1000,
···
73
current_week = Some(op_week);
74
week_ops = 0;
75
week_t0 = now;
76
}
77
s.push('\n'); // hack
78
log::trace!("writing: {s}");
···
86
encoder.shutdown().await?;
87
let now = Instant::now();
88
log::info!(
89
+
"done week {:3 } ({:10 }): {week_ops:7 } ({:5.0 }/s) ops, {:5 }k total ({:5.0 }/s)",
90
+
current_week.map(|w| -w.n_ago()).unwrap_or(0),
91
current_week.unwrap_or(Week(0)).0,
92
(week_ops as f64) / (now - week_t0).as_secs_f64(),
93
total_ops / 1000,