microcosm.blue
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
Address feedback from fig
Most importantly we transformed the composite cursor into an `Option<(u64, u64)>`. This removes a couple of invalid cursor combinations automatically: if the composite cursor is provided it can be reasonable expected that both parts are present and valid (especially considering that we're the ones building the cursor in the first place). When this is not the case we bail early.
In addition, we removed the redundant base64 dependency (the `OpaqueApiCursor` handles the url-safe hex-encoded representation, and addressed some minor instances where we could have written some parts in a more idiomatic way.
No tests were changed and all tests still pass.
authored by
-1
Cargo.lock
-1
Cargo.lock
-1
constellation/Cargo.toml
-1
constellation/Cargo.toml
+7
-8
constellation/src/server/mod.rs
+7
-8
constellation/src/server/mod.rs
···
725
725
return Err(http::StatusCode::BAD_REQUEST);
726
726
}
727
727
728
-
let filter_dids: HashSet<Did> = HashSet::from_iter(
729
-
query
730
-
.did
731
-
.iter()
732
-
.map(|d| d.trim())
733
-
.filter(|d| !d.is_empty())
734
-
.map(|d| Did(d.to_string())),
735
-
);
728
+
let filter_dids: HashSet<Did> = query
729
+
.did
730
+
.iter()
731
+
.map(|d| d.trim())
732
+
.filter(|d| !d.is_empty())
733
+
.map(Did::from)
734
+
.collect();
736
735
737
736
let filter_other_subjects: HashSet<String> = HashSet::from_iter(
738
737
query
+27
-30
constellation/src/storage/mem_store.rs
+27
-30
constellation/src/storage/mem_store.rs
···
1
1
use super::{
2
2
LinkReader, LinkStorage, Order, PagedAppendingCollection, PagedOrderedCollection, StorageStats,
3
3
};
4
+
use crate::storage::CompositeCursor;
4
5
use crate::{ActionableEvent, CountsByCount, Did, RecordId};
5
6
6
7
use anyhow::{anyhow, Result};
7
-
use base64::engine::general_purpose as b64;
8
-
use base64::Engine as _;
9
8
use links::CollectedLink;
10
9
11
10
use std::collections::{HashMap, HashSet};
···
256
255
HashSet::from_iter(filter_targets.iter().map(|s| Target::new(s)));
257
256
258
257
// extract parts form composite cursor
259
-
let (backward_idx, forward_idx) = match after {
258
+
let cursor = match after {
260
259
Some(a) => {
261
-
let after_str = String::from_utf8(b64::URL_SAFE.decode(a)?)?;
262
-
let (b, f) = after_str
263
-
.split_once(',')
264
-
.ok_or_else(|| anyhow!("invalid cursor format"))?;
265
-
(
266
-
(!b.is_empty()).then(|| b.parse::<u64>()).transpose()?,
267
-
(!f.is_empty()).then(|| f.parse::<u64>()).transpose()?,
268
-
)
260
+
let (b, f) = a.split_once(',').ok_or(anyhow!("invalid cursor format"))?;
261
+
let b = b
262
+
.parse::<u64>()
263
+
.map_err(|e| anyhow!("invalid cursor.0: {e}"))?;
264
+
let f = f
265
+
.parse::<u64>()
266
+
.map_err(|e| anyhow!("invalid cursor.1: {e}"))?;
267
+
Some(CompositeCursor {
268
+
backward: b,
269
+
forward: f,
270
+
})
269
271
}
270
-
None => (None, None),
272
+
None => None,
271
273
};
272
274
273
275
let data = self.0.lock().unwrap();
···
285
287
.iter()
286
288
.enumerate()
287
289
.filter_map(|(i, opt)| opt.as_ref().map(|v| (i, v)))
288
-
.skip_while(|(linker_idx, _)| {
289
-
backward_idx.is_some_and(|idx| match forward_idx {
290
-
Some(_) => *linker_idx < idx as usize, // inclusive: depend on link idx for skipping
291
-
None => *linker_idx <= idx as usize, // exclusive: skip right here
292
-
})
293
-
})
290
+
.skip_while(|(linker_idx, _)| cursor.is_some_and(|c| *linker_idx < c.backward as usize))
294
291
.filter(|(_, (did, _))| filter_dids.is_empty() || filter_dids.contains(&did))
295
292
{
296
293
let Some(links) = data.links.get(&did).and_then(|m| {
···
310
307
*p == path_to_other && (filter_targets.is_empty() || filter_targets.contains(t))
311
308
})
312
309
.skip_while(|(link_idx, _)| {
313
-
backward_idx.is_some_and(|bl_idx| {
314
-
linker_idx == bl_idx as usize
315
-
&& forward_idx.is_some_and(|fwd_idx| *link_idx <= fwd_idx as usize)
310
+
cursor.is_some_and(|c| {
311
+
linker_idx == c.backward as usize && *link_idx <= c.forward as usize
316
312
})
317
313
})
318
314
.take(limit as usize + 1 - items.len())
···
335
331
}
336
332
}
337
333
338
-
let next = if items.len() as u64 > limit {
339
-
items.truncate(limit as usize);
340
-
items
341
-
.last()
342
-
.map(|(l, f, _, _)| b64::URL_SAFE.encode(format!("{},{}", *l as u64, *f as u64)))
343
-
} else {
344
-
None
345
-
};
334
+
let next = (items.len() > limit as usize).then(|| {
335
+
let (l, f, _, _) = items[limit as usize - 1];
336
+
format!("{l},{f}")
337
+
});
338
+
339
+
let items = items
340
+
.into_iter()
341
+
.take(limit as usize)
342
+
.map(|(_, _, rid, t)| (rid, t))
343
+
.collect();
346
344
347
-
let items = items.into_iter().map(|(_, _, rid, t)| (rid, t)).collect();
348
345
Ok(PagedOrderedCollection { items, next })
349
346
}
350
347
+7
constellation/src/storage/mod.rs
+7
constellation/src/storage/mod.rs
···
39
39
}
40
40
}
41
41
42
+
// get-many-to-many composite cursor
43
+
#[derive(Copy, Clone, Debug)]
44
+
struct CompositeCursor {
45
+
backward: u64,
46
+
forward: u64,
47
+
}
48
+
42
49
/// A paged collection whose keys are sorted instead of indexed
43
50
///
44
51
/// this has weaker guarantees than PagedAppendingCollection: it might
+35
-45
constellation/src/storage/rocks_store.rs
+35
-45
constellation/src/storage/rocks_store.rs
···
2
2
ActionableEvent, LinkReader, LinkStorage, Order, PagedAppendingCollection,
3
3
PagedOrderedCollection, StorageStats,
4
4
};
5
+
use crate::storage::CompositeCursor;
5
6
use crate::{CountsByCount, Did, RecordId};
6
7
7
8
use anyhow::{anyhow, bail, Result};
8
-
use base64::engine::general_purpose as b64;
9
-
use base64::Engine as _;
10
9
use bincode::Options as BincodeOptions;
11
10
use links::CollectedLink;
12
11
use metrics::{counter, describe_counter, describe_histogram, histogram, Unit};
···
1140
1139
filter_to_targets: &HashSet<String>,
1141
1140
) -> Result<PagedOrderedCollection<(RecordId, String), String>> {
1142
1141
// helper to resolve dids
1143
-
let resolve_active_did = |did_id: &DidId| -> Option<Did> {
1144
-
let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0).ok()? else {
1142
+
let resolve_active_did = |did_id: &DidId| -> Result<Option<Did>> {
1143
+
let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0)? else {
1145
1144
eprintln!("failed to look up did from did_id {did_id:?}");
1146
-
return None;
1145
+
return Ok(None);
1147
1146
};
1148
-
let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did).ok()?
1149
-
else {
1147
+
let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did)? else {
1150
1148
eprintln!("failed to look up did_value from did_id {did_id:?}: {did:?}: data consistency bug?");
1151
-
return None;
1149
+
return Ok(None);
1152
1150
};
1153
-
active.then_some(did)
1151
+
Ok(active.then_some(did))
1154
1152
};
1155
1153
1156
1154
// setup variables that we need later
···
1158
1156
let path = RPath(path.to_string());
1159
1157
1160
1158
// extract parts form composite cursor
1161
-
let (backward_idx, forward_idx) = match after {
1159
+
let cursor = match after {
1162
1160
Some(a) => {
1163
-
eprintln!("a: {:#?}", a);
1164
-
1165
-
let after_str = String::from_utf8(b64::URL_SAFE.decode(a)?)?;
1166
-
1167
-
eprintln!("after_str: {:#?}", after_str);
1168
-
let (b, f) = after_str
1169
-
.split_once(',')
1170
-
.ok_or_else(|| anyhow!("invalid cursor format"))?;
1171
-
(
1172
-
(!b.is_empty()).then(|| b.parse::<u64>()).transpose()?,
1173
-
(!f.is_empty()).then(|| f.parse::<u64>()).transpose()?,
1174
-
)
1161
+
let (b, f) = a.split_once(',').ok_or(anyhow!("invalid cursor format"))?;
1162
+
let b = b
1163
+
.parse::<u64>()
1164
+
.map_err(|e| anyhow!("invalid cursor.0: {e}"))?;
1165
+
let f = f
1166
+
.parse::<u64>()
1167
+
.map_err(|e| anyhow!("invalid cursor.1: {e}"))?;
1168
+
Some(CompositeCursor {
1169
+
backward: b,
1170
+
forward: f,
1171
+
})
1175
1172
}
1176
-
None => (None, None),
1173
+
None => None,
1177
1174
};
1178
1175
1179
-
eprintln!("backward_idx: {:#?}", backward_idx);
1180
-
eprintln!("forward_idx: {:#?}", forward_idx);
1176
+
eprintln!("cursor: {:#?}", cursor);
1181
1177
1182
1178
// (__active__) did ids and filter targets
1183
1179
let filter_did_ids: HashMap<DidId, bool> = filter_dids
···
1206
1202
// iterate backwards (who linked to the target?)
1207
1203
for (linker_idx, (did_id, rkey)) in
1208
1204
linkers.0.iter().enumerate().skip_while(|(linker_idx, _)| {
1209
-
backward_idx.is_some_and(|idx| match forward_idx {
1210
-
Some(_) => *linker_idx < idx as usize, // inclusive: depend on link idx for skipping
1211
-
None => *linker_idx <= idx as usize, // exclusive: skip right here
1212
-
})
1205
+
cursor.is_some_and(|c| *linker_idx < c.backward as usize)
1213
1206
})
1214
1207
{
1215
1208
if did_id.is_empty()
···
1239
1232
|| filter_to_target_ids.contains(target_id))
1240
1233
})
1241
1234
.skip_while(|(link_idx, _)| {
1242
-
backward_idx.is_some_and(|bl_idx| {
1243
-
linker_idx == bl_idx as usize
1244
-
&& forward_idx.is_some_and(|fwd_idx| *link_idx <= fwd_idx as usize)
1235
+
cursor.is_some_and(|c| {
1236
+
linker_idx == c.backward as usize && *link_idx <= c.forward as usize
1245
1237
})
1246
1238
})
1247
1239
.take(limit as usize + 1 - items.len())
1248
1240
{
1249
1241
// extract forward target did (target that links to the __other__ target)
1250
-
let Some(did) = resolve_active_did(did_id) else {
1242
+
let Some(did) = resolve_active_did(did_id)? else {
1251
1243
continue;
1252
1244
};
1253
1245
// resolve to target string
1254
1246
let Some(fwd_target_key) = self
1255
1247
.target_id_table
1256
-
.get_val_from_id(&self.db, fwd_target_id.0)
1257
-
.ok()
1258
-
.flatten()
1248
+
.get_val_from_id(&self.db, fwd_target_id.0)?
1259
1249
else {
1260
1250
continue;
1261
1251
};
···
1284
1274
// and at backlink_vec_idx itself, forward links at or before
1285
1275
// forward_link_idx are skipped. This correctly resumes mid-record when
1286
1276
// a single backlinker has multiple forward links at path_to_other.
1287
-
let next = if items.len() as u64 > limit {
1288
-
items.truncate(limit as usize);
1289
-
items
1290
-
.last()
1291
-
.map(|(l, f, _, _)| b64::URL_SAFE.encode(format!("{},{}", *l as u64, *f as u64)))
1292
-
} else {
1293
-
None
1294
-
};
1277
+
let next = (items.len() > limit as usize).then(|| {
1278
+
let (l, f, _, _) = items[limit as usize - 1];
1279
+
format!("{l},{f}")
1280
+
});
1295
1281
1296
-
let items = items.into_iter().map(|(_, _, rid, t)| (rid, t)).collect();
1282
+
let items = items
1283
+
.into_iter()
1284
+
.take(limit as usize)
1285
+
.map(|(_, _, rid, t)| (rid, t))
1286
+
.collect();
1297
1287
1298
1288
Ok(PagedOrderedCollection { items, next })
1299
1289
}