+1

.gitignore

··· 1 1 /target 2 2 local/ 3 3 + rocks.test

+4

.prettierrc

··· 1 1 + { 2 2 + "tabWidth": 2, 3 3 + "useTabs": false 4 4 + }

+7 -8

Cargo.lock

··· 1375 1375 checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" 1376 1376 dependencies = [ 1377 1377 "data-encoding", 1378 1378 - "syn 2.0.106", 1378 1378 + "syn 1.0.109", 1379 1379 ] 1380 1380 1381 1381 [[package]] ··· 1796 1796 [[package]] 1797 1797 name = "fjall" 1798 1798 version = "2.11.2" 1799 1799 - source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d" 1799 1799 + source = "git+https://github.com/fjall-rs/fjall.git?rev=fb229572bb7d1d6966a596994dc1708e47ec57d8#fb229572bb7d1d6966a596994dc1708e47ec57d8" 1800 1800 dependencies = [ 1801 1801 "byteorder", 1802 1802 "byteview", ··· 3626 3626 3627 3627 [[package]] 3628 3628 name = "num-bigint-dig" 3629 3629 - version = "0.8.4" 3629 3629 + version = "0.8.6" 3630 3630 source = "registry+https://github.com/rust-lang/crates.io-index" 3631 3631 - checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" 3631 3631 + checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" 3632 3632 dependencies = [ 3633 3633 - "byteorder", 3634 3633 "lazy_static", 3635 3634 "libm", 3636 3635 "num-integer", ··· 4644 4643 4645 4644 [[package]] 4646 4645 name = "rsa" 4647 4647 - version = "0.9.8" 4646 4646 + version = "0.9.10" 4648 4647 source = "registry+https://github.com/rust-lang/crates.io-index" 4649 4649 - checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" 4648 4648 + checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" 4650 4649 dependencies = [ 4651 4650 "const-oid", 4652 4651 "digest", ··· 6049 6048 "clap", 6050 6049 "dropshot", 6051 6050 "env_logger", 6052 6052 - "fjall 2.11.2 (git+https://github.com/fjall-rs/fjall.git)", 6051 6051 + "fjall 2.11.2 (git+https://github.com/fjall-rs/fjall.git?rev=fb229572bb7d1d6966a596994dc1708e47ec57d8)", 6053 6052 "getrandom 0.3.3", 6054 6053 "http", 6055 6054 "jetstream",

+19 -3

constellation/src/bin/main.rs

··· 26 26 #[arg(long)] 27 27 #[clap(default_value = "0.0.0.0:6789")] 28 28 bind: SocketAddr, 29 29 + /// optionally disable the metrics server 30 30 + #[arg(long)] 31 31 + #[clap(default_value_t = false)] 32 32 + collect_metrics: bool, 29 33 /// metrics server's listen address 30 34 #[arg(long)] 31 35 #[clap(default_value = "0.0.0.0:8765")] ··· 92 96 let bind = args.bind; 93 97 let metrics_bind = args.bind_metrics; 94 98 99 99 + let collect_metrics = args.collect_metrics; 95 100 let stay_alive = CancellationToken::new(); 96 101 97 102 match args.backend { ··· 102 107 stream, 103 108 bind, 104 109 metrics_bind, 110 110 + collect_metrics, 105 111 stay_alive, 106 112 ), 107 113 #[cfg(feature = "rocks")] ··· 136 142 stream, 137 143 bind, 138 144 metrics_bind, 145 145 + collect_metrics, 139 146 stay_alive, 140 147 ); 141 148 eprintln!("run finished: {r:?}"); ··· 147 154 } 148 155 } 149 156 157 157 + #[allow(clippy::too_many_lines)] 158 158 + #[allow(clippy::too_many_arguments)] 150 159 fn run( 151 160 mut storage: impl LinkStorage, 152 161 fixture: Option<PathBuf>, ··· 154 163 stream: String, 155 164 bind: SocketAddr, 156 165 metrics_bind: SocketAddr, 166 166 + collect_metrics: bool, 157 167 stay_alive: CancellationToken, 158 168 ) -> Result<()> { 159 169 ctrlc::set_handler({ ··· 198 208 .build() 199 209 .expect("axum startup") 200 210 .block_on(async { 201 201 - install_metrics_server(metrics_bind)?; 211 211 + // Install metrics server only if requested 212 212 + if collect_metrics { 213 213 + install_metrics_server(metrics_bind)?; 214 214 + } 202 215 serve(readable, bind, staying_alive).await 203 216 }) 204 217 .unwrap(); ··· 206 219 } 207 220 }); 208 221 209 209 - s.spawn(move || { // monitor thread 222 222 + // only spawn monitoring thread if the metrics server is running 223 223 + if collect_metrics { 224 224 + s.spawn(move || { // monitor thread 210 225 let stay_alive = stay_alive.clone(); 211 226 let check_alive = stay_alive.clone(); 212 227 ··· 258 273 } 259 274 } 260 275 stay_alive.drop_guard(); 261 261 - }); 276 276 + }); 277 277 + } 262 278 }); 263 279 264 280 println!("byeeee");

+13 -6

constellation/src/consumer/jetstream.rs

··· 226 226 println!("jetstream closed the websocket cleanly."); 227 227 break; 228 228 } 229 229 - r => eprintln!("jetstream: close result after error: {r:?}"), 229 229 + Err(_) => { 230 230 + counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "dirty close").increment(1); 231 231 + println!("jetstream failed to close the websocket cleanly."); 232 232 + break; 233 233 + } 234 234 + Ok(r) => { 235 235 + eprintln!("jetstream: close result after error: {r:?}"); 236 236 + counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error") 237 237 + .increment(1); 238 238 + // if we didn't immediately get ConnectionClosed, we should keep polling read 239 239 + // until we get it. 240 240 + continue; 241 241 + } 230 242 } 231 231 - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error") 232 232 - .increment(1); 233 233 - // if we didn't immediately get ConnectionClosed, we should keep polling read 234 234 - // until we get it. 235 235 - continue; 236 243 } 237 244 }; 238 245

+62 -5

constellation/src/server/mod.rs

··· 17 17 use tokio::task::spawn_blocking; 18 18 use tokio_util::sync::CancellationToken; 19 19 20 20 - use crate::storage::{LinkReader, StorageStats}; 20 20 + use crate::storage::{LinkReader, Order, StorageStats}; 21 21 use crate::{CountsByCount, Did, RecordId}; 22 22 23 23 mod acceptable; ··· 25 25 26 26 use acceptable::{acceptable, ExtractAccept}; 27 27 28 28 - const DEFAULT_CURSOR_LIMIT: u64 = 16; 29 29 - const DEFAULT_CURSOR_LIMIT_MAX: u64 = 100; 28 28 + const DEFAULT_CURSOR_LIMIT: u64 = 100; 29 29 + const DEFAULT_CURSOR_LIMIT_MAX: u64 = 1000; 30 30 31 31 fn get_default_cursor_limit() -> u64 { 32 32 DEFAULT_CURSOR_LIMIT ··· 66 66 } 67 67 }), 68 68 ) 69 69 + // deprecated 69 70 .route( 70 71 "/links/count", 71 72 get({ 72 73 let store = store.clone(); 73 74 move |accept, query| async { 74 75 spawn_blocking(|| count_links(accept, query, store)) 76 76 + .await 77 77 + .map_err(to500)? 78 78 + } 79 79 + }), 80 80 + ) 81 81 + .route( 82 82 + "/xrpc/blue.microcosm.links.getBacklinksCount", 83 83 + get({ 84 84 + let store = store.clone(); 85 85 + move |accept, query| async { 86 86 + spawn_blocking(|| get_backlink_counts(accept, query, store)) 75 87 .await 76 88 .map_err(to500)? 77 89 } ··· 343 355 #[serde(skip_serializing)] 344 356 query: GetLinksCountQuery, 345 357 } 358 358 + #[deprecated] 346 359 fn count_links( 347 360 accept: ExtractAccept, 348 361 query: Query<GetLinksCountQuery>, ··· 361 374 } 362 375 363 376 #[derive(Clone, Deserialize)] 377 377 + struct GetItemsCountQuery { 378 378 + subject: String, 379 379 + source: String, 380 380 + } 381 381 + #[derive(Template, Serialize)] 382 382 + #[template(path = "get-backlinks-count.html.j2")] 383 383 + struct GetItemsCountResponse { 384 384 + total: u64, 385 385 + #[serde(skip_serializing)] 386 386 + query: GetItemsCountQuery, 387 387 + } 388 388 + fn get_backlink_counts( 389 389 + accept: ExtractAccept, 390 390 + query: axum_extra::extract::Query<GetItemsCountQuery>, 391 391 + store: impl LinkReader, 392 392 + ) -> Result<impl IntoResponse, http::StatusCode> { 393 393 + let Some((collection, path)) = query.source.split_once(':') else { 394 394 + return Err(http::StatusCode::BAD_REQUEST); 395 395 + }; 396 396 + let path = format!(".{path}"); 397 397 + let total = store 398 398 + .get_count(&query.subject, collection, &path) 399 399 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 400 400 + 401 401 + Ok(acceptable( 402 402 + accept, 403 403 + GetItemsCountResponse { 404 404 + total, 405 405 + query: (*query).clone(), 406 406 + }, 407 407 + )) 408 408 + } 409 409 + 410 410 + #[derive(Clone, Deserialize)] 364 411 struct GetDidsCountQuery { 365 412 target: String, 366 413 collection: String, ··· 409 456 /// Set the max number of links to return per page of results 410 457 #[serde(default = "get_default_cursor_limit")] 411 458 limit: u64, 412 412 - // TODO: allow reverse (er, forward) order as well 459 459 + /// Allow returning links in reverse order (default: false) 460 460 + #[serde(default)] 461 461 + reverse: bool, 413 462 } 414 463 #[derive(Template, Serialize)] 415 464 #[template(path = "get-backlinks.html.j2")] ··· 455 504 }; 456 505 let path = format!(".{path}"); 457 506 507 507 + let order = if query.reverse { 508 508 + Order::OldestToNewest 509 509 + } else { 510 510 + Order::NewestToOldest 511 511 + }; 512 512 + 458 513 let paged = store 459 514 .get_links( 460 515 &query.subject, 461 516 collection, 462 517 &path, 518 518 + order, 463 519 limit, 464 520 until, 465 521 &filter_dids, ··· 508 564 from_dids: Option<String>, // comma separated: gross 509 565 #[serde(default = "get_default_cursor_limit")] 510 566 limit: u64, 511 511 - // TODO: allow reverse (er, forward) order as well 512 567 } 513 568 #[derive(Template, Serialize)] 514 569 #[template(path = "links.html.j2")] ··· 562 617 &query.target, 563 618 &query.collection, 564 619 &query.path, 620 620 + Order::NewestToOldest, 565 621 limit, 566 622 until, 567 623 &filter_dids, ··· 659 715 #[serde(skip_serializing)] 660 716 query: GetAllLinksQuery, 661 717 } 718 718 + #[deprecated] 662 719 fn count_all_links( 663 720 accept: ExtractAccept, 664 721 query: Query<GetAllLinksQuery>,

+55 -53

constellation/src/storage/mem_store.rs

··· 1 1 use super::{ 2 2 - LinkReader, LinkStorage, PagedAppendingCollection, PagedOrderedCollection, StorageStats, 2 2 + LinkReader, LinkStorage, Order, PagedAppendingCollection, PagedOrderedCollection, StorageStats, 3 3 }; 4 4 use crate::{ActionableEvent, CountsByCount, Did, RecordId}; 5 5 use anyhow::Result; ··· 147 147 ) -> Result<PagedOrderedCollection<(String, u64, u64), String>> { 148 148 let data = self.0.lock().unwrap(); 149 149 let Some(paths) = data.targets.get(&Target::new(target)) else { 150 150 - return Ok(PagedOrderedCollection::default()); 150 150 + return Ok(PagedOrderedCollection::empty()); 151 151 }; 152 152 let Some(linkers) = paths.get(&Source::new(collection, path)) else { 153 153 - return Ok(PagedOrderedCollection::default()); 153 153 + return Ok(PagedOrderedCollection::empty()); 154 154 }; 155 155 156 156 let path_to_other = RecordPath::new(path_to_other); ··· 239 239 target: &str, 240 240 collection: &str, 241 241 path: &str, 242 242 + order: Order, 242 243 limit: u64, 243 243 - until: Option<u64>, 244 244 + until: Option<u64>, // paged iteration endpoint 244 245 filter_dids: &HashSet<Did>, 245 246 ) -> Result<PagedAppendingCollection<RecordId>> { 246 247 let data = self.0.lock().unwrap(); 247 248 let Some(paths) = data.targets.get(&Target::new(target)) else { 248 248 - return Ok(PagedAppendingCollection { 249 249 - version: (0, 0), 250 250 - items: Vec::new(), 251 251 - next: None, 252 252 - total: 0, 253 253 - }); 249 249 + return Ok(PagedAppendingCollection::empty()); 254 250 }; 255 251 let Some(did_rkeys) = paths.get(&Source::new(collection, path)) else { 256 256 - return Ok(PagedAppendingCollection { 257 257 - version: (0, 0), 258 258 - items: Vec::new(), 259 259 - next: None, 260 260 - total: 0, 261 261 - }); 252 252 + return Ok(PagedAppendingCollection::empty()); 262 253 }; 263 254 264 255 let did_rkeys: Vec<_> = if !filter_dids.is_empty() { ··· 275 266 did_rkeys.to_vec() 276 267 }; 277 268 278 278 - let total = did_rkeys.len(); 279 279 - let end = until 280 280 - .map(|u| std::cmp::min(u as usize, total)) 281 281 - .unwrap_or(total); 282 282 - let begin = end.saturating_sub(limit as usize); 283 283 - let next = if begin == 0 { None } else { Some(begin as u64) }; 269 269 + let total = did_rkeys.len() as u64; 284 270 285 285 - let alive = did_rkeys.iter().flatten().count(); 271 271 + // backlinks are stored oldest-to-newest (ascending index with increasing age) 272 272 + let (start, take, next_until) = match order { 273 273 + Order::OldestToNewest => { 274 274 + let start = until.unwrap_or(0); 275 275 + let next = start + limit + 1; 276 276 + let next_until = if next < total { Some(next) } else { None }; 277 277 + (start, limit, next_until) 278 278 + } 279 279 + Order::NewestToOldest => { 280 280 + let until = until.unwrap_or(total); 281 281 + match until.checked_sub(limit) { 282 282 + Some(s) if s > 0 => (s, limit, Some(s)), 283 283 + Some(s) => (s, limit, None), 284 284 + None => (0, until, None), 285 285 + } 286 286 + } 287 287 + }; 288 288 + 289 289 + let alive = did_rkeys.iter().flatten().count() as u64; 286 290 let gone = total - alive; 287 291 288 288 - let items: Vec<_> = did_rkeys[begin..end] 292 292 + let items = did_rkeys 289 293 .iter() 290 290 - .rev() 294 294 + .skip(start as usize) 295 295 + .take(take as usize) 291 296 .flatten() 292 297 .filter(|(did, _)| *data.dids.get(did).expect("did must be in dids")) 293 298 .map(|(did, rkey)| RecordId { 294 299 did: did.clone(), 295 300 rkey: rkey.0.clone(), 296 301 collection: collection.to_string(), 297 297 - }) 298 298 - .collect(); 302 302 + }); 303 303 + 304 304 + let items: Vec<_> = match order { 305 305 + Order::OldestToNewest => items.collect(), // links are stored oldest first 306 306 + Order::NewestToOldest => items.rev().collect(), 307 307 + }; 299 308 300 309 Ok(PagedAppendingCollection { 301 301 - version: (total as u64, gone as u64), 310 310 + version: (total, gone), 302 311 items, 303 303 - next, 304 304 - total: alive as u64, 312 312 + next: next_until, 313 313 + total: alive, 305 314 }) 306 315 } 307 316 ··· 315 324 ) -> Result<PagedAppendingCollection<Did>> { 316 325 let data = self.0.lock().unwrap(); 317 326 let Some(paths) = data.targets.get(&Target::new(target)) else { 318 318 - return Ok(PagedAppendingCollection { 319 319 - version: (0, 0), 320 320 - items: Vec::new(), 321 321 - next: None, 322 322 - total: 0, 323 323 - }); 327 327 + return Ok(PagedAppendingCollection::empty()); 324 328 }; 325 329 let Some(did_rkeys) = paths.get(&Source::new(collection, path)) else { 326 326 - return Ok(PagedAppendingCollection { 327 327 - version: (0, 0), 328 328 - items: Vec::new(), 329 329 - next: None, 330 330 - total: 0, 331 331 - }); 330 330 + return Ok(PagedAppendingCollection::empty()); 332 331 }; 333 332 334 333 let dids: Vec<Option<Did>> = { ··· 348 347 .collect() 349 348 }; 350 349 351 351 - let total = dids.len(); 352 352 - let end = until 353 353 - .map(|u| std::cmp::min(u as usize, total)) 354 354 - .unwrap_or(total); 355 355 - let begin = end.saturating_sub(limit as usize); 356 356 - let next = if begin == 0 { None } else { Some(begin as u64) }; 350 350 + let total = dids.len() as u64; 351 351 + let until = until.unwrap_or(total); 352 352 + let (start, take, next_until) = match until.checked_sub(limit) { 353 353 + Some(s) if s > 0 => (s, limit, Some(s)), 354 354 + Some(s) => (s, limit, None), 355 355 + None => (0, until, None), 356 356 + }; 357 357 358 358 - let alive = dids.iter().flatten().count(); 358 358 + let alive = dids.iter().flatten().count() as u64; 359 359 let gone = total - alive; 360 360 361 361 - let items: Vec<Did> = dids[begin..end] 361 361 + let items: Vec<Did> = dids 362 362 .iter() 363 363 + .skip(start as usize) 364 364 + .take(take as usize) 363 365 .rev() 364 366 .flatten() 365 367 .filter(|did| *data.dids.get(did).expect("did must be in dids")) ··· 367 369 .collect(); 368 370 369 371 Ok(PagedAppendingCollection { 370 370 - version: (total as u64, gone as u64), 372 372 + version: (total, gone), 371 373 items, 372 372 - next, 373 373 - total: alive as u64, 374 374 + next: next_until, 375 375 + total: alive, 374 376 }) 375 377 } 376 378

+195 -76

constellation/src/storage/mod.rs

··· 11 11 #[cfg(feature = "rocks")] 12 12 pub use rocks_store::RocksStorage; 13 13 14 14 - #[derive(Debug, PartialEq)] 14 14 + /// Ordering for paginated link queries 15 15 + #[derive(Debug, Clone, Copy, PartialEq, Eq)] 16 16 + pub enum Order { 17 17 + /// Newest links first (default) 18 18 + NewestToOldest, 19 19 + /// Oldest links first 20 20 + OldestToNewest, 21 21 + } 22 22 + 23 23 + #[derive(Debug, Default, PartialEq)] 15 24 pub struct PagedAppendingCollection<T> { 16 25 pub version: (u64, u64), // (collection length, deleted item count) // TODO: change to (total, active)? since dedups isn't "deleted" 17 26 pub items: Vec<T>, ··· 19 28 pub total: u64, 20 29 } 21 30 31 31 + impl<T> PagedAppendingCollection<T> { 32 32 + pub(crate) fn empty() -> Self { 33 33 + Self { 34 34 + version: (0, 0), 35 35 + items: Vec::new(), 36 36 + next: None, 37 37 + total: 0, 38 38 + } 39 39 + } 40 40 + } 41 41 + 22 42 /// A paged collection whose keys are sorted instead of indexed 23 43 /// 24 44 /// this has weaker guarantees than PagedAppendingCollection: it might 25 45 /// return a totally consistent snapshot. but it should avoid duplicates 26 46 /// and each page should at least be internally consistent. 27 27 - #[derive(Debug, PartialEq, Default)] 47 47 + #[derive(Debug, PartialEq)] 28 48 pub struct PagedOrderedCollection<T, K: Ord> { 29 49 pub items: Vec<T>, 30 50 pub next: Option<K>, 31 51 } 32 52 53 53 + impl<T, K: Ord> PagedOrderedCollection<T, K> { 54 54 + pub(crate) fn empty() -> Self { 55 55 + Self { 56 56 + items: Vec::new(), 57 57 + next: None, 58 58 + } 59 59 + } 60 60 + } 61 61 + 33 62 #[derive(Debug, Deserialize, Serialize, PartialEq)] 34 63 pub struct StorageStats { 35 64 /// estimate of how many accounts we've seen create links. the _subjects_ of any links are not represented here. ··· 82 111 83 112 fn get_distinct_did_count(&self, target: &str, collection: &str, path: &str) -> Result<u64>; 84 113 114 114 + #[allow(clippy::too_many_arguments)] 85 115 fn get_links( 86 116 &self, 87 117 target: &str, 88 118 collection: &str, 89 119 path: &str, 120 120 + order: Order, 90 121 limit: u64, 91 122 until: Option<u64>, 92 123 filter_dids: &HashSet<Did>, ··· 180 211 "a.com", 181 212 "app.t.c", 182 213 ".abc.uri", 214 214 + Order::NewestToOldest, 183 215 100, 184 216 None, 185 217 &HashSet::default() 186 218 )?, 187 187 - PagedAppendingCollection { 188 188 - version: (0, 0), 189 189 - items: vec![], 190 190 - next: None, 191 191 - total: 0, 192 192 - } 219 219 + PagedAppendingCollection::empty() 193 220 ); 194 221 assert_eq!( 195 222 storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 100, None)?, 196 196 - PagedAppendingCollection { 197 197 - version: (0, 0), 198 198 - items: vec![], 199 199 - next: None, 200 200 - total: 0, 201 201 - } 223 223 + PagedAppendingCollection::empty() 202 224 ); 203 225 assert_eq!(storage.get_all_counts("bad-example.com")?, HashMap::new()); 204 226 assert_eq!( ··· 683 705 "a.com", 684 706 "app.t.c", 685 707 ".abc.uri", 708 708 + Order::NewestToOldest, 686 709 100, 687 710 None, 688 711 &HashSet::default() ··· 727 750 0, 728 751 )?; 729 752 } 730 730 - let links = 731 731 - storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None, &HashSet::default())?; 732 732 - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, None)?; 753 753 + 754 754 + let sub = "a.com"; 755 755 + let col = "app.t.c"; 756 756 + let path = ".abc.uri"; 757 757 + let order = Order::NewestToOldest; 758 758 + let dids_filter = HashSet::new(); 759 759 + 760 760 + // --- --- round one! --- --- // 761 761 + // all backlinks 762 762 + let links = storage.get_links(sub, col, path, order, 2, None, &dids_filter)?; 733 763 assert_eq!( 734 764 links, 735 765 PagedAppendingCollection { ··· 737 767 items: vec![ 738 768 RecordId { 739 769 did: "did:plc:asdf-5".into(), 740 740 - collection: "app.t.c".into(), 770 770 + collection: col.into(), 741 771 rkey: "asdf".into(), 742 772 }, 743 773 RecordId { 744 774 did: "did:plc:asdf-4".into(), 745 745 - collection: "app.t.c".into(), 775 775 + collection: col.into(), 746 776 rkey: "asdf".into(), 747 777 }, 748 778 ], ··· 750 780 total: 5, 751 781 } 752 782 ); 783 783 + // distinct dids 784 784 + let dids = storage.get_distinct_dids(sub, col, path, 2, None)?; 753 785 assert_eq!( 754 786 dids, 755 787 PagedAppendingCollection { ··· 759 791 total: 5, 760 792 } 761 793 ); 762 762 - let links = storage.get_links( 763 763 - "a.com", 764 764 - "app.t.c", 765 765 - ".abc.uri", 766 766 - 2, 767 767 - links.next, 768 768 - &HashSet::default(), 769 769 - )?; 770 770 - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, dids.next)?; 794 794 + 795 795 + // --- --- round two! --- --- // 796 796 + // all backlinks 797 797 + let links = storage.get_links(sub, col, path, order, 2, links.next, &dids_filter)?; 771 798 assert_eq!( 772 799 links, 773 800 PagedAppendingCollection { ··· 775 802 items: vec![ 776 803 RecordId { 777 804 did: "did:plc:asdf-3".into(), 778 778 - collection: "app.t.c".into(), 805 805 + collection: col.into(), 779 806 rkey: "asdf".into(), 780 807 }, 781 808 RecordId { 782 809 did: "did:plc:asdf-2".into(), 783 783 - collection: "app.t.c".into(), 810 810 + collection: col.into(), 784 811 rkey: "asdf".into(), 785 812 }, 786 813 ], ··· 788 815 total: 5, 789 816 } 790 817 ); 818 818 + // distinct dids 819 819 + let dids = storage.get_distinct_dids(sub, col, path, 2, dids.next)?; 791 820 assert_eq!( 792 821 dids, 793 822 PagedAppendingCollection { ··· 797 826 total: 5, 798 827 } 799 828 ); 800 800 - let links = storage.get_links( 801 801 - "a.com", 802 802 - "app.t.c", 803 803 - ".abc.uri", 804 804 - 2, 805 805 - links.next, 806 806 - &HashSet::default(), 807 807 - )?; 808 808 - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, dids.next)?; 829 829 + 830 830 + // --- --- round three! --- --- // 831 831 + // all backlinks 832 832 + let links = storage.get_links(sub, col, path, order, 2, links.next, &dids_filter)?; 809 833 assert_eq!( 810 834 links, 811 835 PagedAppendingCollection { 812 836 version: (5, 0), 813 837 items: vec![RecordId { 814 838 did: "did:plc:asdf-1".into(), 815 815 - collection: "app.t.c".into(), 839 839 + collection: col.into(), 816 840 rkey: "asdf".into(), 817 841 },], 818 842 next: None, 819 843 total: 5, 820 844 } 821 845 ); 846 846 + // distinct dids 847 847 + let dids = storage.get_distinct_dids(sub, col, path, 2, dids.next)?; 822 848 assert_eq!( 823 849 dids, 824 850 PagedAppendingCollection { ··· 828 854 total: 5, 829 855 } 830 856 ); 857 857 + 831 858 assert_stats(storage.get_stats()?, 5..=5, 1..=1, 5..=5); 832 859 }); 833 860 834 834 - test_each_storage!(get_filtered_links, |storage| { 861 861 + test_each_storage!(get_links_reverse_order, |storage| { 862 862 + for i in 1..=5 { 863 863 + storage.push( 864 864 + &ActionableEvent::CreateLinks { 865 865 + record_id: RecordId { 866 866 + did: format!("did:plc:asdf-{i}").into(), 867 867 + collection: "app.t.c".into(), 868 868 + rkey: "asdf".into(), 869 869 + }, 870 870 + links: vec![CollectedLink { 871 871 + target: Link::Uri("a.com".into()), 872 872 + path: ".abc.uri".into(), 873 873 + }], 874 874 + }, 875 875 + 0, 876 876 + )?; 877 877 + } 878 878 + 879 879 + // Test OldestToNewest order (oldest first) 880 880 + let links = storage.get_links( 881 881 + "a.com", 882 882 + "app.t.c", 883 883 + ".abc.uri", 884 884 + Order::OldestToNewest, 885 885 + 2, 886 886 + None, 887 887 + &HashSet::default(), 888 888 + )?; 889 889 + assert_eq!( 890 890 + links, 891 891 + PagedAppendingCollection { 892 892 + version: (5, 0), 893 893 + items: vec![ 894 894 + RecordId { 895 895 + did: "did:plc:asdf-1".into(), 896 896 + collection: "app.t.c".into(), 897 897 + rkey: "asdf".into(), 898 898 + }, 899 899 + RecordId { 900 900 + did: "did:plc:asdf-2".into(), 901 901 + collection: "app.t.c".into(), 902 902 + rkey: "asdf".into(), 903 903 + }, 904 904 + ], 905 905 + next: Some(3), 906 906 + total: 5, 907 907 + } 908 908 + ); 909 909 + // Test NewestToOldest order (newest first) 835 910 let links = storage.get_links( 836 911 "a.com", 837 912 "app.t.c", 838 913 ".abc.uri", 914 914 + Order::NewestToOldest, 839 915 2, 840 916 None, 841 841 - &HashSet::from([Did("did:plc:linker".to_string())]), 917 917 + &HashSet::default(), 842 918 )?; 843 919 assert_eq!( 844 920 links, 845 921 PagedAppendingCollection { 846 846 - version: (0, 0), 847 847 - items: vec![], 848 848 - next: None, 849 849 - total: 0, 922 922 + version: (5, 0), 923 923 + items: vec![ 924 924 + RecordId { 925 925 + did: "did:plc:asdf-5".into(), 926 926 + collection: "app.t.c".into(), 927 927 + rkey: "asdf".into(), 928 928 + }, 929 929 + RecordId { 930 930 + did: "did:plc:asdf-4".into(), 931 931 + collection: "app.t.c".into(), 932 932 + rkey: "asdf".into(), 933 933 + }, 934 934 + ], 935 935 + next: Some(3), 936 936 + total: 5, 850 937 } 851 938 ); 939 939 + assert_stats(storage.get_stats()?, 5..=5, 1..=1, 5..=5); 940 940 + }); 941 941 + 942 942 + test_each_storage!(get_filtered_links, |storage| { 943 943 + let links = storage.get_links( 944 944 + "a.com", 945 945 + "app.t.c", 946 946 + ".abc.uri", 947 947 + Order::NewestToOldest, 948 948 + 2, 949 949 + None, 950 950 + &HashSet::from([Did("did:plc:linker".to_string())]), 951 951 + )?; 952 952 + assert_eq!(links, PagedAppendingCollection::empty()); 852 953 853 954 storage.push( 854 955 &ActionableEvent::CreateLinks { ··· 869 970 "a.com", 870 971 "app.t.c", 871 972 ".abc.uri", 973 973 + Order::NewestToOldest, 872 974 2, 873 975 None, 874 976 &HashSet::from([Did("did:plc:linker".to_string())]), ··· 891 993 "a.com", 892 994 "app.t.c", 893 995 ".abc.uri", 996 996 + Order::NewestToOldest, 894 997 2, 895 998 None, 896 999 &HashSet::from([Did("did:plc:someone-else".to_string())]), 897 1000 )?; 898 898 - assert_eq!( 899 899 - links, 900 900 - PagedAppendingCollection { 901 901 - version: (0, 0), 902 902 - items: vec![], 903 903 - next: None, 904 904 - total: 0, 905 905 - } 906 906 - ); 1001 1001 + assert_eq!(links, PagedAppendingCollection::empty()); 907 1002 908 1003 storage.push( 909 1004 &ActionableEvent::CreateLinks { ··· 938 1033 "a.com", 939 1034 "app.t.c", 940 1035 ".abc.uri", 1036 1036 + Order::NewestToOldest, 941 1037 2, 942 1038 None, 943 1039 &HashSet::from([Did("did:plc:linker".to_string())]), ··· 967 1063 "a.com", 968 1064 "app.t.c", 969 1065 ".abc.uri", 1066 1066 + Order::NewestToOldest, 970 1067 2, 971 1068 None, 972 1069 &HashSet::from([ ··· 999 1096 "a.com", 1000 1097 "app.t.c", 1001 1098 ".abc.uri", 1099 1099 + Order::NewestToOldest, 1002 1100 2, 1003 1101 None, 1004 1102 &HashSet::from([Did("did:plc:someone-unknown".to_string())]), 1005 1103 )?; 1006 1006 - assert_eq!( 1007 1007 - links, 1008 1008 - PagedAppendingCollection { 1009 1009 - version: (0, 0), 1010 1010 - items: vec![], 1011 1011 - next: None, 1012 1012 - total: 0, 1013 1013 - } 1014 1014 - ); 1104 1104 + assert_eq!(links, PagedAppendingCollection::empty()); 1015 1105 }); 1016 1106 1017 1107 test_each_storage!(get_links_exact_multiple, |storage| { ··· 1031 1121 0, 1032 1122 )?; 1033 1123 } 1034 1034 - let links = 1035 1035 - storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None, &HashSet::default())?; 1124 1124 + let links = storage.get_links( 1125 1125 + "a.com", 1126 1126 + "app.t.c", 1127 1127 + ".abc.uri", 1128 1128 + Order::NewestToOldest, 1129 1129 + 2, 1130 1130 + None, 1131 1131 + &HashSet::default(), 1132 1132 + )?; 1036 1133 assert_eq!( 1037 1134 links, 1038 1135 PagedAppendingCollection { ··· 1057 1154 "a.com", 1058 1155 "app.t.c", 1059 1156 ".abc.uri", 1157 1157 + Order::NewestToOldest, 1060 1158 2, 1061 1159 links.next, 1062 1160 &HashSet::default(), ··· 1101 1199 0, 1102 1200 )?; 1103 1201 } 1104 1104 - let links = 1105 1105 - storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None, &HashSet::default())?; 1202 1202 + let links = storage.get_links( 1203 1203 + "a.com", 1204 1204 + "app.t.c", 1205 1205 + ".abc.uri", 1206 1206 + Order::NewestToOldest, 1207 1207 + 2, 1208 1208 + None, 1209 1209 + &HashSet::default(), 1210 1210 + )?; 1106 1211 assert_eq!( 1107 1212 links, 1108 1213 PagedAppendingCollection { ··· 1141 1246 "a.com", 1142 1247 "app.t.c", 1143 1248 ".abc.uri", 1249 1249 + Order::NewestToOldest, 1144 1250 2, 1145 1251 links.next, 1146 1252 &HashSet::default(), ··· 1185 1291 0, 1186 1292 )?; 1187 1293 } 1188 1188 - let links = 1189 1189 - storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None, &HashSet::default())?; 1294 1294 + let links = storage.get_links( 1295 1295 + "a.com", 1296 1296 + "app.t.c", 1297 1297 + ".abc.uri", 1298 1298 + Order::NewestToOldest, 1299 1299 + 2, 1300 1300 + None, 1301 1301 + &HashSet::default(), 1302 1302 + )?; 1190 1303 assert_eq!( 1191 1304 links, 1192 1305 PagedAppendingCollection { ··· 1219 1332 "a.com", 1220 1333 "app.t.c", 1221 1334 ".abc.uri", 1335 1335 + Order::NewestToOldest, 1222 1336 2, 1223 1337 links.next, 1224 1338 &HashSet::default(), ··· 1256 1370 0, 1257 1371 )?; 1258 1372 } 1259 1259 - let links = 1260 1260 - storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None, &HashSet::default())?; 1373 1373 + let links = storage.get_links( 1374 1374 + "a.com", 1375 1375 + "app.t.c", 1376 1376 + ".abc.uri", 1377 1377 + Order::NewestToOldest, 1378 1378 + 2, 1379 1379 + None, 1380 1380 + &HashSet::default(), 1381 1381 + )?; 1261 1382 assert_eq!( 1262 1383 links, 1263 1384 PagedAppendingCollection { ··· 1286 1407 "a.com", 1287 1408 "app.t.c", 1288 1409 ".abc.uri", 1410 1410 + Order::NewestToOldest, 1289 1411 2, 1290 1412 links.next, 1291 1413 &HashSet::default(), ··· 1372 1494 &HashSet::new(), 1373 1495 &HashSet::new(), 1374 1496 )?, 1375 1375 - PagedOrderedCollection { 1376 1376 - items: vec![], 1377 1377 - next: None, 1378 1378 - } 1497 1497 + PagedOrderedCollection::empty() 1379 1498 ); 1380 1499 }); 1381 1500

+41 -25

constellation/src/storage/rocks_store.rs

··· 1 1 use super::{ 2 2 - ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, PagedOrderedCollection, 3 3 - StorageStats, 2 2 + ActionableEvent, LinkReader, LinkStorage, Order, PagedAppendingCollection, 3 3 + PagedOrderedCollection, StorageStats, 4 4 }; 5 5 use crate::{CountsByCount, Did, RecordId}; 6 6 use anyhow::{bail, Result}; ··· 960 960 961 961 let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { 962 962 eprintln!("nothin doin for this target, {target_key:?}"); 963 963 - return Ok(Default::default()); 963 963 + return Ok(PagedOrderedCollection::empty()); 964 964 }; 965 965 966 966 let filter_did_ids: HashMap<DidId, bool> = filter_dids ··· 1127 1127 target: &str, 1128 1128 collection: &str, 1129 1129 path: &str, 1130 1130 + order: Order, 1130 1131 limit: u64, 1131 1132 until: Option<u64>, 1132 1133 filter_dids: &HashSet<Did>, ··· 1138 1139 ); 1139 1140 1140 1141 let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { 1141 1141 - return Ok(PagedAppendingCollection { 1142 1142 - version: (0, 0), 1143 1143 - items: Vec::new(), 1144 1144 - next: None, 1145 1145 - total: 0, 1146 1146 - }); 1142 1142 + return Ok(PagedAppendingCollection::empty()); 1147 1143 }; 1148 1144 1149 1145 let mut linkers = self.get_target_linkers(&target_id)?; ··· 1167 1163 1168 1164 let (alive, gone) = linkers.count(); 1169 1165 let total = alive + gone; 1170 1170 - let end = until.map(|u| std::cmp::min(u, total)).unwrap_or(total) as usize; 1171 1171 - let begin = end.saturating_sub(limit as usize); 1172 1172 - let next = if begin == 0 { None } else { Some(begin as u64) }; 1173 1166 1174 1174 - let did_id_rkeys = linkers.0[begin..end].iter().rev().collect::<Vec<_>>(); 1167 1167 + let (start, take, next_until) = match order { 1168 1168 + // OldestToNewest: start from the beginning, paginate forward 1169 1169 + Order::OldestToNewest => { 1170 1170 + let start = until.unwrap_or(0); 1171 1171 + let next = start + limit + 1; 1172 1172 + let next_until = if next < total { Some(next) } else { None }; 1173 1173 + (start, limit, next_until) 1174 1174 + } 1175 1175 + // NewestToOldest: start from the end, paginate backward 1176 1176 + Order::NewestToOldest => { 1177 1177 + let until = until.unwrap_or(total); 1178 1178 + match until.checked_sub(limit) { 1179 1179 + Some(s) if s > 0 => (s, limit, Some(s)), 1180 1180 + Some(s) => (s, limit, None), 1181 1181 + None => (0, until, None), 1182 1182 + } 1183 1183 + } 1184 1184 + }; 1185 1185 + 1186 1186 + let did_id_rkeys = linkers.0.iter().skip(start as usize).take(take as usize); 1187 1187 + let did_id_rkeys: Vec<_> = match order { 1188 1188 + Order::OldestToNewest => did_id_rkeys.collect(), 1189 1189 + Order::NewestToOldest => did_id_rkeys.rev().collect(), 1190 1190 + }; 1175 1191 1176 1192 let mut items = Vec::with_capacity(did_id_rkeys.len()); 1177 1193 // TODO: use get-many (or multi-get or whatever it's called) ··· 1201 1217 Ok(PagedAppendingCollection { 1202 1218 version: (total, gone), 1203 1219 items, 1204 1204 - next, 1220 1220 + next: next_until, 1205 1221 total: alive, 1206 1222 }) 1207 1223 } ··· 1221 1237 ); 1222 1238 1223 1239 let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { 1224 1224 - return Ok(PagedAppendingCollection { 1225 1225 - version: (0, 0), 1226 1226 - items: Vec::new(), 1227 1227 - next: None, 1228 1228 - total: 0, 1229 1229 - }); 1240 1240 + return Ok(PagedAppendingCollection::empty()); 1230 1241 }; 1231 1242 1232 1243 let linkers = self.get_distinct_target_linkers(&target_id)?; 1233 1244 1234 1245 let (alive, gone) = linkers.count(); 1235 1246 let total = alive + gone; 1236 1236 - let end = until.map(|u| std::cmp::min(u, total)).unwrap_or(total) as usize; 1237 1237 - let begin = end.saturating_sub(limit as usize); 1238 1238 - let next = if begin == 0 { None } else { Some(begin as u64) }; 1247 1247 + 1248 1248 + let until = until.unwrap_or(total); 1249 1249 + let (start, take, next_until) = match until.checked_sub(limit) { 1250 1250 + Some(s) if s > 0 => (s, limit, Some(s)), 1251 1251 + Some(s) => (s, limit, None), 1252 1252 + None => (0, until, None), 1253 1253 + }; 1239 1254 1240 1240 - let did_id_rkeys = linkers.0[begin..end].iter().rev().collect::<Vec<_>>(); 1255 1255 + let did_id_rkeys = linkers.0.iter().skip(start as usize).take(take as usize); 1256 1256 + let did_id_rkeys: Vec<_> = did_id_rkeys.rev().collect(); 1241 1257 1242 1258 let mut items = Vec::with_capacity(did_id_rkeys.len()); 1243 1259 // TODO: use get-many (or multi-get or whatever it's called) ··· 1263 1279 Ok(PagedAppendingCollection { 1264 1280 version: (total, gone), 1265 1281 items, 1266 1266 - next, 1282 1282 + next: next_until, 1267 1283 total: alive, 1268 1284 }) 1269 1285 }

+4

constellation/templates/base.html.j2

··· 40 40 padding: 0.5em 0.3em; 41 41 max-width: 100%; 42 42 } 43 43 + pre.code input { 44 44 + margin: 0; 45 45 + padding: 0; 46 46 + } 43 47 .stat { 44 48 color: #f90; 45 49 font-size: 1.618rem;

+1 -1

constellation/templates/dids.html.j2

··· 27 27 {% for did in linking_dids %} 28 28 <pre style="display: block; margin: 1em 2em" class="code"><strong>DID</strong>: {{ did.0 }} 29 29 -> see <a href="/links/all?target={{ did.0|urlencode }}">links to this DID</a> 30 30 - -> browse <a href="https://pdsls.dev/at://{{ did.0|urlencode }}">this DID record</a></pre> 30 30 + -> browse <a href="https://pdsls.dev/at://{{ did.0 }}">this DID record</a></pre> 31 31 {% endfor %} 32 32 33 33 {% if let Some(c) = cursor %}

+38

constellation/templates/get-backlinks-count.html.j2

··· 1 1 + {% extends "base.html.j2" %} 2 2 + {% import "try-it-macros.html.j2" as try_it %} 3 3 + 4 4 + {% block title %}Link Count{% endblock %} 5 5 + {% block description %}Count of {{ query.source }} records linking to {{ query.subject }}{% endblock %} 6 6 + 7 7 + {% block content %} 8 8 + 9 9 + {% call try_it::get_backlinks_count( 10 10 + query.subject, 11 11 + query.source, 12 12 + ) %} 13 13 + 14 14 + <h2> 15 15 + Total links to <code>{{ query.subject }}</code> 16 16 + {% if let Some(browseable_uri) = query.subject|to_browseable %} 17 17 + <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 18 18 + {% endif %} 19 19 + </h2> 20 20 + 21 21 + <p><strong><code>{{ total|human_number }}</code></strong> total links from <code>{{ query.source }}</code> to <code>{{ query.subject }}</code></p> 22 22 + 23 23 + <ul> 24 24 + <li> 25 25 + See direct backlinks at <code>/xrpc/blue.microcosm.links.getBacklinks</code>: 26 26 + <a href="/xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject|urlencode }}&source={{ query.source|urlencode }}"> 27 27 + /xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject }}&source={{ query.source }} 28 28 + </a> 29 29 + </li> 30 30 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.subject|urlencode }}">/links/all?target={{ query.subject }}</a></li> 31 31 + </ul> 32 32 + 33 33 + <details> 34 34 + <summary>Raw JSON response</summary> 35 35 + <pre class="code">{{ self|tojson }}</pre> 36 36 + </details> 37 37 + 38 38 + {% endblock %}

+2 -1

constellation/templates/get-backlinks.html.j2

··· 6 6 7 7 {% block content %} 8 8 9 9 - {% call try_it::get_backlinks(query.subject, query.source, query.did, query.limit) %} 9 9 + {% call try_it::get_backlinks(query.subject, query.source, query.did, query.limit, query.reverse) %} 10 10 11 11 <h2> 12 12 Links to <code>{{ query.subject }}</code> ··· 40 40 <input type="hidden" name="did" value="{{ did }}" /> 41 41 {% endfor %} 42 42 <input type="hidden" name="cursor" value={{ c|json|safe }} /> 43 43 + <input type="hidden" name="reverse" value="{{ query.reverse }}"> 43 44 <button type="submit">next page&hellip;</button> 44 45 </form> 45 46 {% else %}

+18 -2

constellation/templates/hello.html.j2

··· 49 49 <li><p><code>source</code>: required. Example: <code>app.bsky.feed.like:subject.uri</code></p></li> 50 50 <li><p><code>did</code>: optional, filter links to those from specific users. Include multiple times to filter by multiple users. Example: <code>did=did:plc:vc7f4oafdgxsihk4cry2xpze&did=did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 51 51 <li><p><code>limit</code>: optional. Default: <code>16</code>. Maximum: <code>100</code></p></li> 52 52 + <li><p><code>reverse</code>: optional, return links in reverse order. Default: <code>false</code></p></li> 52 53 </ul> 53 54 54 55 <p style="margin-bottom: 0"><strong>Try it:</strong></p> 55 55 - {% call try_it::get_backlinks("at://did:plc:a4pqq234yw7fqbddawjo7y35/app.bsky.feed.post/3m237ilwc372e", "app.bsky.feed.like:subject.uri", [""], 16) %} 56 56 + {% call 57 57 + try_it::get_backlinks("at://did:plc:a4pqq234yw7fqbddawjo7y35/app.bsky.feed.post/3m237ilwc372e", "app.bsky.feed.like:subject.uri", [""], 16, false) %} 56 58 57 59 58 60 <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getManyToManyCounts</code></h3> ··· 96 98 <li><p><code>did</code>: optional, filter links to those from specific users. Include multiple times to filter by multiple users. Example: <code>did=did:plc:vc7f4oafdgxsihk4cry2xpze&did=did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 97 99 <li><p><code>from_dids</code> [deprecated]: optional. Use <code>did</code> instead. Example: <code>from_dids=did:plc:vc7f4oafdgxsihk4cry2xpze,did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 98 100 <li><p><code>limit</code>: optional. Default: <code>16</code>. Maximum: <code>100</code></p></li> 101 101 + <li><p><code>reverse</code>: optional, return links in reverse order. Default: <code>false</code></p></li> 99 102 </ul> 100 103 101 104 <p style="margin-bottom: 0"><strong>Try it:</strong></p> ··· 118 121 {% call try_it::dids("at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r", "app.bsky.feed.like", ".subject.uri") %} 119 122 120 123 121 121 - <h3 class="route"><code>GET /links/count</code></h3> 124 124 + <h3 class="route deprecated"><code>[deprecated] GET /links/count</code></h3> 122 125 123 126 <p>The total number of links pointing at a given target.</p> 124 127 ··· 134 137 <p style="margin-bottom: 0"><strong>Try it:</strong></p> 135 138 {% call try_it::links_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block", ".subject") %} 136 139 140 140 + <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getBacklinksCount</code></h3> 141 141 + 142 142 + <p>The total number of links pointing at a given target.</p> 143 143 + 144 144 + <h4>Query parameters:</h4> 145 145 + 146 146 + <ul> 147 147 + <li><code>subject</code>: required, must url-encode. The target being linked to. Example: <code>did:plc:vc7f4oafdgxsihk4cry2xpze</code> or <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></li> 148 148 + <li><code>source</code>: required. Collection and path specification for the primary link. Example: <code>app.bsky.feed.like:subject.uri</code></li> 149 149 + </ul> 150 150 + 151 151 + <p style="margin-bottom: 0"><strong>Try it:</strong></p> 152 152 + {% call try_it::get_backlinks_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block:subject") %} 137 153 138 154 <h3 class="route"><code>GET /links/count/distinct-dids</code></h3> 139 155

+11 -2

constellation/templates/try-it-macros.html.j2

··· 1 1 - {% macro get_backlinks(subject, source, dids, limit) %} 1 1 + {% macro get_backlinks(subject, source, dids, limit, reverse) %} 2 2 <form method="get" action="/xrpc/blue.microcosm.links.getBacklinks"> 3 3 <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getBacklinks 4 4 ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="at-uri, did, uri..." /> ··· 6 6 {%- for did in dids %}{% if !did.is_empty() %} 7 7 &did= <input type="text" name="did" value="{{ did }}" placeholder="did:plc:..." />{% endif %}{% endfor %} 8 8 <span id="did-placeholder"></span> <button id="add-did">+ did filter</button> 9 9 - &limit= <input type="number" name="limit" value="{{ limit }}" max="100" placeholder="100" /> <button type="submit">get links</button></pre> 9 9 + &limit= <input type="number" name="limit" value="{{ limit }}" max="100" placeholder="100" /> 10 10 + &reverse= <input type="checkbox" name="reverse" value="true" {% if reverse %}checked{% endif %}> 11 11 + <button type="submit">get links</button></pre> 10 12 </form> 11 13 <script> 12 14 const addDidButton = document.getElementById('add-did'); ··· 102 104 </form> 103 105 {% endmacro %} 104 106 107 107 + {% macro get_backlinks_count(subject, source) %} 108 108 + <form method="get" action="/xrpc/blue.microcosm.links.getBacklinksCount"> 109 109 + <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getBacklinksCount 110 110 + ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="subject" /> 111 111 + &source= <input type="text" name="source" value="{{ source }}" placeholder="source" /> <button type="submit">get links count</button></pre> 112 112 + </form> 113 113 + {% endmacro %} 105 114 106 115 {% macro links_count(target, collection, path) %} 107 116 <form method="get" action="/links/count">

+95

lexicons/blue.microcosm/links/getBacklinks.json

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "blue.microcosm.links.getBacklinks", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "a list of records linking to any record, identity, or uri", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": [ 11 11 + "subject", 12 12 + "source" 13 13 + ], 14 14 + "properties": { 15 15 + "subject": { 16 16 + "type": "string", 17 17 + "format": "uri", 18 18 + "description": "the target being linked to (at-uri, did, or uri)" 19 19 + }, 20 20 + "source": { 21 21 + "type": "string", 22 22 + "description": "collection and path specification (e.g., 'app.bsky.feed.like:subject.uri')" 23 23 + }, 24 24 + "did": { 25 25 + "type": "array", 26 26 + "description": "filter links to those from specific users", 27 27 + "items": { 28 28 + "type": "string", 29 29 + "format": "did" 30 30 + } 31 31 + }, 32 32 + "limit": { 33 33 + "type": "integer", 34 34 + "minimum": 1, 35 35 + "maximum": 100, 36 36 + "default": 16, 37 37 + "description": "number of results to return" 38 38 + } 39 39 + } 40 40 + }, 41 41 + "output": { 42 42 + "encoding": "application/json", 43 43 + "schema": { 44 44 + "type": "object", 45 45 + "required": [ 46 46 + "total", 47 47 + "records" 48 48 + ], 49 49 + "properties": { 50 50 + "total": { 51 51 + "type": "integer", 52 52 + "description": "total number of matching links" 53 53 + }, 54 54 + "records": { 55 55 + "type": "array", 56 56 + "items": { 57 57 + "type": "ref", 58 58 + "ref": "#linkRecord" 59 59 + } 60 60 + }, 61 61 + "cursor": { 62 62 + "type": "string", 63 63 + "description": "pagination cursor" 64 64 + } 65 65 + } 66 66 + } 67 67 + } 68 68 + }, 69 69 + "linkRecord": { 70 70 + "type": "object", 71 71 + "required": [ 72 72 + "did", 73 73 + "collection", 74 74 + "rkey" 75 75 + ], 76 76 + "properties": { 77 77 + "did": { 78 78 + "type": "string", 79 79 + "format": "did", 80 80 + "description": "the DID of the linking record's repository" 81 81 + }, 82 82 + "collection": { 83 83 + "type": "string", 84 84 + "format": "nsid", 85 85 + "description": "the collection of the linking record" 86 86 + }, 87 87 + "rkey": { 88 88 + "type": "string", 89 89 + "format": "record-key", 90 90 + "description": "the record key of the linking record" 91 91 + } 92 92 + } 93 93 + } 94 94 + } 95 95 + }

+38

lexicons/blue.microcosm/links/getBacklinksCount.json

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "blue.microcosm.links.getBacklinksCount", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "count records that link to another record", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": ["subject", "source"], 11 11 + "properties": { 12 12 + "subject": { 13 13 + "type": "string", 14 14 + "format": "uri", 15 15 + "description": "the primary target being linked to (at-uri, did, or uri)" 16 16 + }, 17 17 + "source": { 18 18 + "type": "string", 19 19 + "description": "collection and path specification for the primary link" 20 20 + } 21 21 + } 22 22 + }, 23 23 + "output": { 24 24 + "encoding": "application/json", 25 25 + "schema": { 26 26 + "type": "object", 27 27 + "required": ["total"], 28 28 + "properties": { 29 29 + "total": { 30 30 + "type": "integer", 31 31 + "description": "total number of matching links" 32 32 + } 33 33 + } 34 34 + } 35 35 + } 36 36 + } 37 37 + } 38 38 + }

+99

lexicons/blue.microcosm/links/getManyToManyCounts.json

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "blue.microcosm.links.getManyToManyCounts", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "count many-to-many relationships with secondary link paths", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": [ 11 11 + "subject", 12 12 + "source", 13 13 + "pathToOther" 14 14 + ], 15 15 + "properties": { 16 16 + "subject": { 17 17 + "type": "string", 18 18 + "format": "uri", 19 19 + "description": "the primary target being linked to (at-uri, did, or uri)" 20 20 + }, 21 21 + "source": { 22 22 + "type": "string", 23 23 + "description": "collection and path specification for the primary link" 24 24 + }, 25 25 + "pathToOther": { 26 26 + "type": "string", 27 27 + "description": "path to the secondary link in the many-to-many record (e.g., 'otherThing.uri')" 28 28 + }, 29 29 + "did": { 30 30 + "type": "array", 31 31 + "description": "filter links to those from specific users", 32 32 + "items": { 33 33 + "type": "string", 34 34 + "format": "did" 35 35 + } 36 36 + }, 37 37 + "otherSubject": { 38 38 + "type": "array", 39 39 + "description": "filter secondary links to specific subjects", 40 40 + "items": { 41 41 + "type": "string" 42 42 + } 43 43 + }, 44 44 + "limit": { 45 45 + "type": "integer", 46 46 + "minimum": 1, 47 47 + "maximum": 100, 48 48 + "default": 16, 49 49 + "description": "number of results to return" 50 50 + } 51 51 + } 52 52 + }, 53 53 + "output": { 54 54 + "encoding": "application/json", 55 55 + "schema": { 56 56 + "type": "object", 57 57 + "required": [ 58 58 + "counts_by_other_subject" 59 59 + ], 60 60 + "properties": { 61 61 + "counts_by_other_subject": { 62 62 + "type": "array", 63 63 + "items": { 64 64 + "type": "ref", 65 65 + "ref": "#countBySubject" 66 66 + } 67 67 + }, 68 68 + "cursor": { 69 69 + "type": "string", 70 70 + "description": "pagination cursor" 71 71 + } 72 72 + } 73 73 + } 74 74 + } 75 75 + }, 76 76 + "countBySubject": { 77 77 + "type": "object", 78 78 + "required": [ 79 79 + "subject", 80 80 + "total", 81 81 + "distinct" 82 82 + ], 83 83 + "properties": { 84 84 + "subject": { 85 85 + "type": "string", 86 86 + "description": "the secondary subject being counted" 87 87 + }, 88 88 + "total": { 89 89 + "type": "integer", 90 90 + "description": "total number of links to this subject" 91 91 + }, 92 92 + "distinct": { 93 93 + "type": "integer", 94 94 + "description": "number of distinct DIDs linking to this subject" 95 95 + } 96 96 + } 97 97 + } 98 98 + } 99 99 + }

+56

lexicons/com.bad-example/identity/resolveMiniDoc.json

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "com.bad-example.identity.resolveMiniDoc", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "like com.atproto.identity.resolveIdentity but instead of the full didDoc it returns an atproto-relevant subset", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": [ 11 11 + "identifier" 12 12 + ], 13 13 + "properties": { 14 14 + "identifier": { 15 15 + "type": "string", 16 16 + "format": "at-identifier", 17 17 + "description": "handle or DID to resolve" 18 18 + } 19 19 + } 20 20 + }, 21 21 + "output": { 22 22 + "encoding": "application/json", 23 23 + "schema": { 24 24 + "type": "object", 25 25 + "required": [ 26 26 + "did", 27 27 + "handle", 28 28 + "pds", 29 29 + "signing_key" 30 30 + ], 31 31 + "properties": { 32 32 + "did": { 33 33 + "type": "string", 34 34 + "format": "did", 35 35 + "description": "DID, bi-directionally verified if a handle was provided in the query" 36 36 + }, 37 37 + "handle": { 38 38 + "type": "string", 39 39 + "format": "handle", 40 40 + "description": "the validated handle of the account or 'handle.invalid' if the handle did not bi-directionally match the DID document" 41 41 + }, 42 42 + "pds": { 43 43 + "type": "string", 44 44 + "format": "uri", 45 45 + "description": "the identity's PDS URL" 46 46 + }, 47 47 + "signing_key": { 48 48 + "type": "string", 49 49 + "description": "the atproto signing key publicKeyMultibase" 50 50 + } 51 51 + } 52 52 + } 53 53 + } 54 54 + } 55 55 + } 56 56 + }

+54

lexicons/com.bad-example/repo/getUriRecord.json

··· 1 1 + { 2 2 + "lexicon": 1, 3 3 + "id": "com.bad-example.repo.getUriRecord", 4 4 + "defs": { 5 5 + "main": { 6 6 + "type": "query", 7 7 + "description": "ergonomic complement to com.atproto.repo.getRecord which accepts an at-uri instead of individual repo/collection/rkey params", 8 8 + "parameters": { 9 9 + "type": "params", 10 10 + "required": [ 11 11 + "at_uri" 12 12 + ], 13 13 + "properties": { 14 14 + "at_uri": { 15 15 + "type": "string", 16 16 + "format": "at-uri", 17 17 + "description": "the at-uri of the record (identifier can be a DID or handle)" 18 18 + }, 19 19 + "cid": { 20 20 + "type": "string", 21 21 + "format": "cid", 22 22 + "description": "optional CID of the version of the record. if not specified, return the most recent version. if specified and a newer version exists, returns 404." 23 23 + } 24 24 + } 25 25 + }, 26 26 + "output": { 27 27 + "encoding": "application/json", 28 28 + "schema": { 29 29 + "type": "object", 30 30 + "required": [ 31 31 + "uri", 32 32 + "value" 33 33 + ], 34 34 + "properties": { 35 35 + "uri": { 36 36 + "type": "string", 37 37 + "format": "at-uri", 38 38 + "description": "at-uri for this record" 39 39 + }, 40 40 + "cid": { 41 41 + "type": "string", 42 42 + "format": "cid", 43 43 + "description": "CID for this exact version of the record" 44 44 + }, 45 45 + "value": { 46 46 + "type": "unknown", 47 47 + "description": "the record itself" 48 48 + } 49 49 + } 50 50 + } 51 51 + } 52 52 + } 53 53 + } 54 54 + }

+1 -1

readme.md

··· 10 10 Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://discord.gg/tcDfe4PGVB), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere. 11 11 12 12 > [!tip] 13 13 - > This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.sh/@microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! 13 13 + > This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.org/microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! 14 14 15 15 16 16 🌌 [Constellation](./constellation/)

+1 -1

slingshot/api-description.md

··· 90 90 - [🎇 Spacedust](https://spacedust.microcosm.blue/), a firehose of all social interactions 91 91 92 92 > [!success] 93 93 - > All microcosm projects are [open source](https://tangled.sh/@bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)! 93 93 + > All microcosm projects are [open source](https://tangled.org/bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)!

+2

spacedust/src/error.rs

··· 30 30 TooManySourcesWanted, 31 31 #[error("more wantedSubjectDids were requested than allowed (max 10,000)")] 32 32 TooManyDidsWanted, 33 33 + #[error("more wantedSubjectPrefixes were requested than allowed (max 100)")] 34 34 + TooManySubjectPrefixesWanted, 33 35 #[error("more wantedSubjects were requested than allowed (max 50,000)")] 34 36 TooManySubjectsWanted, 35 37 }

+11 -2

spacedust/src/server.rs

··· 228 228 #[serde(default)] 229 229 pub wanted_subjects: HashSet<String>, 230 230 #[serde(default)] 231 231 + pub wanted_subject_prefixes: HashSet<String>, 232 232 + #[serde(default)] 231 233 pub wanted_subject_dids: HashSet<String>, 232 234 #[serde(default)] 233 235 pub wanted_sources: HashSet<String>, ··· 242 244 /// 243 245 /// The at-uri must be url-encoded 244 246 /// 245 245 - /// Pass this parameter multiple times to specify multiple collections, like 247 247 + /// Pass this parameter multiple times to specify multiple subjects, like 246 248 /// `wantedSubjects=[...]&wantedSubjects=[...]` 247 249 pub wanted_subjects: String, 250 250 + /// One or more at-uri, URI, or DID prefixes to receive links about 251 251 + /// 252 252 + /// The uri must be url-encoded 253 253 + /// 254 254 + /// Pass this parameter multiple times to specify multiple prefixes, like 255 255 + /// `wantedSubjectPrefixes=[...]&wantedSubjectPrefixes=[...]` 256 256 + pub wanted_subject_prefixes: String, 248 257 /// One or more DIDs to receive links about 249 258 /// 250 250 - /// Pass this parameter multiple times to specify multiple collections 259 259 + /// Pass this parameter multiple times to specify multiple subjects 251 260 pub wanted_subject_dids: String, 252 261 /// One or more link sources to receive links about 253 262 ///

+10 -1

spacedust/src/subscriber.rs

··· 124 124 let query = &self.query; 125 125 126 126 // subject + subject DIDs are logical OR 127 127 - if !(query.wanted_subjects.is_empty() && query.wanted_subject_dids.is_empty() 127 127 + if !(query.wanted_subjects.is_empty() 128 128 + && query.wanted_subject_prefixes.is_empty() 129 129 + && query.wanted_subject_dids.is_empty() 128 130 || query.wanted_subjects.contains(&properties.subject) 131 131 + || query 132 132 + .wanted_subject_prefixes 133 133 + .iter() 134 134 + .any(|p| properties.subject.starts_with(p)) 129 135 || properties 130 136 .subject_did 131 137 .as_ref() ··· 154 160 } 155 161 if opts.wanted_subject_dids.len() > 10_000 { 156 162 return Err(SubscriberUpdateError::TooManyDidsWanted); 163 163 + } 164 164 + if opts.wanted_subject_prefixes.len() > 100 { 165 165 + return Err(SubscriberUpdateError::TooManySubjectPrefixesWanted); 157 166 } 158 167 if opts.wanted_subjects.len() > 50_000 { 159 168 return Err(SubscriberUpdateError::TooManySubjectsWanted);

+1 -1

ufos/Cargo.toml

··· 13 13 clap = { version = "4.5.31", features = ["derive"] } 14 14 dropshot = "0.16.0" 15 15 env_logger = "0.11.7" 16 16 - fjall = { git = "https://github.com/fjall-rs/fjall.git", features = ["lz4"] } 16 16 + fjall = { git = "https://github.com/fjall-rs/fjall.git", rev = "fb229572bb7d1d6966a596994dc1708e47ec57d8", features = ["lz4"] } 17 17 getrandom = "0.3.3" 18 18 http = "1.3.1" 19 19 jetstream = { path = "../jetstream", features = ["metrics"] }