+4

Cargo.lock

··· 3381 3381 name = "spacedust" 3382 3382 version = "0.1.0" 3383 3383 dependencies = [ 3384 3384 + "async-trait", 3384 3385 "clap", 3385 3386 "dropshot", 3386 3387 "futures", 3388 3388 + "http", 3387 3389 "jetstream", 3388 3390 "links", 3389 3391 "metrics", 3390 3392 "schemars", 3393 3393 + "semver", 3391 3394 "serde", 3392 3395 "serde_json", 3396 3396 + "serde_qs", 3393 3397 "tinyjson", 3394 3398 "tokio", 3395 3399 "tokio-tungstenite 0.27.0",

+4

spacedust/Cargo.toml

··· 4 4 edition = "2024" 5 5 6 6 [dependencies] 7 7 + async-trait = "0.1.88" 7 8 clap = { version = "4.5.40", features = ["derive"] } 8 9 dropshot = "0.16.2" 9 10 futures = "0.3.31" 11 11 + http = "1.3.1" 10 12 jetstream = { path = "../jetstream", features = ["metrics"] } 11 13 links = { path = "../links" } 12 14 metrics = "0.24.2" 13 15 schemars = "0.8.22" 16 16 + semver = "1.0.26" 14 17 serde = { version = "1.0.219", features = ["derive"] } 15 18 serde_json = "1.0.140" 19 19 + serde_qs = "1.0.0-rc.3" 16 20 tinyjson = "2.5.1" 17 21 tokio = { version = "1.45.1", features = ["full"] } 18 22 tokio-tungstenite = "0.27.0"

+1

spacedust/src/consumer.rs

··· 72 72 &*commit.collection, 73 73 &*commit.rkey, 74 74 ), 75 75 + rev: commit.rev.to_string(), 75 76 target: link.target.into_string(), 76 77 }; 77 78 let _ = b.send(link_ev); // only errors if no subscribers are connected, which is just fine.

+11

spacedust/src/lib.rs

··· 1 1 pub mod consumer; 2 2 pub mod server; 3 3 + pub mod subscriber; 3 4 4 5 use serde::Serialize; 5 6 ··· 9 10 path: String, 10 11 origin: String, 11 12 target: String, 13 13 + rev: String, 14 14 + } 15 15 + 16 16 + #[derive(Debug, Serialize)] 17 17 + #[serde(rename_all="snake_case")] 18 18 + pub struct ClientEvent { 19 19 + kind: String, 20 20 + link: ClientLinkEvent, 12 21 } 13 22 14 23 #[derive(Debug, Serialize)] ··· 16 25 operation: String, 17 26 source: String, 18 27 source_record: String, 28 28 + source_rev: String, 19 29 subject: String, 20 30 // TODO: include the record too? would save clients a level of hydration 21 31 } ··· 30 40 operation: "create".to_string(), 31 41 source: format!("{}:{undotted}", link.collection), 32 42 source_record: link.origin, 43 43 + source_rev: link.rev, 33 44 subject: link.target, 34 45 } 35 46 }

+244 -41

spacedust/src/server.rs

··· 1 1 - use crate::{ClientLinkEvent, LinkEvent}; 1 1 + use crate::subscriber; 2 2 + use metrics::{histogram, counter}; 3 3 + use std::sync::Arc; 4 4 + use crate::LinkEvent; 5 5 + use http::{ 6 6 + header::{ORIGIN, USER_AGENT}, 7 7 + Response, StatusCode, 8 8 + }; 2 9 use dropshot::{ 10 10 + Body, 3 11 ApiDescription, ConfigDropshot, ConfigLogging, ConfigLoggingLevel, Query, RequestContext, 4 4 - ServerBuilder, WebsocketConnection, channel, 12 12 + ServerBuilder, WebsocketConnection, channel, endpoint, HttpResponse, 13 13 + ApiEndpointBodyContentType, ExtractorMetadata, HttpError, ServerContext, 14 14 + SharedExtractor, 5 15 }; 6 6 - use futures::SinkExt; 16 16 + 7 17 use schemars::JsonSchema; 8 18 use serde::{Deserialize, Serialize}; 9 19 use tokio::sync::broadcast; 10 10 - use tokio_tungstenite::tungstenite::Message; 20 20 + use tokio::time::Instant; 11 21 use tokio_tungstenite::tungstenite::protocol::Role; 22 22 + use async_trait::async_trait; 23 23 + use std::collections::HashSet; 24 24 + 25 25 + const INDEX_HTML: &str = include_str!("../static/index.html"); 26 26 + const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); 12 27 13 28 pub async fn serve(b: broadcast::Sender<LinkEvent>) -> Result<(), String> { 14 29 let config_logging = ConfigLogging::StderrTerminal { ··· 20 35 .map_err(|error| format!("failed to create logger: {}", error))?; 21 36 22 37 let mut api = ApiDescription::new(); 38 38 + api.register(index).unwrap(); 39 39 + api.register(favicon).unwrap(); 40 40 + api.register(openapi).unwrap(); 23 41 api.register(subscribe).unwrap(); 24 42 25 25 - let server = ServerBuilder::new(api, b, log) 43 43 + // TODO: put spec in a once cell / lazy lock thing? 44 44 + let spec = Arc::new( 45 45 + api.openapi( 46 46 + "Spacedust", 47 47 + env!("CARGO_PKG_VERSION") 48 48 + .parse() 49 49 + .inspect_err(|e| { 50 50 + eprintln!("failed to parse cargo package version for openapi: {e:?}") 51 51 + }) 52 52 + .unwrap_or(semver::Version::new(0, 0, 1)), 53 53 + ) 54 54 + .description("A configurable ATProto notifications firehose.") 55 55 + .contact_name("part of @microcosm.blue") 56 56 + .contact_url("https://microcosm.blue") 57 57 + .json() 58 58 + .map_err(|e| e.to_string())?, 59 59 + ); 60 60 + 61 61 + let ctx = Context { spec, b }; 62 62 + 63 63 + let server = ServerBuilder::new(api, ctx, log) 26 64 .config(ConfigDropshot { 27 65 bind_address: "0.0.0.0:9998".parse().unwrap(), 28 66 ..Default::default() ··· 33 71 server.await 34 72 } 35 73 36 36 - #[derive(Debug, Serialize)] 37 37 - #[serde(rename_all="snake_case")] 38 38 - struct ClientEvent { 39 39 - r#type: String, 40 40 - link: ClientLinkEvent, 74 74 + #[derive(Debug, Clone)] 75 75 + struct Context { 76 76 + pub spec: Arc<serde_json::Value>, 77 77 + pub b: broadcast::Sender<LinkEvent>, 78 78 + } 79 79 + 80 80 + async fn instrument_handler<T, H, R>(ctx: &RequestContext<T>, handler: H) -> Result<R, HttpError> 81 81 + where 82 82 + R: HttpResponse, 83 83 + H: Future<Output = Result<R, HttpError>>, 84 84 + T: ServerContext, 85 85 + { 86 86 + let start = Instant::now(); 87 87 + let result = handler.await; 88 88 + let latency = start.elapsed(); 89 89 + let status_code = match &result { 90 90 + Ok(response) => response.status_code(), 91 91 + Err(e) => e.status_code.as_status(), 92 92 + } 93 93 + .as_str() // just the number (.to_string()'s Display does eg `200 OK`) 94 94 + .to_string(); 95 95 + let endpoint = ctx.endpoint.operation_id.clone(); 96 96 + let headers = ctx.request.headers(); 97 97 + let origin = headers 98 98 + .get(ORIGIN) 99 99 + .and_then(|v| v.to_str().ok()) 100 100 + .unwrap_or("") 101 101 + .to_string(); 102 102 + let ua = headers 103 103 + .get(USER_AGENT) 104 104 + .and_then(|v| v.to_str().ok()) 105 105 + .map(|ua| { 106 106 + if ua.starts_with("Mozilla/5.0 ") { 107 107 + "browser" 108 108 + } else { 109 109 + ua 110 110 + } 111 111 + }) 112 112 + .unwrap_or("") 113 113 + .to_string(); 114 114 + counter!("server_requests_total", 115 115 + "endpoint" => endpoint.clone(), 116 116 + "origin" => origin, 117 117 + "ua" => ua, 118 118 + "status_code" => status_code, 119 119 + ) 120 120 + .increment(1); 121 121 + histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64); 122 122 + result 123 123 + } 124 124 + 125 125 + use dropshot::{HttpResponseHeaders, HttpResponseOk}; 126 126 + 127 127 + pub type OkCorsResponse<T> = Result<HttpResponseHeaders<HttpResponseOk<T>>, HttpError>; 128 128 + 129 129 + /// Helper for constructing Ok responses: return OkCors(T).into() 130 130 + /// (not happy with this yet) 131 131 + pub struct OkCors<T: Serialize + JsonSchema + Send + Sync>(pub T); 132 132 + 133 133 + impl<T> From<OkCors<T>> for OkCorsResponse<T> 134 134 + where 135 135 + T: Serialize + JsonSchema + Send + Sync, 136 136 + { 137 137 + fn from(ok: OkCors<T>) -> OkCorsResponse<T> { 138 138 + let mut res = HttpResponseHeaders::new_unnamed(HttpResponseOk(ok.0)); 139 139 + res.headers_mut() 140 140 + .insert("access-control-allow-origin", "*".parse().unwrap()); 141 141 + Ok(res) 142 142 + } 143 143 + } 144 144 + 145 145 + // TODO: cors for HttpError 146 146 + 147 147 + 148 148 + /// Serve index page as html 149 149 + #[endpoint { 150 150 + method = GET, 151 151 + path = "/", 152 152 + /* 153 153 + * not useful to have this in openapi 154 154 + */ 155 155 + unpublished = true, 156 156 + }] 157 157 + async fn index(ctx: RequestContext<Context>) -> Result<Response<Body>, HttpError> { 158 158 + instrument_handler(&ctx, async { 159 159 + Ok(Response::builder() 160 160 + .status(StatusCode::OK) 161 161 + .header(http::header::CONTENT_TYPE, "text/html") 162 162 + .body(INDEX_HTML.into())?) 163 163 + }) 164 164 + .await 165 165 + } 166 166 + 167 167 + /// Serve index page as html 168 168 + #[endpoint { 169 169 + method = GET, 170 170 + path = "/favicon.ico", 171 171 + /* 172 172 + * not useful to have this in openapi 173 173 + */ 174 174 + unpublished = true, 175 175 + }] 176 176 + async fn favicon(ctx: RequestContext<Context>) -> Result<Response<Body>, HttpError> { 177 177 + instrument_handler(&ctx, async { 178 178 + Ok(Response::builder() 179 179 + .status(StatusCode::OK) 180 180 + .header(http::header::CONTENT_TYPE, "image/x-icon") 181 181 + .body(FAVICON.to_vec().into())?) 182 182 + }) 183 183 + .await 184 184 + } 185 185 + 186 186 + /// Meta: get the openapi spec for this api 187 187 + #[endpoint { 188 188 + method = GET, 189 189 + path = "/openapi", 190 190 + /* 191 191 + * not useful to have this in openapi 192 192 + */ 193 193 + unpublished = true, 194 194 + }] 195 195 + async fn openapi(ctx: RequestContext<Context>) -> OkCorsResponse<serde_json::Value> { 196 196 + instrument_handler(&ctx, async { 197 197 + let spec = (*ctx.context().spec).clone(); 198 198 + OkCors(spec).into() 199 199 + }) 200 200 + .await 201 201 + } 202 202 + 203 203 + /// The real type that gets deserialized 204 204 + #[derive(Debug, Deserialize, JsonSchema)] 205 205 + #[serde(rename_all = "camelCase")] 206 206 + pub struct MultiSubscribeQuery { 207 207 + #[serde(default)] 208 208 + pub wanted_subjects: HashSet<String>, 209 209 + #[serde(default)] 210 210 + pub wanted_subject_dids: HashSet<String>, 211 211 + #[serde(default)] 212 212 + pub wanted_sources: HashSet<String>, 213 213 + } 214 214 + /// The fake corresponding type for docs that dropshot won't freak out about a 215 215 + /// vec for 216 216 + #[derive(Deserialize, JsonSchema)] 217 217 + #[allow(dead_code)] 218 218 + #[serde(rename_all = "camelCase")] 219 219 + struct MultiSubscribeQueryForDocs { 220 220 + /// One or more at-uris to receive links about 221 221 + /// 222 222 + /// The at-uri must be url-encoded 223 223 + /// 224 224 + /// Pass this parameter multiple times to specify multiple collections, like 225 225 + /// `wantedSubjects=[...]&wantedSubjects=[...]` 226 226 + pub wanted_subjects: String, 227 227 + /// One or more DIDs to receive links about 228 228 + /// 229 229 + /// Pass this parameter multiple times to specify multiple collections 230 230 + pub wanted_subject_dids: String, 231 231 + /// One or more link sources to receive links about 232 232 + /// 233 233 + /// TODO: docs about link sources 234 234 + /// 235 235 + /// eg, a bluesky like's link source: `app.bsky.feed.like:subject.uri` 236 236 + /// 237 237 + /// Pass this parameter multiple times to specify multiple sources 238 238 + pub wanted_sources: String, 239 239 + } 240 240 + 241 241 + // The `SharedExtractor` implementation for Query<QueryType> describes how to 242 242 + // construct an instance of `Query<QueryType>` from an HTTP request: namely, by 243 243 + // parsing the query string to an instance of `QueryType`. 244 244 + #[async_trait] 245 245 + impl SharedExtractor for MultiSubscribeQuery { 246 246 + async fn from_request<Context: ServerContext>( 247 247 + ctx: &RequestContext<Context>, 248 248 + ) -> Result<MultiSubscribeQuery, HttpError> { 249 249 + let raw_query = ctx.request.uri().query().unwrap_or(""); 250 250 + let q = serde_qs::from_str(raw_query).map_err(|e| { 251 251 + HttpError::for_bad_request(None, format!("unable to parse query string: {}", e)) 252 252 + })?; 253 253 + Ok(q) 254 254 + } 255 255 + 256 256 + fn metadata(body_content_type: ApiEndpointBodyContentType) -> ExtractorMetadata { 257 257 + // HACK: query type switcheroo: passing MultiSubscribeQuery to 258 258 + // `metadata` would "helpfully" panic because dropshot believes we can 259 259 + // only have scalar types in a query. 260 260 + // 261 261 + // so instead we have a fake second type whose only job is to look the 262 262 + // same as MultiSubscribeQuery exept that it has `String` instead of 263 263 + // `Vec<String>`, which dropshot will accept, and generate ~close-enough 264 264 + // docs for. 265 265 + <Query<MultiSubscribeQueryForDocs> as SharedExtractor>::metadata(body_content_type) 266 266 + } 41 267 } 42 268 43 269 #[derive(Deserialize, JsonSchema)] ··· 50 276 path = "/subscribe", 51 277 }] 52 278 async fn subscribe( 53 53 - ctx: RequestContext<broadcast::Sender<LinkEvent>>, 54 54 - _qp: Query<QueryParams>, 279 279 + ctx: RequestContext<Context>, 280 280 + query: MultiSubscribeQuery, 55 281 upgraded: WebsocketConnection, 56 282 ) -> dropshot::WebsocketChannelResult { 57 57 - let mut ws = tokio_tungstenite::WebSocketStream::from_raw_socket( 283 283 + let ws = tokio_tungstenite::WebSocketStream::from_raw_socket( 58 284 upgraded.into_inner(), 59 285 Role::Server, 60 286 None, 61 287 ) 62 288 .await; 63 63 - let mut sub = ctx.context().subscribe(); 64 289 65 65 - // TODO: pingpong 66 66 - // TODO: filtering subscription 290 290 + let b = ctx.context().b.subscribe(); 67 291 68 68 - loop { 69 69 - match sub.recv().await { 70 70 - Ok(link) => { 71 71 - let ev = ClientEvent { 72 72 - r#type: "link".to_string(), 73 73 - link: link.into(), 74 74 - }; 75 75 - let json = serde_json::to_string(&ev)?; 76 76 - if let Err(e) = ws.send(Message::Text(json.into())).await { 77 77 - eprintln!("client: failed to send event: {e:?}"); 78 78 - ws.close(None).await?; // TODO: do we need this one?? 79 79 - break; 80 80 - } 81 81 - } 82 82 - Err(broadcast::error::RecvError::Closed) => { 83 83 - ws.close(None).await?; // TODO: send reason 84 84 - break; 85 85 - } 86 86 - Err(broadcast::error::RecvError::Lagged(_n_missed)) => { 87 87 - eprintln!("client lagged, closing"); 88 88 - ws.close(None).await?; // TODO: send reason 89 89 - break; 90 90 - } 91 91 - } 92 92 - } 292 292 + subscriber::subscribe(b, ws, query) 293 293 + .await 294 294 + .map_err(|e| format!("boo: {e:?}"))?; 295 295 + 93 296 Ok(()) 94 297 }

+75

spacedust/src/subscriber.rs

··· 1 1 + use crate::ClientEvent; 2 2 + use crate::LinkEvent; 3 3 + use crate::server::MultiSubscribeQuery; 4 4 + use futures::SinkExt; 5 5 + use std::error::Error; 6 6 + use tokio::sync::broadcast; 7 7 + use tokio_tungstenite::{WebSocketStream, tungstenite::Message}; 8 8 + use dropshot::WebsocketConnectionRaw; 9 9 + 10 10 + pub async fn subscribe( 11 11 + mut sub: broadcast::Receiver<LinkEvent>, 12 12 + mut ws: WebSocketStream<WebsocketConnectionRaw>, 13 13 + query: MultiSubscribeQuery, 14 14 + ) -> Result<(), Box<dyn Error>> { 15 15 + // TODO: pingpong 16 16 + 17 17 + loop { 18 18 + match sub.recv().await { 19 19 + Ok(link) => { 20 20 + 21 21 + // subject + subject DIDs are logical OR 22 22 + let target_did = if link.target.starts_with("did:") { 23 23 + link.target.clone() 24 24 + } else { 25 25 + let Some(rest) = link.target.strip_prefix("at://") else { 26 26 + continue; 27 27 + }; 28 28 + if let Some((did, _)) = rest.split_once("/") { 29 29 + did 30 30 + } else { 31 31 + rest 32 32 + }.to_string() 33 33 + }; 34 34 + if !(query.wanted_subjects.contains(&link.target) || query.wanted_subject_dids.contains(&target_did) || query.wanted_subjects.is_empty() && query.wanted_subject_dids.is_empty()) { 35 35 + // wowwww ^^ fix that 36 36 + continue; 37 37 + } 38 38 + 39 39 + // subjects together with sources are logical AND 40 40 + 41 41 + if !query.wanted_sources.is_empty() { 42 42 + let undotted = link.path.strip_prefix('.').unwrap_or_else(|| { 43 43 + eprintln!("link path did not have expected '.' prefix: {}", link.path); 44 44 + "" 45 45 + }); 46 46 + let source = format!("{}:{undotted}", link.collection); 47 47 + if !query.wanted_sources.contains(&source) { 48 48 + continue; 49 49 + } 50 50 + } 51 51 + 52 52 + let ev = ClientEvent { 53 53 + kind: "link".to_string(), 54 54 + link: link.into(), 55 55 + }; 56 56 + let json = serde_json::to_string(&ev)?; 57 57 + if let Err(e) = ws.send(Message::Text(json.into())).await { 58 58 + eprintln!("client: failed to send event: {e:?}"); 59 59 + ws.close(None).await?; // TODO: do we need this one?? 60 60 + break; 61 61 + } 62 62 + } 63 63 + Err(broadcast::error::RecvError::Closed) => { 64 64 + ws.close(None).await?; // TODO: send reason 65 65 + break; 66 66 + } 67 67 + Err(broadcast::error::RecvError::Lagged(_n_missed)) => { 68 68 + eprintln!("client lagged, closing"); 69 69 + ws.close(None).await?; // TODO: send reason 70 70 + break; 71 71 + } 72 72 + } 73 73 + } 74 74 + Ok(()) 75 75 + }

spacedust/static/favicon.ico

This is a binary file and will not be displayed.

+54

spacedust/static/index.html

··· 1 1 + <!doctype html> 2 2 + <html lang="en"> 3 3 + <head> 4 4 + <meta charset="utf-8" /> 5 5 + <title>Spacedust documentation</title> 6 6 + <meta name="viewport" content="width=device-width, initial-scale=1" /> 7 7 + <meta name="description" content="API Documentation for Spacedust, a configurable ATProto notifications firehose" /> 8 8 + <style> 9 9 + .custom-header { 10 10 + height: 42px; 11 11 + background-color: #221828; 12 12 + box-shadow: inset 0 -1px 0 var(--scalar-border-color); 13 13 + color: var(--scalar-color-1); 14 14 + font-size: var(--scalar-font-size-3); 15 15 + font-family: 'Iowan Old Style', 'Palatino Linotype', 'URW Palladio L', P052, serif; 16 16 + padding: 0 18px; 17 17 + justify-content: space-between; 18 18 + } 19 19 + .custom-header, 20 20 + .custom-header nav { 21 21 + display: flex; 22 22 + align-items: center; 23 23 + gap: 18px; 24 24 + } 25 25 + .custom-header a:hover { 26 26 + color: var(--scalar-color-2); 27 27 + } 28 28 + </style> 29 29 + </head> 30 30 + <body> 31 31 + <header class="custom-header scalar-app"> 32 32 + <p> 33 33 + TODO: pdsls jetstream link 34 34 + <a href="https://ufos.microcosm.blue">Launch 🛸 UFOs app</a>: Explore lexicons 35 35 + </p> 36 36 + <nav> 37 37 + <b>a <a href="https://microcosm.blue">microcosm</a> project</b> 38 38 + <a href="https://bsky.app/profile/microcosm.blue">@microcosm.blue</a> 39 39 + <a href="https://github.com/at-microcosm">github</a> 40 40 + </nav> 41 41 + </header> 42 42 + 43 43 + <script id="api-reference" type="application/json" data-url="/openapi""></script> 44 44 + 45 45 + <script> 46 46 + var configuration = { 47 47 + theme: 'purple', 48 48 + } 49 49 + document.getElementById('api-reference').dataset.configuration = JSON.stringify(configuration) 50 50 + </script> 51 51 + 52 52 + <script src="https://cdn.jsdelivr.net/npm/@scalar/api-reference"></script> 53 53 + </body> 54 54 + </html>

+1 -1

ufos/src/index_html.rs

··· 2 2 <html lang="en"> 3 3 <head> 4 4 <meta charset="utf-8" /> 5 5 - <title>UFOs API Documentation</title> 5 5 + <title>UFOs API documentation</title> 6 6 <meta name="viewport" content="width=device-width, initial-scale=1" /> 7 7 <meta name="description" content="API Documentation for UFOs: Samples and stats for all atproto lexicons." /> 8 8 <style>