hotsocket.fyi
forked from
microcosm.blue/microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
event shape and link filtering and stuff
bad-example.com
+4
Cargo.lock
+4
Cargo.lock
···
3381
name = "spacedust"
3382
version = "0.1.0"
3383
dependencies = [
3384
+
"async-trait",
3385
"clap",
3386
"dropshot",
3387
"futures",
3388
+
"http",
3389
"jetstream",
3390
"links",
3391
"metrics",
3392
"schemars",
3393
+
"semver",
3394
"serde",
3395
"serde_json",
3396
+
"serde_qs",
3397
"tinyjson",
3398
"tokio",
3399
"tokio-tungstenite 0.27.0",
+4
spacedust/Cargo.toml
+4
spacedust/Cargo.toml
···
4
edition = "2024"
5
6
[dependencies]
7
clap = { version = "4.5.40", features = ["derive"] }
8
dropshot = "0.16.2"
9
futures = "0.3.31"
10
jetstream = { path = "../jetstream", features = ["metrics"] }
11
links = { path = "../links" }
12
metrics = "0.24.2"
13
schemars = "0.8.22"
14
serde = { version = "1.0.219", features = ["derive"] }
15
serde_json = "1.0.140"
16
tinyjson = "2.5.1"
17
tokio = { version = "1.45.1", features = ["full"] }
18
tokio-tungstenite = "0.27.0"
···
4
edition = "2024"
5
6
[dependencies]
7
+
async-trait = "0.1.88"
8
clap = { version = "4.5.40", features = ["derive"] }
9
dropshot = "0.16.2"
10
futures = "0.3.31"
11
+
http = "1.3.1"
12
jetstream = { path = "../jetstream", features = ["metrics"] }
13
links = { path = "../links" }
14
metrics = "0.24.2"
15
schemars = "0.8.22"
16
+
semver = "1.0.26"
17
serde = { version = "1.0.219", features = ["derive"] }
18
serde_json = "1.0.140"
19
+
serde_qs = "1.0.0-rc.3"
20
tinyjson = "2.5.1"
21
tokio = { version = "1.45.1", features = ["full"] }
22
tokio-tungstenite = "0.27.0"
+1
spacedust/src/consumer.rs
+1
spacedust/src/consumer.rs
+11
spacedust/src/lib.rs
+11
spacedust/src/lib.rs
···
1
pub mod consumer;
2
pub mod server;
3
4
use serde::Serialize;
5
···
9
path: String,
10
origin: String,
11
target: String,
12
}
13
14
#[derive(Debug, Serialize)]
···
16
operation: String,
17
source: String,
18
source_record: String,
19
subject: String,
20
// TODO: include the record too? would save clients a level of hydration
21
}
···
30
operation: "create".to_string(),
31
source: format!("{}:{undotted}", link.collection),
32
source_record: link.origin,
33
subject: link.target,
34
}
35
}
···
1
pub mod consumer;
2
pub mod server;
3
+
pub mod subscriber;
4
5
use serde::Serialize;
6
···
10
path: String,
11
origin: String,
12
target: String,
13
+
rev: String,
14
+
}
15
+
16
+
#[derive(Debug, Serialize)]
17
+
#[serde(rename_all="snake_case")]
18
+
pub struct ClientEvent {
19
+
kind: String,
20
+
link: ClientLinkEvent,
21
}
22
23
#[derive(Debug, Serialize)]
···
25
operation: String,
26
source: String,
27
source_record: String,
28
+
source_rev: String,
29
subject: String,
30
// TODO: include the record too? would save clients a level of hydration
31
}
···
40
operation: "create".to_string(),
41
source: format!("{}:{undotted}", link.collection),
42
source_record: link.origin,
43
+
source_rev: link.rev,
44
subject: link.target,
45
}
46
}
+244
-41
spacedust/src/server.rs
+244
-41
spacedust/src/server.rs
···
1
-
use crate::{ClientLinkEvent, LinkEvent};
2
use dropshot::{
3
ApiDescription, ConfigDropshot, ConfigLogging, ConfigLoggingLevel, Query, RequestContext,
4
-
ServerBuilder, WebsocketConnection, channel,
5
};
6
-
use futures::SinkExt;
7
use schemars::JsonSchema;
8
use serde::{Deserialize, Serialize};
9
use tokio::sync::broadcast;
10
-
use tokio_tungstenite::tungstenite::Message;
11
use tokio_tungstenite::tungstenite::protocol::Role;
12
13
pub async fn serve(b: broadcast::Sender<LinkEvent>) -> Result<(), String> {
14
let config_logging = ConfigLogging::StderrTerminal {
···
20
.map_err(|error| format!("failed to create logger: {}", error))?;
21
22
let mut api = ApiDescription::new();
23
api.register(subscribe).unwrap();
24
25
-
let server = ServerBuilder::new(api, b, log)
26
.config(ConfigDropshot {
27
bind_address: "0.0.0.0:9998".parse().unwrap(),
28
..Default::default()
···
33
server.await
34
}
35
36
-
#[derive(Debug, Serialize)]
37
-
#[serde(rename_all="snake_case")]
38
-
struct ClientEvent {
39
-
r#type: String,
40
-
link: ClientLinkEvent,
41
}
42
43
#[derive(Deserialize, JsonSchema)]
···
50
path = "/subscribe",
51
}]
52
async fn subscribe(
53
-
ctx: RequestContext<broadcast::Sender<LinkEvent>>,
54
-
_qp: Query<QueryParams>,
55
upgraded: WebsocketConnection,
56
) -> dropshot::WebsocketChannelResult {
57
-
let mut ws = tokio_tungstenite::WebSocketStream::from_raw_socket(
58
upgraded.into_inner(),
59
Role::Server,
60
None,
61
)
62
.await;
63
-
let mut sub = ctx.context().subscribe();
64
65
-
// TODO: pingpong
66
-
// TODO: filtering subscription
67
68
-
loop {
69
-
match sub.recv().await {
70
-
Ok(link) => {
71
-
let ev = ClientEvent {
72
-
r#type: "link".to_string(),
73
-
link: link.into(),
74
-
};
75
-
let json = serde_json::to_string(&ev)?;
76
-
if let Err(e) = ws.send(Message::Text(json.into())).await {
77
-
eprintln!("client: failed to send event: {e:?}");
78
-
ws.close(None).await?; // TODO: do we need this one??
79
-
break;
80
-
}
81
-
}
82
-
Err(broadcast::error::RecvError::Closed) => {
83
-
ws.close(None).await?; // TODO: send reason
84
-
break;
85
-
}
86
-
Err(broadcast::error::RecvError::Lagged(_n_missed)) => {
87
-
eprintln!("client lagged, closing");
88
-
ws.close(None).await?; // TODO: send reason
89
-
break;
90
-
}
91
-
}
92
-
}
93
Ok(())
94
}
···
1
+
use crate::subscriber;
2
+
use metrics::{histogram, counter};
3
+
use std::sync::Arc;
4
+
use crate::LinkEvent;
5
+
use http::{
6
+
header::{ORIGIN, USER_AGENT},
7
+
Response, StatusCode,
8
+
};
9
use dropshot::{
10
+
Body,
11
ApiDescription, ConfigDropshot, ConfigLogging, ConfigLoggingLevel, Query, RequestContext,
12
+
ServerBuilder, WebsocketConnection, channel, endpoint, HttpResponse,
13
+
ApiEndpointBodyContentType, ExtractorMetadata, HttpError, ServerContext,
14
+
SharedExtractor,
15
};
16
+
17
use schemars::JsonSchema;
18
use serde::{Deserialize, Serialize};
19
use tokio::sync::broadcast;
20
+
use tokio::time::Instant;
21
use tokio_tungstenite::tungstenite::protocol::Role;
22
+
use async_trait::async_trait;
23
+
use std::collections::HashSet;
24
+
25
+
const INDEX_HTML: &str = include_str!("../static/index.html");
26
+
const FAVICON: &[u8] = include_bytes!("../static/favicon.ico");
27
28
pub async fn serve(b: broadcast::Sender<LinkEvent>) -> Result<(), String> {
29
let config_logging = ConfigLogging::StderrTerminal {
···
35
.map_err(|error| format!("failed to create logger: {}", error))?;
36
37
let mut api = ApiDescription::new();
38
+
api.register(index).unwrap();
39
+
api.register(favicon).unwrap();
40
+
api.register(openapi).unwrap();
41
api.register(subscribe).unwrap();
42
43
+
// TODO: put spec in a once cell / lazy lock thing?
44
+
let spec = Arc::new(
45
+
api.openapi(
46
+
"Spacedust",
47
+
env!("CARGO_PKG_VERSION")
48
+
.parse()
49
+
.inspect_err(|e| {
50
+
eprintln!("failed to parse cargo package version for openapi: {e:?}")
51
+
})
52
+
.unwrap_or(semver::Version::new(0, 0, 1)),
53
+
)
54
+
.description("A configurable ATProto notifications firehose.")
55
+
.contact_name("part of @microcosm.blue")
56
+
.contact_url("https://microcosm.blue")
57
+
.json()
58
+
.map_err(|e| e.to_string())?,
59
+
);
60
+
61
+
let ctx = Context { spec, b };
62
+
63
+
let server = ServerBuilder::new(api, ctx, log)
64
.config(ConfigDropshot {
65
bind_address: "0.0.0.0:9998".parse().unwrap(),
66
..Default::default()
···
71
server.await
72
}
73
74
+
#[derive(Debug, Clone)]
75
+
struct Context {
76
+
pub spec: Arc<serde_json::Value>,
77
+
pub b: broadcast::Sender<LinkEvent>,
78
+
}
79
+
80
+
async fn instrument_handler<T, H, R>(ctx: &RequestContext<T>, handler: H) -> Result<R, HttpError>
81
+
where
82
+
R: HttpResponse,
83
+
H: Future<Output = Result<R, HttpError>>,
84
+
T: ServerContext,
85
+
{
86
+
let start = Instant::now();
87
+
let result = handler.await;
88
+
let latency = start.elapsed();
89
+
let status_code = match &result {
90
+
Ok(response) => response.status_code(),
91
+
Err(e) => e.status_code.as_status(),
92
+
}
93
+
.as_str() // just the number (.to_string()'s Display does eg `200 OK`)
94
+
.to_string();
95
+
let endpoint = ctx.endpoint.operation_id.clone();
96
+
let headers = ctx.request.headers();
97
+
let origin = headers
98
+
.get(ORIGIN)
99
+
.and_then(|v| v.to_str().ok())
100
+
.unwrap_or("")
101
+
.to_string();
102
+
let ua = headers
103
+
.get(USER_AGENT)
104
+
.and_then(|v| v.to_str().ok())
105
+
.map(|ua| {
106
+
if ua.starts_with("Mozilla/5.0 ") {
107
+
"browser"
108
+
} else {
109
+
ua
110
+
}
111
+
})
112
+
.unwrap_or("")
113
+
.to_string();
114
+
counter!("server_requests_total",
115
+
"endpoint" => endpoint.clone(),
116
+
"origin" => origin,
117
+
"ua" => ua,
118
+
"status_code" => status_code,
119
+
)
120
+
.increment(1);
121
+
histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64);
122
+
result
123
+
}
124
+
125
+
use dropshot::{HttpResponseHeaders, HttpResponseOk};
126
+
127
+
pub type OkCorsResponse<T> = Result<HttpResponseHeaders<HttpResponseOk<T>>, HttpError>;
128
+
129
+
/// Helper for constructing Ok responses: return OkCors(T).into()
130
+
/// (not happy with this yet)
131
+
pub struct OkCors<T: Serialize + JsonSchema + Send + Sync>(pub T);
132
+
133
+
impl<T> From<OkCors<T>> for OkCorsResponse<T>
134
+
where
135
+
T: Serialize + JsonSchema + Send + Sync,
136
+
{
137
+
fn from(ok: OkCors<T>) -> OkCorsResponse<T> {
138
+
let mut res = HttpResponseHeaders::new_unnamed(HttpResponseOk(ok.0));
139
+
res.headers_mut()
140
+
.insert("access-control-allow-origin", "*".parse().unwrap());
141
+
Ok(res)
142
+
}
143
+
}
144
+
145
+
// TODO: cors for HttpError
146
+
147
+
148
+
/// Serve index page as html
149
+
#[endpoint {
150
+
method = GET,
151
+
path = "/",
152
+
/*
153
+
* not useful to have this in openapi
154
+
*/
155
+
unpublished = true,
156
+
}]
157
+
async fn index(ctx: RequestContext<Context>) -> Result<Response<Body>, HttpError> {
158
+
instrument_handler(&ctx, async {
159
+
Ok(Response::builder()
160
+
.status(StatusCode::OK)
161
+
.header(http::header::CONTENT_TYPE, "text/html")
162
+
.body(INDEX_HTML.into())?)
163
+
})
164
+
.await
165
+
}
166
+
167
+
/// Serve index page as html
168
+
#[endpoint {
169
+
method = GET,
170
+
path = "/favicon.ico",
171
+
/*
172
+
* not useful to have this in openapi
173
+
*/
174
+
unpublished = true,
175
+
}]
176
+
async fn favicon(ctx: RequestContext<Context>) -> Result<Response<Body>, HttpError> {
177
+
instrument_handler(&ctx, async {
178
+
Ok(Response::builder()
179
+
.status(StatusCode::OK)
180
+
.header(http::header::CONTENT_TYPE, "image/x-icon")
181
+
.body(FAVICON.to_vec().into())?)
182
+
})
183
+
.await
184
+
}
185
+
186
+
/// Meta: get the openapi spec for this api
187
+
#[endpoint {
188
+
method = GET,
189
+
path = "/openapi",
190
+
/*
191
+
* not useful to have this in openapi
192
+
*/
193
+
unpublished = true,
194
+
}]
195
+
async fn openapi(ctx: RequestContext<Context>) -> OkCorsResponse<serde_json::Value> {
196
+
instrument_handler(&ctx, async {
197
+
let spec = (*ctx.context().spec).clone();
198
+
OkCors(spec).into()
199
+
})
200
+
.await
201
+
}
202
+
203
+
/// The real type that gets deserialized
204
+
#[derive(Debug, Deserialize, JsonSchema)]
205
+
#[serde(rename_all = "camelCase")]
206
+
pub struct MultiSubscribeQuery {
207
+
#[serde(default)]
208
+
pub wanted_subjects: HashSet<String>,
209
+
#[serde(default)]
210
+
pub wanted_subject_dids: HashSet<String>,
211
+
#[serde(default)]
212
+
pub wanted_sources: HashSet<String>,
213
+
}
214
+
/// The fake corresponding type for docs that dropshot won't freak out about a
215
+
/// vec for
216
+
#[derive(Deserialize, JsonSchema)]
217
+
#[allow(dead_code)]
218
+
#[serde(rename_all = "camelCase")]
219
+
struct MultiSubscribeQueryForDocs {
220
+
/// One or more at-uris to receive links about
221
+
///
222
+
/// The at-uri must be url-encoded
223
+
///
224
+
/// Pass this parameter multiple times to specify multiple collections, like
225
+
/// `wantedSubjects=[...]&wantedSubjects=[...]`
226
+
pub wanted_subjects: String,
227
+
/// One or more DIDs to receive links about
228
+
///
229
+
/// Pass this parameter multiple times to specify multiple collections
230
+
pub wanted_subject_dids: String,
231
+
/// One or more link sources to receive links about
232
+
///
233
+
/// TODO: docs about link sources
234
+
///
235
+
/// eg, a bluesky like's link source: `app.bsky.feed.like:subject.uri`
236
+
///
237
+
/// Pass this parameter multiple times to specify multiple sources
238
+
pub wanted_sources: String,
239
+
}
240
+
241
+
// The `SharedExtractor` implementation for Query<QueryType> describes how to
242
+
// construct an instance of `Query<QueryType>` from an HTTP request: namely, by
243
+
// parsing the query string to an instance of `QueryType`.
244
+
#[async_trait]
245
+
impl SharedExtractor for MultiSubscribeQuery {
246
+
async fn from_request<Context: ServerContext>(
247
+
ctx: &RequestContext<Context>,
248
+
) -> Result<MultiSubscribeQuery, HttpError> {
249
+
let raw_query = ctx.request.uri().query().unwrap_or("");
250
+
let q = serde_qs::from_str(raw_query).map_err(|e| {
251
+
HttpError::for_bad_request(None, format!("unable to parse query string: {}", e))
252
+
})?;
253
+
Ok(q)
254
+
}
255
+
256
+
fn metadata(body_content_type: ApiEndpointBodyContentType) -> ExtractorMetadata {
257
+
// HACK: query type switcheroo: passing MultiSubscribeQuery to
258
+
// `metadata` would "helpfully" panic because dropshot believes we can
259
+
// only have scalar types in a query.
260
+
//
261
+
// so instead we have a fake second type whose only job is to look the
262
+
// same as MultiSubscribeQuery exept that it has `String` instead of
263
+
// `Vec<String>`, which dropshot will accept, and generate ~close-enough
264
+
// docs for.
265
+
<Query<MultiSubscribeQueryForDocs> as SharedExtractor>::metadata(body_content_type)
266
+
}
267
}
268
269
#[derive(Deserialize, JsonSchema)]
···
276
path = "/subscribe",
277
}]
278
async fn subscribe(
279
+
ctx: RequestContext<Context>,
280
+
query: MultiSubscribeQuery,
281
upgraded: WebsocketConnection,
282
) -> dropshot::WebsocketChannelResult {
283
+
let ws = tokio_tungstenite::WebSocketStream::from_raw_socket(
284
upgraded.into_inner(),
285
Role::Server,
286
None,
287
)
288
.await;
289
290
+
let b = ctx.context().b.subscribe();
291
292
+
subscriber::subscribe(b, ws, query)
293
+
.await
294
+
.map_err(|e| format!("boo: {e:?}"))?;
295
+
296
Ok(())
297
}
+75
spacedust/src/subscriber.rs
+75
spacedust/src/subscriber.rs
···
···
1
+
use crate::ClientEvent;
2
+
use crate::LinkEvent;
3
+
use crate::server::MultiSubscribeQuery;
4
+
use futures::SinkExt;
5
+
use std::error::Error;
6
+
use tokio::sync::broadcast;
7
+
use tokio_tungstenite::{WebSocketStream, tungstenite::Message};
8
+
use dropshot::WebsocketConnectionRaw;
9
+
10
+
pub async fn subscribe(
11
+
mut sub: broadcast::Receiver<LinkEvent>,
12
+
mut ws: WebSocketStream<WebsocketConnectionRaw>,
13
+
query: MultiSubscribeQuery,
14
+
) -> Result<(), Box<dyn Error>> {
15
+
// TODO: pingpong
16
+
17
+
loop {
18
+
match sub.recv().await {
19
+
Ok(link) => {
20
+
21
+
// subject + subject DIDs are logical OR
22
+
let target_did = if link.target.starts_with("did:") {
23
+
link.target.clone()
24
+
} else {
25
+
let Some(rest) = link.target.strip_prefix("at://") else {
26
+
continue;
27
+
};
28
+
if let Some((did, _)) = rest.split_once("/") {
29
+
did
30
+
} else {
31
+
rest
32
+
}.to_string()
33
+
};
34
+
if !(query.wanted_subjects.contains(&link.target) || query.wanted_subject_dids.contains(&target_did) || query.wanted_subjects.is_empty() && query.wanted_subject_dids.is_empty()) {
35
+
// wowwww ^^ fix that
36
+
continue;
37
+
}
38
+
39
+
// subjects together with sources are logical AND
40
+
41
+
if !query.wanted_sources.is_empty() {
42
+
let undotted = link.path.strip_prefix('.').unwrap_or_else(|| {
43
+
eprintln!("link path did not have expected '.' prefix: {}", link.path);
44
+
""
45
+
});
46
+
let source = format!("{}:{undotted}", link.collection);
47
+
if !query.wanted_sources.contains(&source) {
48
+
continue;
49
+
}
50
+
}
51
+
52
+
let ev = ClientEvent {
53
+
kind: "link".to_string(),
54
+
link: link.into(),
55
+
};
56
+
let json = serde_json::to_string(&ev)?;
57
+
if let Err(e) = ws.send(Message::Text(json.into())).await {
58
+
eprintln!("client: failed to send event: {e:?}");
59
+
ws.close(None).await?; // TODO: do we need this one??
60
+
break;
61
+
}
62
+
}
63
+
Err(broadcast::error::RecvError::Closed) => {
64
+
ws.close(None).await?; // TODO: send reason
65
+
break;
66
+
}
67
+
Err(broadcast::error::RecvError::Lagged(_n_missed)) => {
68
+
eprintln!("client lagged, closing");
69
+
ws.close(None).await?; // TODO: send reason
70
+
break;
71
+
}
72
+
}
73
+
}
74
+
Ok(())
75
+
}
spacedust/static/favicon.ico
spacedust/static/favicon.ico
This is a binary file and will not be displayed.
+54
spacedust/static/index.html
+54
spacedust/static/index.html
···
···
1
+
<!doctype html>
2
+
<html lang="en">
3
+
<head>
4
+
<meta charset="utf-8" />
5
+
<title>Spacedust documentation</title>
6
+
<meta name="viewport" content="width=device-width, initial-scale=1" />
7
+
<meta name="description" content="API Documentation for Spacedust, a configurable ATProto notifications firehose" />
8
+
<style>
9
+
.custom-header {
10
+
height: 42px;
11
+
background-color: #221828;
12
+
box-shadow: inset 0 -1px 0 var(--scalar-border-color);
13
+
color: var(--scalar-color-1);
14
+
font-size: var(--scalar-font-size-3);
15
+
font-family: 'Iowan Old Style', 'Palatino Linotype', 'URW Palladio L', P052, serif;
16
+
padding: 0 18px;
17
+
justify-content: space-between;
18
+
}
19
+
.custom-header,
20
+
.custom-header nav {
21
+
display: flex;
22
+
align-items: center;
23
+
gap: 18px;
24
+
}
25
+
.custom-header a:hover {
26
+
color: var(--scalar-color-2);
27
+
}
28
+
</style>
29
+
</head>
30
+
<body>
31
+
<header class="custom-header scalar-app">
32
+
<p>
33
+
TODO: pdsls jetstream link
34
+
<a href="https://ufos.microcosm.blue">Launch 🛸 UFOs app</a>: Explore lexicons
35
+
</p>
36
+
<nav>
37
+
<b>a <a href="https://microcosm.blue">microcosm</a> project</b>
38
+
<a href="https://bsky.app/profile/microcosm.blue">@microcosm.blue</a>
39
+
<a href="https://github.com/at-microcosm">github</a>
40
+
</nav>
41
+
</header>
42
+
43
+
<script id="api-reference" type="application/json" data-url="/openapi""></script>
44
+
45
+
<script>
46
+
var configuration = {
47
+
theme: 'purple',
48
+
}
49
+
document.getElementById('api-reference').dataset.configuration = JSON.stringify(configuration)
50
+
</script>
51
+
52
+
<script src="https://cdn.jsdelivr.net/npm/@scalar/api-reference"></script>
53
+
</body>
54
+
</html>
+1
-1
ufos/src/index_html.rs
+1
-1
ufos/src/index_html.rs