microcosm.blue / microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
fork atom

Web: stats and summary

bad-example.com bb6e4f9f c44ab956

+80 -18
unified split
+17
Cargo.lock
··· 110 110 checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" 111 111 112 112 [[package]] 113 + name = "arrayvec" 114 + version = "0.7.6" 115 + source = "registry+https://github.com/rust-lang/crates.io-index" 116 + checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" 117 + 118 + [[package]] 113 119 name = "askama" 114 120 version = "0.12.1" 115 121 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 513 519 "metrics", 514 520 "metrics-exporter-prometheus", 515 521 "metrics-process", 522 + "num-format", 516 523 "rocksdb", 517 524 "serde", 518 525 "serde_with", ··· 1394 1401 version = "0.1.0" 1395 1402 source = "registry+https://github.com/rust-lang/crates.io-index" 1396 1403 checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" 1404 + 1405 + [[package]] 1406 + name = "num-format" 1407 + version = "0.4.4" 1408 + source = "registry+https://github.com/rust-lang/crates.io-index" 1409 + checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" 1410 + dependencies = [ 1411 + "arrayvec", 1412 + "itoa", 1413 + ] 1397 1414 1398 1415 [[package]] 1399 1416 name = "num-traits"
+1
constellation/Cargo.toml
··· 20 20 metrics = "0.24.1" 21 21 metrics-exporter-prometheus = { version = "0.16.1", default-features = false, features = ["http-listener"] } 22 22 metrics-process = "2.4.0" 23 + num-format = "0.4.4" 23 24 rocksdb = { version = "0.23.0", optional = true } 24 25 serde = { version = "1.0.215", features = ["derive"] } 25 26 serde_with = { version = "3.12.0", features = ["hex"] }
+5
constellation/src/server/filters.rs
··· 1 1 use links::{parse_any_link, Link}; 2 + use num_format::{Locale, ToFormattedString}; 2 3 3 4 pub fn to_browseable(s: &str) -> askama::Result<Option<String>> { 4 5 Ok({ ··· 17 18 } 18 19 }) 19 20 } 21 + 22 + pub fn human_number(n: &u64) -> askama::Result<String> { 23 + Ok(n.to_formatted_string(&Locale::en)) 24 + }
+27 -6
constellation/src/server/mod.rs
··· 4 4 use serde::{Deserialize, Serialize}; 5 5 use serde_with::serde_as; 6 6 use std::collections::HashMap; 7 + use std::time::{UNIX_EPOCH, Duration}; 7 8 use tokio::net::{TcpListener, ToSocketAddrs}; 8 9 use tokio::task::block_in_place; 9 10 use tokio_util::sync::CancellationToken; 10 11 11 - use crate::storage::LinkReader; 12 + use crate::storage::{LinkReader, StorageStats}; 12 13 use constellation::{CountsByCount, Did, RecordId}; 13 14 14 15 mod acceptable; ··· 19 20 const DEFAULT_CURSOR_LIMIT: u64 = 16; 20 21 const DEFAULT_CURSOR_LIMIT_MAX: u64 = 100; 21 22 23 + const INDEX_BEGAN_AT_TS: u64 = 1738083600; // TODO: not this 24 + 25 + 22 26 pub async fn serve<S, A>(store: S, addr: A, stay_alive: CancellationToken) -> anyhow::Result<()> 23 27 where 24 28 S: LinkReader, 25 29 A: ToSocketAddrs, 26 30 { 27 31 let app = Router::new() 28 - .route("/", get(hello)) 32 + .route( 33 + "/", 34 + get({ 35 + let store = store.clone(); 36 + move |accept| async { block_in_place(|| hello(accept, store)) } 37 + }), 38 + ) 29 39 .route( 30 40 "/links/count", 31 41 get({ ··· 93 103 #[template(path = "hello.html.j2")] 94 104 struct HelloReponse { 95 105 help: &'static str, 106 + days_indexed: u64, 107 + stats: StorageStats, 96 108 } 97 - async fn hello(accept: ExtractAccept) -> impl IntoResponse { 98 - acceptable(accept, HelloReponse { 99 - help: "open this URL in a web browser (or request with Accept: text/html) for information about this API." 100 - }) 109 + fn hello(accept: ExtractAccept, store: impl LinkReader) -> Result<impl IntoResponse, http::StatusCode> { 110 + let stats = store 111 + .get_stats() 112 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 113 + let days_indexed = (UNIX_EPOCH + Duration::from_secs(INDEX_BEGAN_AT_TS)) 114 + .elapsed() 115 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)? 116 + .as_secs() / 86400; 117 + Ok(acceptable(accept, HelloReponse { 118 + help: "open this URL in a web browser (or request with Accept: text/html) for information about this API.", 119 + days_indexed, 120 + stats, 121 + })) 101 122 } 102 123 103 124 #[derive(Clone, Deserialize)]
+2 -1
constellation/src/storage/mod.rs
··· 1 1 use anyhow::Result; 2 2 use constellation::{ActionableEvent, CountsByCount, Did, RecordId}; 3 3 use std::collections::HashMap; 4 + use serde::{Deserialize, Serialize}; 4 5 5 6 pub mod mem_store; 6 7 pub use mem_store::MemStorage; ··· 17 18 pub next: Option<u64>, 18 19 } 19 20 20 - #[derive(Debug, PartialEq)] 21 + #[derive(Debug, Deserialize, Serialize, PartialEq)] 21 22 pub struct StorageStats { 22 23 /// estimate of how many accounts we've seen create links. the _subjects_ of any links are not represented here. 23 24 /// for example: new user A follows users B and C. this count will only increment by one, for A.
+6 -1
constellation/templates/base.html.j2
··· 39 39 padding: 0.5em 0.3em; 40 40 max-width: 100%; 41 41 } 42 + .stat { 43 + color: #f90; 44 + font-size: 1.618rem; 45 + font-weight: bold; 46 + } 42 47 details { 43 48 margin: 2em 0 3em; 44 49 } ··· 49 54 </style> 50 55 </head> 51 56 <body class="{% block body_classes %}{% endblock %}"> 52 - <h1><a href="/">This</a> is a <a href="https://github.com/at-ucosm/links/tree/main/constellation">constellation 🌌</a> server from <a href="https://github.com/at-ucosm">microcosm</a> ✨</h1> 57 + <h1><a href="/">This</a> is a <a href="https://github.com/at-ucosm/links/tree/main/constellation">constellation 🌌</a> API server from <a href="https://github.com/at-ucosm">microcosm</a> ✨</h1> 53 58 {% block content %}{% endblock %} 54 59 55 60 <footer>
+1 -1
constellation/templates/dids-count.html.j2
··· 14 14 {% endif %} 15 15 </h2> 16 16 17 - <p><strong><code>{{ total }}</code></strong> total linking DIDs from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p> 17 + <p><strong><code>{{ total|human_number }}</code></strong> total linking DIDs from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p> 18 18 19 19 <ul> 20 20 <li>See these dids at <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode() }}">/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}</a></li>
+1 -1
constellation/templates/dids.html.j2
··· 14 14 {% endif %} 15 15 </h2> 16 16 17 - <p><strong>{{ total }} dids</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p> 17 + <p><strong>{{ total|human_number }} dids</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p> 18 18 19 19 <ul> 20 20 <li>See linking records to this target at <code>/links</code>: <a href="/links?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li>
+1 -1
constellation/templates/explore-links.html.j2
··· 20 20 {%- for (collection, collection_links) in links -%} 21 21 <strong>{{ collection }}</strong> 22 22 {%- for (path, counts) in collection_links %} 23 - {{ path }}: <a href="/links?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.records }} links</a> from <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.distinct_dids }} distinct DIDs</a></li> 23 + {{ path }}: <a href="/links?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.records|human_number }} links</a> from <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ collection|urlencode }}&path={{ path|urlencode }}">{{ counts.distinct_dids|human_number }} distinct DIDs</a></li> 24 24 {%- endfor %} 25 25 26 26 {% else -%}
+16 -4
constellation/templates/hello.html.j2
··· 5 5 {% block body_classes %}home{% endblock %} 6 6 7 7 {% block content %} 8 - <p>Every interaction in Bluesky and atproto at large tends to appear as a <em>link</em> from a new repository record to <em>somewhere</em>: liking a post creates a record with a link to the post, blocking a spammer creates a record with reference to their DID.</p> 9 8 10 - <p>This service attempts to aggregate all of these links, globally, from all content coming throught the firehose. It provides generic API endpoints to answer questions like <strong>how many likes does a post have</strong> and <strong>who follows a user</strong>.</p> 9 + <p>Constellation is a self-hosted JSON API to an atproto-wide index of PDS record back-links, so you can query social interactions in real time. It can answer questions like:</p> 11 10 12 - <p>It is very much a <strong>work in progress</strong>. The database has not been backfilled, so any interactions occurring before its last reset will be missing.</p> 11 + <ul> 12 + <li><a href="/links/count/distinct-dids?target={{ "at://did:plc:44ybard66vv44zksje25o7dz/app.bsky.feed.post/3lhhz7k2yqk2h"|urlencode }}&collection=app.bsky.feed.like&path=.subject.uri">How many people liked a liked a bluesky post?</a></li> 13 + <li><a href="/links/distinct-dids?target=did:plc:oky5czdrnfjpqslsw2a5iclo&collection=app.bsky.graph.follow&path=.subject">Who are all the bluesky followers of an identity?</a></li> 14 + <li><a href="/links?target=at://did:plc:nlromb2qyyl6rszaluwhfy6j/fyi.unravel.frontpage.post/3lhd2ivyc422n&collection=fyi.unravel.frontpage.comment&path=.post.uri">What are all the replies to a Frontpage submission?</a></li> 15 + <li><a href="/links/all?target=did:plc:vc7f4oafdgxsihk4cry2xpze">What are <em>all</em> the sources of links to an identity?</a></li> 16 + <li>and more</li> 17 + </ul> 13 18 19 + <p> 20 + This server has indexed <span class="stat">{{ stats.linking_records|human_number }}</span> links between <span class="stat">{{ stats.targetables|human_number }}</span> targets and sources from <span class="stat">{{ stats.dids|human_number }}</span> identities over <span class="stat">{{ days_indexed|human_number }}</span> days.<br/> 21 + <small>(indexing new records in real time, backfill still TODO)</small> 22 + </p> 14 23 15 - <h2>Endpoints</h2> 24 + <p>The API is currently <strong>unstable</strong>. But feel free to use it! If you want to be nice, put your project name and bsky username (or email) in your user-agent header for api requests.</p> 25 + 26 + 27 + <h2>API Endpoints</h2> 16 28 17 29 <h3 class="route"><code>GET /links</code></h3> 18 30
+1 -1
constellation/templates/links.html.j2
··· 14 14 {% endif %} 15 15 </h2> 16 16 17 - <p><strong>{{ total }} links</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p> 17 + <p><strong>{{ total|human_number }} links</strong> from <code>{{ query.collection }}</code> at <code>{{ query.path }}</code></p> 18 18 19 19 <ul> 20 20 <li>See distinct linking DIDs at <code>/links/distinct-dids</code>: <a href="/links/distinct-dids?target={{ query.target|urlencode }}&collection={{ query.collection|urlencode }}&path={{ query.path|urlencode }}">/links/distinct-dids?target={{ query.target }}&collection={{ query.collection }}&path={{ query.path }}</a></li>