The smokesignal.events web application

feature: tap admin

+606
+395
src/http/handle_admin_tap.rs
··· 1 + use anyhow::Result; 2 + use axum::{Form, response::IntoResponse}; 3 + use axum_template::RenderHtml; 4 + use minijinja::context as template_context; 5 + use serde::Deserialize; 6 + use std::time::Duration; 7 + 8 + use crate::{ 9 + contextual_error, 10 + http::{ 11 + context::{AdminRequestContext, admin_template_context}, 12 + errors::WebError, 13 + }, 14 + select_template, 15 + }; 16 + 17 + /// TAP stats response types 18 + mod tap_stats { 19 + use serde::{Deserialize, Serialize}; 20 + 21 + #[derive(Debug, Deserialize, Serialize)] 22 + pub struct RepoCount { 23 + pub repo_count: i64, 24 + } 25 + 26 + #[derive(Debug, Deserialize, Serialize)] 27 + pub struct RecordCount { 28 + pub record_count: i64, 29 + } 30 + 31 + #[derive(Debug, Deserialize, Serialize)] 32 + pub struct OutboxBuffer { 33 + pub outbox_buffer: i64, 34 + } 35 + 36 + #[derive(Debug, Deserialize, Serialize)] 37 + pub struct ResyncBuffer { 38 + pub resync_buffer: i64, 39 + } 40 + 41 + #[derive(Debug, Deserialize, Serialize)] 42 + pub struct Cursors { 43 + pub firehose: i64, 44 + pub list_repos: Option<String>, 45 + } 46 + 47 + #[derive(Debug, Deserialize, Serialize)] 48 + pub struct RepoInfo { 49 + pub did: String, 50 + pub error: Option<String>, 51 + pub handle: Option<String>, 52 + pub records: i64, 53 + pub retries: i64, 54 + pub rev: Option<String>, 55 + pub state: String, 56 + } 57 + } 58 + 59 + #[derive(Debug, Deserialize)] 60 + pub(crate) struct TapSubmitForm { 61 + subject: String, 62 + } 63 + 64 + #[derive(Debug, Deserialize)] 65 + pub(crate) struct TapInfoForm { 66 + subject: String, 67 + } 68 + 69 + /// GET /admin/tap - TAP instance stats 70 + pub(crate) async fn handle_admin_tap( 71 + admin_ctx: AdminRequestContext, 72 + ) -> Result<impl IntoResponse, WebError> { 73 + let canonical_url = format!( 74 + "https://{}/admin/tap", 75 + admin_ctx.web_context.config.external_base 76 + ); 77 + let default_context = admin_template_context(&admin_ctx, &canonical_url, "tap"); 78 + 79 + let render_template = select_template!("admin_tap", false, false, admin_ctx.language); 80 + 81 + let tap_enabled = admin_ctx.web_context.config.enable_tap; 82 + let tap_hostname = &admin_ctx.web_context.config.tap_hostname; 83 + 84 + // If TAP is not enabled, show a simple message 85 + if !tap_enabled { 86 + return Ok(RenderHtml( 87 + &render_template, 88 + admin_ctx.web_context.engine.clone(), 89 + template_context! { ..default_context, ..template_context! { 90 + tap_enabled => false, 91 + tap_hostname => tap_hostname, 92 + }}, 93 + ) 94 + .into_response()); 95 + } 96 + 97 + let base_url = format!("http://{}", tap_hostname); 98 + let client = &admin_ctx.web_context.http_client; 99 + 100 + // Fetch all stats endpoints concurrently 101 + let (repo_count, record_count, outbox_buffer, resync_buffer, cursors) = tokio::join!( 102 + fetch_tap_stat::<tap_stats::RepoCount>(client, &base_url, "/stats/repo-count"), 103 + fetch_tap_stat::<tap_stats::RecordCount>(client, &base_url, "/stats/record-count"), 104 + fetch_tap_stat::<tap_stats::OutboxBuffer>(client, &base_url, "/stats/outbox-buffer"), 105 + fetch_tap_stat::<tap_stats::ResyncBuffer>(client, &base_url, "/stats/resync-buffer"), 106 + fetch_tap_stat::<tap_stats::Cursors>(client, &base_url, "/stats/cursors"), 107 + ); 108 + 109 + // Determine connection status 110 + let connected = repo_count.is_some(); 111 + 112 + Ok(RenderHtml( 113 + &render_template, 114 + admin_ctx.web_context.engine.clone(), 115 + template_context! { ..default_context, ..template_context! { 116 + tap_enabled => true, 117 + tap_hostname => tap_hostname, 118 + connected => connected, 119 + repo_count => repo_count.map(|r| r.repo_count), 120 + record_count => record_count.map(|r| r.record_count), 121 + outbox_buffer => outbox_buffer.map(|r| r.outbox_buffer), 122 + resync_buffer => resync_buffer.map(|r| r.resync_buffer), 123 + firehose_cursor => cursors.as_ref().map(|c| c.firehose), 124 + list_repos_cursor => cursors.as_ref().and_then(|c| c.list_repos.clone()), 125 + }}, 126 + ) 127 + .into_response()) 128 + } 129 + 130 + /// Helper to fetch a TAP stats endpoint 131 + async fn fetch_tap_stat<T: serde::de::DeserializeOwned>( 132 + client: &reqwest::Client, 133 + base_url: &str, 134 + path: &str, 135 + ) -> Option<T> { 136 + let url = format!("{}{}", base_url, path); 137 + match client.get(&url).timeout(Duration::from_secs(30)).send().await { 138 + Ok(response) if response.status().is_success() => response.json::<T>().await.ok(), 139 + Ok(response) => { 140 + tracing::debug!(url = %url, status = %response.status(), "TAP stats request failed"); 141 + None 142 + } 143 + Err(e) => { 144 + tracing::debug!(url = %url, error = %e, "TAP stats request error"); 145 + None 146 + } 147 + } 148 + } 149 + 150 + /// POST /admin/tap/submit - Submit a DID to TAP for crawling 151 + pub(crate) async fn handle_admin_tap_submit( 152 + admin_ctx: AdminRequestContext, 153 + Form(form): Form<TapSubmitForm>, 154 + ) -> Result<impl IntoResponse, WebError> { 155 + let canonical_url = format!( 156 + "https://{}/admin/tap", 157 + admin_ctx.web_context.config.external_base 158 + ); 159 + let default_context = admin_template_context(&admin_ctx, &canonical_url, "tap"); 160 + 161 + let render_template = select_template!("admin_tap", false, false, admin_ctx.language); 162 + let error_template = select_template!(false, false, admin_ctx.language); 163 + 164 + let tap_enabled = admin_ctx.web_context.config.enable_tap; 165 + let tap_hostname = &admin_ctx.web_context.config.tap_hostname; 166 + 167 + if !tap_enabled { 168 + return contextual_error!( 169 + admin_ctx.web_context, 170 + admin_ctx.language, 171 + error_template, 172 + default_context, 173 + "TAP is not enabled" 174 + ); 175 + } 176 + 177 + let subject = form.subject.trim(); 178 + 179 + // Resolve the subject (handle or DID) to a DID 180 + let document = match admin_ctx.web_context.identity_resolver.resolve(subject).await { 181 + Ok(doc) => doc, 182 + Err(err) => { 183 + return contextual_error!( 184 + admin_ctx.web_context, 185 + admin_ctx.language, 186 + error_template, 187 + default_context, 188 + err 189 + ); 190 + } 191 + }; 192 + 193 + let did = &document.id; 194 + 195 + let url = format!("http://{}/repos/add", tap_hostname); 196 + 197 + // Build the request payload 198 + let payload = serde_json::json!({ 199 + "dids": [did] 200 + }); 201 + 202 + // Send the request to TAP 203 + let response = match admin_ctx 204 + .web_context 205 + .http_client 206 + .post(&url) 207 + .json(&payload) 208 + .timeout(Duration::from_secs(10)) 209 + .send() 210 + .await 211 + { 212 + Ok(resp) => resp, 213 + Err(err) => { 214 + tracing::error!(error = %err, "TAP request failed"); 215 + return contextual_error!( 216 + admin_ctx.web_context, 217 + admin_ctx.language, 218 + error_template, 219 + default_context, 220 + format!("TAP request failed: {}", err) 221 + ); 222 + } 223 + }; 224 + 225 + if !response.status().is_success() { 226 + let status = response.status(); 227 + let body = response.text().await.unwrap_or_default(); 228 + tracing::warn!(subject = %subject, did = %did, status = %status, body = %body, "TAP submit failed"); 229 + return contextual_error!( 230 + admin_ctx.web_context, 231 + admin_ctx.language, 232 + error_template, 233 + default_context, 234 + format!("TAP returned error: {} - {}", status, body) 235 + ); 236 + } 237 + 238 + tracing::info!(subject = %subject, did = %did, "Submitted DID to TAP for crawling"); 239 + 240 + // Re-fetch stats and render the page with a success message 241 + let base_url = format!("http://{}", tap_hostname); 242 + let client = &admin_ctx.web_context.http_client; 243 + 244 + let (repo_count, record_count, outbox_buffer, resync_buffer, cursors) = tokio::join!( 245 + fetch_tap_stat::<tap_stats::RepoCount>(client, &base_url, "/stats/repo-count"), 246 + fetch_tap_stat::<tap_stats::RecordCount>(client, &base_url, "/stats/record-count"), 247 + fetch_tap_stat::<tap_stats::OutboxBuffer>(client, &base_url, "/stats/outbox-buffer"), 248 + fetch_tap_stat::<tap_stats::ResyncBuffer>(client, &base_url, "/stats/resync-buffer"), 249 + fetch_tap_stat::<tap_stats::Cursors>(client, &base_url, "/stats/cursors"), 250 + ); 251 + 252 + let connected = repo_count.is_some(); 253 + 254 + Ok(RenderHtml( 255 + &render_template, 256 + admin_ctx.web_context.engine.clone(), 257 + template_context! { ..default_context, ..template_context! { 258 + tap_enabled => true, 259 + tap_hostname => tap_hostname, 260 + connected => connected, 261 + repo_count => repo_count.map(|r| r.repo_count), 262 + record_count => record_count.map(|r| r.record_count), 263 + outbox_buffer => outbox_buffer.map(|r| r.outbox_buffer), 264 + resync_buffer => resync_buffer.map(|r| r.resync_buffer), 265 + firehose_cursor => cursors.as_ref().map(|c| c.firehose), 266 + list_repos_cursor => cursors.as_ref().and_then(|c| c.list_repos.clone()), 267 + submit_success => true, 268 + submit_subject => subject, 269 + submit_did => did.clone(), 270 + }}, 271 + ) 272 + .into_response()) 273 + } 274 + 275 + /// POST /admin/tap/info - Get info about a DID from TAP 276 + pub(crate) async fn handle_admin_tap_info( 277 + admin_ctx: AdminRequestContext, 278 + Form(form): Form<TapInfoForm>, 279 + ) -> Result<impl IntoResponse, WebError> { 280 + let canonical_url = format!( 281 + "https://{}/admin/tap", 282 + admin_ctx.web_context.config.external_base 283 + ); 284 + let default_context = admin_template_context(&admin_ctx, &canonical_url, "tap"); 285 + 286 + let render_template = select_template!("admin_tap", false, false, admin_ctx.language); 287 + let error_template = select_template!(false, false, admin_ctx.language); 288 + 289 + let tap_enabled = admin_ctx.web_context.config.enable_tap; 290 + let tap_hostname = &admin_ctx.web_context.config.tap_hostname; 291 + 292 + if !tap_enabled { 293 + return contextual_error!( 294 + admin_ctx.web_context, 295 + admin_ctx.language, 296 + error_template, 297 + default_context, 298 + "TAP is not enabled" 299 + ); 300 + } 301 + 302 + let subject = form.subject.trim(); 303 + 304 + // Resolve the subject (handle or DID) to a DID 305 + let document = match admin_ctx.web_context.identity_resolver.resolve(subject).await { 306 + Ok(doc) => doc, 307 + Err(err) => { 308 + return contextual_error!( 309 + admin_ctx.web_context, 310 + admin_ctx.language, 311 + error_template, 312 + default_context, 313 + err 314 + ); 315 + } 316 + }; 317 + 318 + let did = &document.id; 319 + 320 + let url = format!("http://{}/info/{}", tap_hostname, did); 321 + 322 + // Send the request to TAP 323 + let response = match admin_ctx 324 + .web_context 325 + .http_client 326 + .get(&url) 327 + .timeout(Duration::from_secs(10)) 328 + .send() 329 + .await 330 + { 331 + Ok(resp) => resp, 332 + Err(err) => { 333 + tracing::error!(error = %err, "TAP info request failed"); 334 + return contextual_error!( 335 + admin_ctx.web_context, 336 + admin_ctx.language, 337 + error_template, 338 + default_context, 339 + format!("TAP request failed: {}", err) 340 + ); 341 + } 342 + }; 343 + 344 + let info: Option<tap_stats::RepoInfo> = if response.status().is_success() { 345 + response.json().await.ok() 346 + } else { 347 + None 348 + }; 349 + 350 + // Re-fetch the TAP stats for the page 351 + let base_url = format!("http://{}", tap_hostname); 352 + let client = &admin_ctx.web_context.http_client; 353 + 354 + let (repo_count, record_count, outbox_buffer, resync_buffer, cursors) = tokio::join!( 355 + fetch_tap_stat::<tap_stats::RepoCount>(client, &base_url, "/stats/repo-count"), 356 + fetch_tap_stat::<tap_stats::RecordCount>(client, &base_url, "/stats/record-count"), 357 + fetch_tap_stat::<tap_stats::OutboxBuffer>(client, &base_url, "/stats/outbox-buffer"), 358 + fetch_tap_stat::<tap_stats::ResyncBuffer>(client, &base_url, "/stats/resync-buffer"), 359 + fetch_tap_stat::<tap_stats::Cursors>(client, &base_url, "/stats/cursors"), 360 + ); 361 + 362 + let connected = repo_count.is_some(); 363 + 364 + let info_context = info.as_ref().map(|i| { 365 + template_context! { 366 + did => i.did.clone(), 367 + error => i.error.clone(), 368 + handle => i.handle.clone(), 369 + records => i.records, 370 + retries => i.retries, 371 + rev => i.rev.clone(), 372 + state => i.state.clone(), 373 + } 374 + }); 375 + 376 + Ok(RenderHtml( 377 + &render_template, 378 + admin_ctx.web_context.engine.clone(), 379 + template_context! { ..default_context, ..template_context! { 380 + tap_enabled => true, 381 + tap_hostname => tap_hostname, 382 + connected => connected, 383 + repo_count => repo_count.map(|r| r.repo_count), 384 + record_count => record_count.map(|r| r.record_count), 385 + outbox_buffer => outbox_buffer.map(|r| r.outbox_buffer), 386 + resync_buffer => resync_buffer.map(|r| r.resync_buffer), 387 + firehose_cursor => cursors.as_ref().map(|c| c.firehose), 388 + list_repos_cursor => cursors.as_ref().and_then(|c| c.list_repos.clone()), 389 + info_subject => subject, 390 + info_result => info_context, 391 + info_not_found => info.is_none(), 392 + }}, 393 + ) 394 + .into_response()) 395 + }
+1
src/http/mod.rs
··· 27 27 pub mod handle_admin_rsvp_accepts; 28 28 pub mod handle_admin_rsvps; 29 29 pub mod handle_admin_search_index; 30 + pub mod handle_admin_tap; 30 31 pub mod handle_api_mcp_configuration; 31 32 pub mod handle_blob; 32 33 pub mod handle_bulk_accept_rsvps;
+4
src/http/server.rs
··· 51 51 handle_admin_search_index, handle_admin_search_index_delete, 52 52 handle_admin_search_index_rebuild, 53 53 }, 54 + handle_admin_tap::{handle_admin_tap, handle_admin_tap_info, handle_admin_tap_submit}, 54 55 handle_api_mcp_configuration::{ 55 56 handle_api_mcp_configuration_get, handle_api_mcp_configuration_post, 56 57 }, ··· 288 289 "/admin/search-index/profiles/rebuild", 289 290 post(handle_admin_profile_index_rebuild), 290 291 ) 292 + .route("/admin/tap", get(handle_admin_tap)) 293 + .route("/admin/tap/submit", post(handle_admin_tap_submit)) 294 + .route("/admin/tap/info", post(handle_admin_tap_info)) 291 295 .route("/content/{cid}", get(handle_content)) 292 296 .route("/logout", get(handle_auth_logout)) 293 297 .route("/language", post(handle_set_language))
+6
templates/en-us/admin_base.html
··· 35 35 <li><a href="/admin/search-index/events" {% if active_section == "search-index-events" %}class="is-active"{% endif %}>Event Index</a></li> 36 36 <li><a href="/admin/search-index/profiles" {% if active_section == "search-index-profiles" %}class="is-active"{% endif %}>Profile Index</a></li> 37 37 </ul> 38 + <p class="menu-label"> 39 + Infrastructure 40 + </p> 41 + <ul class="menu-list"> 42 + <li><a href="/admin/tap" {% if active_section == "tap" %}class="is-active"{% endif %}>TAP</a></li> 43 + </ul> 38 44 </aside> 39 45 </div> 40 46 <div class="column">
+200
templates/en-us/admin_tap.html
··· 1 + {% extends "en-us/admin_base.html" %} 2 + {% block admin_title %}TAP Management{% endblock %} 3 + {% block admin_content %} 4 + <div class="content"> 5 + <h1 class="title">TAP Management</h1> 6 + <p class="subtitle">Monitor and manage the TAP (Timeline Aggregation Protocol) instance</p> 7 + 8 + {% if not tap_enabled %} 9 + <article class="message is-warning"> 10 + <div class="message-header"> 11 + <p>TAP Disabled</p> 12 + </div> 13 + <div class="message-body"> 14 + TAP is not enabled in the current configuration. Set <code>ENABLE_TAP=true</code> to enable TAP functionality. 15 + <br><br> 16 + <strong>Configured hostname:</strong> <code>{{ tap_hostname }}</code> 17 + </div> 18 + </article> 19 + {% else %} 20 + 21 + {% if submit_success %} 22 + <article class="message is-success"> 23 + <div class="message-body"> 24 + Successfully submitted <strong>{{ submit_subject }}</strong> (<code>{{ submit_did }}</code>) to TAP for crawling. 25 + </div> 26 + </article> 27 + {% endif %} 28 + 29 + <div class="box mb-5"> 30 + <h2 class="title is-4">Connection Status</h2> 31 + <div class="columns"> 32 + <div class="column"> 33 + <p> 34 + <strong>Hostname:</strong> <code>{{ tap_hostname }}</code> 35 + </p> 36 + <p> 37 + <strong>Status:</strong> 38 + {% if connected %} 39 + <span class="tag is-success">Connected</span> 40 + {% else %} 41 + <span class="tag is-danger">Disconnected</span> 42 + {% endif %} 43 + </p> 44 + </div> 45 + </div> 46 + </div> 47 + 48 + {% if connected %} 49 + <div class="box mb-5"> 50 + <h2 class="title is-4">Instance Statistics</h2> 51 + <div class="columns is-multiline"> 52 + <div class="column is-one-quarter"> 53 + <div class="notification is-light"> 54 + <p class="heading">Repositories</p> 55 + <p class="title is-4">{{ repo_count | default("-") }}</p> 56 + </div> 57 + </div> 58 + <div class="column is-one-quarter"> 59 + <div class="notification is-light"> 60 + <p class="heading">Records</p> 61 + <p class="title is-4">{{ record_count | default("-") }}</p> 62 + </div> 63 + </div> 64 + <div class="column is-one-quarter"> 65 + <div class="notification is-light"> 66 + <p class="heading">Outbox Buffer</p> 67 + <p class="title is-4">{{ outbox_buffer | default("-") }}</p> 68 + </div> 69 + </div> 70 + <div class="column is-one-quarter"> 71 + <div class="notification is-light"> 72 + <p class="heading">Resync Buffer</p> 73 + <p class="title is-4">{{ resync_buffer | default("-") }}</p> 74 + </div> 75 + </div> 76 + </div> 77 + <div class="columns"> 78 + <div class="column"> 79 + <p><strong>Firehose Cursor:</strong> <code>{{ firehose_cursor | default("N/A") }}</code></p> 80 + <p><strong>List Repos Cursor:</strong> <code>{{ list_repos_cursor | default("N/A") }}</code></p> 81 + </div> 82 + </div> 83 + </div> 84 + {% endif %} 85 + 86 + <div class="columns"> 87 + <div class="column is-half"> 88 + <div class="box"> 89 + <h2 class="title is-4">Submit DID for Crawling</h2> 90 + <form action="/admin/tap/submit" method="POST"> 91 + <div class="field"> 92 + <label class="label" for="submitSubjectInput">Handle or DID</label> 93 + <div class="control has-icons-left"> 94 + <input class="input" type="text" id="submitSubjectInput" placeholder="handle.bsky.social or did:plc:..." name="subject" required="required"> 95 + <span class="icon is-small is-left"> 96 + <i class="fas fa-user"></i> 97 + </span> 98 + </div> 99 + <p class="help">Enter a handle or DID to submit for crawling</p> 100 + </div> 101 + <div class="field"> 102 + <div class="control"> 103 + <button type="submit" class="button is-primary">Submit</button> 104 + </div> 105 + </div> 106 + </form> 107 + </div> 108 + </div> 109 + 110 + <div class="column is-half"> 111 + <div class="box"> 112 + <h2 class="title is-4">Lookup Repository Info</h2> 113 + <form action="/admin/tap/info" method="POST"> 114 + <div class="field"> 115 + <label class="label" for="infoSubjectInput">Handle or DID</label> 116 + <div class="control has-icons-left"> 117 + <input class="input" type="text" id="infoSubjectInput" placeholder="handle.bsky.social or did:plc:..." name="subject" required="required" {% if info_subject %}value="{{ info_subject }}"{% endif %}> 118 + <span class="icon is-small is-left"> 119 + <i class="fas fa-search"></i> 120 + </span> 121 + </div> 122 + <p class="help">Enter a handle or DID to lookup repository info</p> 123 + </div> 124 + <div class="field"> 125 + <div class="control"> 126 + <button type="submit" class="button is-info">Lookup</button> 127 + </div> 128 + </div> 129 + </form> 130 + </div> 131 + </div> 132 + </div> 133 + 134 + {% if info_subject %} 135 + <div class="box"> 136 + <h2 class="title is-4">Repository Info: {{ info_subject }}</h2> 137 + {% if info_not_found %} 138 + <article class="message is-warning"> 139 + <div class="message-body"> 140 + No repository found for <strong>{{ info_subject }}</strong>. The DID may not be tracked by this TAP instance. 141 + </div> 142 + </article> 143 + {% elif info_result %} 144 + <table class="table is-fullwidth"> 145 + <tbody> 146 + <tr> 147 + <th style="width: 150px;">DID</th> 148 + <td><code>{{ info_result.did }}</code></td> 149 + </tr> 150 + {% if info_result.handle %} 151 + <tr> 152 + <th>Handle</th> 153 + <td>{{ info_result.handle }}</td> 154 + </tr> 155 + {% endif %} 156 + <tr> 157 + <th>State</th> 158 + <td> 159 + {% if info_result.state == "synced" %} 160 + <span class="tag is-success">{{ info_result.state }}</span> 161 + {% elif info_result.state == "syncing" %} 162 + <span class="tag is-info">{{ info_result.state }}</span> 163 + {% elif info_result.state == "pending" %} 164 + <span class="tag is-warning">{{ info_result.state }}</span> 165 + {% elif info_result.state == "failed" %} 166 + <span class="tag is-danger">{{ info_result.state }}</span> 167 + {% else %} 168 + <span class="tag">{{ info_result.state }}</span> 169 + {% endif %} 170 + </td> 171 + </tr> 172 + <tr> 173 + <th>Records</th> 174 + <td>{{ info_result.records }}</td> 175 + </tr> 176 + <tr> 177 + <th>Retries</th> 178 + <td>{{ info_result.retries }}</td> 179 + </tr> 180 + {% if info_result.rev %} 181 + <tr> 182 + <th>Revision</th> 183 + <td><code class="is-size-7">{{ info_result.rev }}</code></td> 184 + </tr> 185 + {% endif %} 186 + {% if info_result.error %} 187 + <tr> 188 + <th>Error</th> 189 + <td><span class="has-text-danger">{{ info_result.error }}</span></td> 190 + </tr> 191 + {% endif %} 192 + </tbody> 193 + </table> 194 + {% endif %} 195 + </div> 196 + {% endif %} 197 + 198 + {% endif %} 199 + </div> 200 + {% endblock %}