+31

server/Cargo.lock

··· 359 359 ] 360 360 361 361 [[package]] 362 362 + name = "crossbeam-deque" 363 363 + version = "0.8.6" 364 364 + source = "registry+https://github.com/rust-lang/crates.io-index" 365 365 + checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" 366 366 + dependencies = [ 367 367 + "crossbeam-epoch", 368 368 + "crossbeam-utils", 369 369 + ] 370 370 + 371 371 + [[package]] 362 372 name = "crossbeam-epoch" 363 373 version = "0.9.18" 364 374 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1147 1157 ] 1148 1158 1149 1159 [[package]] 1160 1160 + name = "rayon" 1161 1161 + version = "1.10.0" 1162 1162 + source = "registry+https://github.com/rust-lang/crates.io-index" 1163 1163 + checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" 1164 1164 + dependencies = [ 1165 1165 + "either", 1166 1166 + "rayon-core", 1167 1167 + ] 1168 1168 + 1169 1169 + [[package]] 1170 1170 + name = "rayon-core" 1171 1171 + version = "1.12.1" 1172 1172 + source = "registry+https://github.com/rust-lang/crates.io-index" 1173 1173 + checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" 1174 1174 + dependencies = [ 1175 1175 + "crossbeam-deque", 1176 1176 + "crossbeam-utils", 1177 1177 + ] 1178 1178 + 1179 1179 + [[package]] 1150 1180 name = "redox_syscall" 1151 1181 version = "0.5.15" 1152 1182 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1499 1529 "itertools", 1500 1530 "ordered-varint", 1501 1531 "quanta", 1532 1532 + "rayon", 1502 1533 "rkyv", 1503 1534 "rustls", 1504 1535 "scc",

+1

server/Cargo.toml

··· 28 28 quanta = "0.12.6" 29 29 itertools = "0.14.0" 30 30 byteview = "0.6.1" 31 31 + rayon = "1.10.0"

+1 -1

server/src/api.rs

··· 146 146 to: Option<u64>, 147 147 } 148 148 149 149 - #[derive(Serialize)] 149 149 + #[derive(Debug, Serialize)] 150 150 struct Hit { 151 151 timestamp: u64, 152 152 deleted: bool,

-2

server/src/db/block.rs

··· 8 8 marker::PhantomData, 9 9 }; 10 10 11 11 - use crate::error::AppResult; 12 12 - 13 11 pub struct Item<T> { 14 12 pub timestamp: u64, 15 13 data: AlignedVec,

+76 -48

server/src/db/mod.rs

··· 1 1 use std::{ 2 2 - io::{self, Cursor, Write}, 3 3 - marker::PhantomData, 2 2 + io::{Cursor, Write}, 4 3 ops::{Bound, Deref, RangeBounds}, 5 4 path::Path, 6 5 sync::{ 7 6 Arc, 8 8 - atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering as AtomicOrdering}, 7 7 + atomic::{AtomicU64, AtomicUsize, Ordering as AtomicOrdering}, 9 8 }, 10 9 time::Duration, 11 10 }; ··· 14 13 use fjall::{Config, Keyspace, Partition, PartitionCreateOptions, Slice}; 15 14 use itertools::{Either, Itertools}; 16 15 use ordered_varint::Variable; 16 16 + use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; 17 17 use rkyv::{Archive, Deserialize, Serialize, rancor::Error}; 18 18 use smol_str::SmolStr; 19 19 use tokio::sync::broadcast; 20 20 - use tokio_util::bytes::{self, BufMut}; 20 20 + use tokio_util::sync::CancellationToken; 21 21 22 22 use crate::{ 23 23 db::block::{ReadVariableExt, WriteVariableExt}, ··· 127 127 } 128 128 } 129 129 130 130 + struct Block { 131 131 + written: usize, 132 132 + key: ByteView, 133 133 + data: ByteView, 134 134 + } 135 135 + 130 136 pub struct LexiconHandle { 131 137 tree: Partition, 138 138 + nsid: SmolStr, 132 139 buf: Arc<scc::Queue<EventRecord>>, 133 140 // this is stored here since scc::Queue does not have O(1) length 134 134 - buf_len: AtomicUsize, // relaxed 141 141 + buf_len: AtomicUsize, // seqcst 135 142 last_insert: AtomicU64, // relaxed 136 143 eps: DefaultRateTracker, 137 144 } ··· 141 148 let opts = PartitionCreateOptions::default().compression(fjall::CompressionType::Miniz(9)); 142 149 Self { 143 150 tree: keyspace.open_partition(nsid, opts).unwrap(), 151 151 + nsid: nsid.into(), 144 152 buf: Default::default(), 145 153 buf_len: AtomicUsize::new(0), 146 154 last_insert: AtomicU64::new(0), ··· 149 157 } 150 158 151 159 fn item_count(&self) -> usize { 152 152 - self.buf_len.load(AtomicOrdering::Relaxed) 160 160 + self.buf_len.load(AtomicOrdering::SeqCst) 153 161 } 154 162 155 163 fn since_last_activity(&self) -> u64 { ··· 162 170 163 171 fn insert(&self, event: EventRecord) { 164 172 self.buf.push(event); 165 165 - self.buf_len.fetch_add(1, AtomicOrdering::Relaxed); 173 173 + self.buf_len.fetch_add(1, AtomicOrdering::SeqCst); 166 174 self.last_insert.store(CLOCK.raw(), AtomicOrdering::Relaxed); 167 175 self.eps.observe(); 168 176 } 169 177 170 170 - fn sync(&self, max_block_size: usize) -> AppResult<usize> { 178 178 + fn encode_block(&self, max_block_size: usize) -> AppResult<Option<Block>> { 171 179 let buf_size = 172 180 size_of::<u64>() + self.item_count().min(max_block_size) * size_of::<(u64, NsidHit)>(); 173 181 let mut writer = ItemEncoder::new(WritableByteView::with_size(buf_size)); ··· 192 200 written += 1; 193 201 } 194 202 if let (Some(start_timestamp), Some(end_timestamp)) = (start_timestamp, end_timestamp) { 195 195 - self.buf_len.store(0, AtomicOrdering::Relaxed); 203 203 + self.buf_len.store(0, AtomicOrdering::SeqCst); 196 204 let value = writer.finish()?; 197 197 - let mut key = Vec::with_capacity(size_of::<u64>() * 2); 205 205 + let mut key = WritableByteView::with_size(size_of::<u64>() * 2); 198 206 key.write_varint(start_timestamp)?; 199 207 key.write_varint(end_timestamp)?; 200 200 - self.tree.insert(key, value.into_inner())?; 208 208 + return Ok(Some(Block { 209 209 + written, 210 210 + key: key.into_inner(), 211 211 + data: value.into_inner(), 212 212 + })); 201 213 } 202 202 - Ok(written) 214 214 + Ok(None) 203 215 } 204 216 } 205 217 ··· 209 221 inner: Keyspace, 210 222 counts: Partition, 211 223 hits: scc::HashIndex<SmolStr, Arc<LexiconHandle>>, 212 212 - syncpool: threadpool::ThreadPool, 224 224 + sync_pool: threadpool::ThreadPool, 213 225 event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>, 214 226 eps: RateTracker<100>, 215 215 - shutting_down: AtomicBool, 227 227 + cancel_token: CancellationToken, 216 228 min_block_size: usize, 217 229 max_block_size: usize, 218 230 max_last_activity: u64, 219 231 } 220 232 221 233 impl Db { 222 222 - pub fn new(path: impl AsRef<Path>) -> AppResult<Self> { 234 234 + pub fn new(path: impl AsRef<Path>, cancel_token: CancellationToken) -> AppResult<Self> { 223 235 tracing::info!("opening db..."); 224 236 let ks = Config::new(path) 225 237 .cache_size(8 * 1024 * 1024) // from talna 226 238 .open()?; 227 239 Ok(Self { 228 240 hits: Default::default(), 229 229 - syncpool: threadpool::Builder::new().num_threads(256).build(), 241 241 + sync_pool: threadpool::Builder::new() 242 242 + .num_threads(rayon::current_num_threads() * 2) 243 243 + .build(), 230 244 counts: ks.open_partition( 231 245 "_counts", 232 246 PartitionCreateOptions::default().compression(fjall::CompressionType::None), ··· 234 248 inner: ks, 235 249 event_broadcaster: broadcast::channel(1000).0, 236 250 eps: RateTracker::new(Duration::from_secs(1)), 237 237 - shutting_down: AtomicBool::new(false), 251 251 + cancel_token, 238 252 min_block_size: 512, 239 253 max_block_size: 500_000, 240 254 max_last_activity: Duration::from_secs(10).as_nanos() as u64, 241 255 }) 242 256 } 243 257 244 244 - pub fn shutdown(&self) -> AppResult<()> { 245 245 - self.shutting_down.store(true, AtomicOrdering::Release); 246 246 - self.sync(true) 258 258 + pub fn shutting_down(&self) -> impl Future<Output = ()> { 259 259 + self.cancel_token.cancelled() 247 260 } 248 261 249 262 pub fn is_shutting_down(&self) -> bool { 250 250 - self.shutting_down.load(AtomicOrdering::Acquire) 263 263 + self.cancel_token.is_cancelled() 251 264 } 252 265 253 266 pub fn sync(&self, all: bool) -> AppResult<()> { 254 254 - let mut execs = Vec::with_capacity(self.hits.len()); 267 267 + // prepare all the data 268 268 + let mut data = Vec::with_capacity(self.hits.len()); 255 269 let _guard = scc::ebr::Guard::new(); 256 256 - for (nsid, tree) in self.hits.iter(&_guard) { 257 257 - let count = tree.item_count(); 258 258 - let is_max_block_size = count > self.min_block_size.max(tree.suggested_block_size()); 259 259 - let is_too_old = tree.since_last_activity() > self.max_last_activity; 260 260 - if count > 0 && (all || is_max_block_size || is_too_old) { 261 261 - let nsid = nsid.clone(); 262 262 - let tree = tree.clone(); 263 263 - let max_block_size = self.max_block_size; 264 264 - execs.push(move || { 265 265 - loop { 266 266 - let synced = match tree.sync(max_block_size) { 267 267 - Ok(synced) => synced, 268 268 - Err(err) => { 269 269 - tracing::error!("failed to sync {nsid}: {err}"); 270 270 - break; 271 271 - } 272 272 - }; 273 273 - if synced == 0 { 274 274 - break; 275 275 - } 276 276 - tracing::info!("synced {synced} of {nsid} to db"); 277 277 - } 278 278 - }); 270 270 + for (_, handle) in self.hits.iter(&_guard) { 271 271 + let block_size = self 272 272 + .max_block_size 273 273 + .min(self.min_block_size.max(handle.suggested_block_size())); 274 274 + let count = handle.item_count(); 275 275 + let data_count = count / block_size; 276 276 + let is_too_old = handle.since_last_activity() > self.max_last_activity; 277 277 + if count > 0 && (all || data_count > 0 || is_too_old) { 278 278 + for i in 0..data_count { 279 279 + data.push((i, handle.clone(), block_size)); 280 280 + } 281 281 + // only sync remainder if we haven't met block size 282 282 + let remainder = count % block_size; 283 283 + if data_count == 0 && remainder > 0 { 284 284 + data.push((data_count, handle.clone(), remainder)); 285 285 + } 279 286 } 280 287 } 281 288 drop(_guard); 282 289 283 283 - for exec in execs { 284 284 - self.syncpool.execute(exec); 290 290 + // process the blocks 291 291 + let mut blocks = Vec::with_capacity(data.len()); 292 292 + data.into_par_iter() 293 293 + .map(|(i, handle, max_block_size)| { 294 294 + handle 295 295 + .encode_block(max_block_size) 296 296 + .transpose() 297 297 + .map(|r| r.map(|block| (i, block, handle.clone()))) 298 298 + }) 299 299 + .collect_into_vec(&mut blocks); 300 300 + 301 301 + // execute into db 302 302 + for item in blocks.into_iter() { 303 303 + let Some((i, block, handle)) = item.transpose()? else { 304 304 + continue; 305 305 + }; 306 306 + self.sync_pool 307 307 + .execute(move || match handle.tree.insert(block.key, block.data) { 308 308 + Ok(_) => { 309 309 + tracing::info!("[{i}] synced {} of {} to db", block.written, handle.nsid) 310 310 + } 311 311 + Err(err) => tracing::error!("failed to sync block: {}", err), 312 312 + }); 285 313 } 286 286 - self.syncpool.join(); 314 314 + self.sync_pool.join(); 287 315 288 316 Ok(()) 289 317 }

+51 -20

server/src/main.rs

··· 49 49 None => {} 50 50 } 51 51 52 52 - let db = Arc::new(Db::new(".fjall_data").expect("couldnt create db")); 52 52 + let cancel_token = CancellationToken::new(); 53 53 + 54 54 + let db = 55 55 + Arc::new(Db::new(".fjall_data", cancel_token.child_token()).expect("couldnt create db")); 53 56 54 57 rustls::crypto::ring::default_provider() 55 58 .install_default() ··· 64 67 } 65 68 }; 66 69 67 67 - let cancel_token = CancellationToken::new(); 68 68 - 70 70 + let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(1000); 69 71 let consume_events = tokio::spawn({ 70 72 let consume_cancel = cancel_token.child_token(); 71 71 - let db = db.clone(); 72 73 async move { 73 74 jetstream.connect().await?; 74 75 loop { ··· 78 79 let Some(record) = EventRecord::from_jetstream(event) else { 79 80 continue; 80 81 }; 81 81 - let db = db.clone(); 82 82 - tokio::task::spawn_blocking(move || { 83 83 - if let Err(err) = db.record_event(record) { 84 84 - tracing::error!("failed to record event: {}", err); 85 85 - } 86 86 - }); 82 82 + event_tx.send(record).await?; 87 83 } 88 84 Err(err) => return Err(err), 89 85 }, ··· 93 89 } 94 90 }); 95 91 96 96 - std::thread::spawn({ 92 92 + let ingest_events = std::thread::spawn({ 97 93 let db = db.clone(); 98 94 move || { 95 95 + let mut buffer = Vec::new(); 99 96 loop { 100 100 - if db.is_shutting_down() { 97 97 + let read = event_rx.blocking_recv_many(&mut buffer, 100); 98 98 + if let Err(err) = db.ingest_events(buffer.drain(..)) { 99 99 + tracing::error!("failed to ingest events: {}", err); 100 100 + } 101 101 + if read == 0 || db.is_shutting_down() { 101 102 break; 102 103 } 103 103 - match db.sync(false) { 104 104 - Ok(_) => (), 105 105 - Err(e) => tracing::error!("failed to sync db: {}", e), 104 104 + } 105 105 + } 106 106 + }); 107 107 + 108 108 + let sync_task = tokio::task::spawn({ 109 109 + let db = db.clone(); 110 110 + async move { 111 111 + loop { 112 112 + let sync_db = tokio::task::spawn_blocking({ 113 113 + let db = db.clone(); 114 114 + move || { 115 115 + if db.is_shutting_down() { 116 116 + return; 117 117 + } 118 118 + match db.sync(false) { 119 119 + Ok(_) => (), 120 120 + Err(e) => tracing::error!("failed to sync db: {}", e), 121 121 + } 122 122 + } 123 123 + }); 124 124 + tokio::select! { 125 125 + _ = sync_db => {} 126 126 + _ = db.shutting_down() => break, 127 127 + } 128 128 + tokio::select! { 129 129 + _ = tokio::time::sleep(std::time::Duration::from_secs(10)) => {} 130 130 + _ = db.shutting_down() => break, 106 131 } 107 107 - std::thread::sleep(std::time::Duration::from_secs(10)); 108 132 } 109 133 } 110 134 }); ··· 130 154 } 131 155 132 156 tracing::info!("shutting down..."); 133 133 - db.shutdown().expect("couldnt shutdown db"); 157 157 + cancel_token.cancel(); 158 158 + ingest_events.join().expect("failed to join ingest events"); 159 159 + sync_task.await.expect("cant join sync task"); 160 160 + db.sync(true).expect("cant sync db"); 134 161 } 135 162 136 163 fn debug() { 137 137 - let db = Db::new(".fjall_data").expect("couldnt create db"); 164 164 + let db = Db::new(".fjall_data", CancellationToken::new()).expect("couldnt create db"); 138 165 for nsid in db.get_nsids() { 139 166 let nsid = nsid.deref(); 140 167 for hit in db.get_hits(nsid, ..) { ··· 145 172 } 146 173 147 174 fn compact() { 148 148 - let from = Arc::new(Db::new(".fjall_data_from").expect("couldnt create db")); 149 149 - let to = Arc::new(Db::new(".fjall_data_to").expect("couldnt create db")); 175 175 + let cancel_token = CancellationToken::new(); 176 176 + let from = Arc::new( 177 177 + Db::new(".fjall_data_from", cancel_token.child_token()).expect("couldnt create db"), 178 178 + ); 179 179 + let to = 180 180 + Arc::new(Db::new(".fjall_data_to", cancel_token.child_token()).expect("couldnt create db")); 150 181 151 182 let nsids = from.get_nsids().collect::<Vec<_>>(); 152 183 let mut threads = Vec::with_capacity(nsids.len());

+13 -12

server/src/utils.rs

··· 9 9 #[derive(Debug)] 10 10 pub struct RateTracker<const BUCKET_WINDOW: u64> { 11 11 buckets: Vec<AtomicU64>, 12 12 + last_bucket_time: AtomicU64, 12 13 bucket_duration_nanos: u64, 13 14 window_duration: Duration, 14 14 - last_bucket_time: AtomicU64, 15 15 start_time: u64, // raw time when tracker was created 16 16 } 17 17 ··· 39 39 } 40 40 } 41 41 42 42 + #[inline(always)] 43 43 + fn elapsed(&self) -> u64 { 44 44 + CLOCK.delta_as_nanos(self.start_time, CLOCK.raw()) 45 45 + } 46 46 + 42 47 /// record an event 43 48 pub fn observe(&self) { 44 44 - let now = CLOCK.raw(); 45 45 - self.maybe_advance_buckets(now); 49 49 + self.maybe_advance_buckets(); 46 50 47 47 - let bucket_index = self.get_current_bucket_index(now); 51 51 + let bucket_index = self.get_current_bucket_index(); 48 52 self.buckets[bucket_index].fetch_add(1, Ordering::Relaxed); 49 53 } 50 54 51 55 /// get the current rate in events per second 52 56 pub fn rate(&self) -> f64 { 53 53 - let now = CLOCK.raw(); 54 54 - self.maybe_advance_buckets(now); 57 57 + self.maybe_advance_buckets(); 55 58 56 59 let total_events: u64 = self 57 60 .buckets ··· 62 65 total_events as f64 / self.window_duration.as_secs_f64() 63 66 } 64 67 65 65 - fn get_current_bucket_index(&self, now: u64) -> usize { 66 66 - let elapsed_nanos = CLOCK.delta_as_nanos(self.start_time, now); 67 67 - let bucket_number = elapsed_nanos / self.bucket_duration_nanos; 68 68 + fn get_current_bucket_index(&self) -> usize { 69 69 + let bucket_number = self.elapsed() / self.bucket_duration_nanos; 68 70 (bucket_number as usize) % self.buckets.len() 69 71 } 70 72 71 71 - fn maybe_advance_buckets(&self, now: u64) { 72 72 - let elapsed_nanos = CLOCK.delta_as_nanos(self.start_time, now); 73 73 + fn maybe_advance_buckets(&self) { 73 74 let current_bucket_time = 74 74 - (elapsed_nanos / self.bucket_duration_nanos) * self.bucket_duration_nanos; 75 75 + (self.elapsed() / self.bucket_duration_nanos) * self.bucket_duration_nanos; 75 76 let last_bucket_time = self.last_bucket_time.load(Ordering::Relaxed); 76 77 77 78 if current_bucket_time > last_bucket_time {