AT-based link agregator. Mirror of https://github.com/likeandscribe/frontpage

Add cursor lag to log, first pass at prometheus metrics

tom.sherman.is 153236dd e610986e

verified
+479 -14
+445 -9
Cargo.lock
··· 3 version = 4 4 5 [[package]] 6 name = "aho-corasick" 7 version = "1.1.3" 8 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 143 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" 144 145 [[package]] 146 name = "base-x" 147 version = "0.2.11" 148 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 174 ] 175 176 [[package]] 177 name = "bitflags" 178 version = "1.3.2" 179 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 225 ] 226 227 [[package]] 228 name = "cfg-if" 229 version = "1.0.4" 230 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 259 ] 260 261 [[package]] 262 name = "clap" 263 version = "2.34.0" 264 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 274 ] 275 276 [[package]] 277 name = "colorchoice" 278 version = "1.0.4" 279 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 296 ] 297 298 [[package]] 299 name = "core-foundation-sys" 300 version = "0.8.7" 301 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 419 "futures-util", 420 "jetstream", 421 "log", 422 "reqwest", 423 "serde", 424 "serde_json", 425 "sled", 426 "tokio", 427 - "tokio-metrics", 428 ] 429 430 [[package]] ··· 452 ] 453 454 [[package]] 455 name = "encoding_rs" 456 version = "0.8.35" 457 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 461 ] 462 463 [[package]] 464 name = "env_filter" 465 version = "0.1.4" 466 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 530 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 531 532 [[package]] 533 name = "foreign-types" 534 version = "0.3.2" 535 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 564 ] 565 566 [[package]] 567 name = "futures-channel" 568 version = "0.3.31" 569 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 661 ] 662 663 [[package]] 664 name = "h2" 665 version = "0.4.12" 666 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 681 682 [[package]] 683 name = "hashbrown" 684 version = "0.16.0" 685 source = "registry+https://github.com/rust-lang/crates.io-index" 686 checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" ··· 744 checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" 745 746 [[package]] 747 name = "hyper" 748 version = "1.7.0" 749 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 757 "http", 758 "http-body", 759 "httparse", 760 "itoa", 761 "pin-project-lite", 762 "pin-utils", ··· 775 "hyper", 776 "hyper-util", 777 "rustls", 778 "rustls-pki-types", 779 "tokio", 780 "tokio-rustls", ··· 961 checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" 962 dependencies = [ 963 "equivalent", 964 - "hashbrown", 965 ] 966 967 [[package]] ··· 1007 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 1008 1009 [[package]] 1010 name = "itoa" 1011 version = "1.0.15" 1012 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1096 checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" 1097 1098 [[package]] 1099 name = "linux-raw-sys" 1100 version = "0.11.0" 1101 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1140 checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" 1141 1142 [[package]] 1143 name = "mime" 1144 version = "0.3.17" 1145 source = "registry+https://github.com/rust-lang/crates.io-index" 1146 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 1147 1148 [[package]] 1149 name = "mio" 1150 version = "1.1.0" 1151 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1200 "openssl-probe", 1201 "openssl-sys", 1202 "schannel", 1203 - "security-framework", 1204 "security-framework-sys", 1205 "tempfile", 1206 ] 1207 1208 [[package]] 1209 name = "num-traits" 1210 version = "0.2.19" 1211 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1271 ] 1272 1273 [[package]] 1274 name = "parking_lot" 1275 version = "0.11.2" 1276 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1376 ] 1377 1378 [[package]] 1379 name = "proc-macro-error" 1380 version = "1.0.4" 1381 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1409 ] 1410 1411 [[package]] 1412 name = "quote" 1413 version = "1.0.41" 1414 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1424 checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" 1425 1426 [[package]] 1427 name = "rand" 1428 version = "0.8.5" 1429 source = "registry+https://github.com/rust-lang/crates.io-index" 1430 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 1431 dependencies = [ 1432 "libc", 1433 - "rand_chacha", 1434 - "rand_core", 1435 ] 1436 1437 [[package]] ··· 1441 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 1442 dependencies = [ 1443 "ppv-lite86", 1444 - "rand_core", 1445 ] 1446 1447 [[package]] ··· 1451 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 1452 dependencies = [ 1453 "getrandom 0.2.16", 1454 ] 1455 1456 [[package]] ··· 1555 ] 1556 1557 [[package]] 1558 name = "rustix" 1559 version = "1.1.2" 1560 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1573 source = "registry+https://github.com/rust-lang/crates.io-index" 1574 checksum = "751e04a496ca00bb97a5e043158d23d66b5aabf2e1d5aa2a0aaebb1aafe6f82c" 1575 dependencies = [ 1576 "once_cell", 1577 "rustls-pki-types", 1578 "rustls-webpki", ··· 1581 ] 1582 1583 [[package]] 1584 name = "rustls-pki-types" 1585 version = "1.12.0" 1586 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1595 source = "registry+https://github.com/rust-lang/crates.io-index" 1596 checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf" 1597 dependencies = [ 1598 "ring", 1599 "rustls-pki-types", 1600 "untrusted", ··· 1634 checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" 1635 dependencies = [ 1636 "bitflags 2.9.4", 1637 - "core-foundation", 1638 "core-foundation-sys", 1639 "libc", 1640 "security-framework-sys", ··· 1753 dependencies = [ 1754 "libc", 1755 ] 1756 1757 [[package]] 1758 name = "slab" ··· 1892 checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" 1893 dependencies = [ 1894 "bitflags 2.9.4", 1895 - "core-foundation", 1896 "system-configuration-sys", 1897 ] 1898 ··· 2016 "pin-project-lite", 2017 "tokio", 2018 "tokio-stream", 2019 ] 2020 2021 [[package]] ··· 2170 "httparse", 2171 "log", 2172 "native-tls", 2173 - "rand", 2174 "sha1", 2175 "thiserror 1.0.69", 2176 "utf-8",
··· 3 version = 4 4 5 [[package]] 6 + name = "ahash" 7 + version = "0.8.12" 8 + source = "registry+https://github.com/rust-lang/crates.io-index" 9 + checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" 10 + dependencies = [ 11 + "cfg-if", 12 + "once_cell", 13 + "version_check", 14 + "zerocopy", 15 + ] 16 + 17 + [[package]] 18 name = "aho-corasick" 19 version = "1.1.3" 20 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 155 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" 156 157 [[package]] 158 + name = "aws-lc-rs" 159 + version = "1.14.1" 160 + source = "registry+https://github.com/rust-lang/crates.io-index" 161 + checksum = "879b6c89592deb404ba4dc0ae6b58ffd1795c78991cbb5b8bc441c48a070440d" 162 + dependencies = [ 163 + "aws-lc-sys", 164 + "zeroize", 165 + ] 166 + 167 + [[package]] 168 + name = "aws-lc-sys" 169 + version = "0.32.3" 170 + source = "registry+https://github.com/rust-lang/crates.io-index" 171 + checksum = "107a4e9d9cab9963e04e84bb8dee0e25f2a987f9a8bad5ed054abd439caa8f8c" 172 + dependencies = [ 173 + "bindgen", 174 + "cc", 175 + "cmake", 176 + "dunce", 177 + "fs_extra", 178 + ] 179 + 180 + [[package]] 181 name = "base-x" 182 version = "0.2.11" 183 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 209 ] 210 211 [[package]] 212 + name = "bindgen" 213 + version = "0.72.1" 214 + source = "registry+https://github.com/rust-lang/crates.io-index" 215 + checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" 216 + dependencies = [ 217 + "bitflags 2.9.4", 218 + "cexpr", 219 + "clang-sys", 220 + "itertools", 221 + "log", 222 + "prettyplease", 223 + "proc-macro2", 224 + "quote", 225 + "regex", 226 + "rustc-hash", 227 + "shlex", 228 + "syn 2.0.106", 229 + ] 230 + 231 + [[package]] 232 name = "bitflags" 233 version = "1.3.2" 234 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 280 ] 281 282 [[package]] 283 + name = "cexpr" 284 + version = "0.6.0" 285 + source = "registry+https://github.com/rust-lang/crates.io-index" 286 + checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" 287 + dependencies = [ 288 + "nom", 289 + ] 290 + 291 + [[package]] 292 name = "cfg-if" 293 version = "1.0.4" 294 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 323 ] 324 325 [[package]] 326 + name = "clang-sys" 327 + version = "1.8.1" 328 + source = "registry+https://github.com/rust-lang/crates.io-index" 329 + checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" 330 + dependencies = [ 331 + "glob", 332 + "libc", 333 + "libloading", 334 + ] 335 + 336 + [[package]] 337 name = "clap" 338 version = "2.34.0" 339 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 349 ] 350 351 [[package]] 352 + name = "cmake" 353 + version = "0.1.54" 354 + source = "registry+https://github.com/rust-lang/crates.io-index" 355 + checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" 356 + dependencies = [ 357 + "cc", 358 + ] 359 + 360 + [[package]] 361 name = "colorchoice" 362 version = "1.0.4" 363 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 380 ] 381 382 [[package]] 383 + name = "core-foundation" 384 + version = "0.10.1" 385 + source = "registry+https://github.com/rust-lang/crates.io-index" 386 + checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" 387 + dependencies = [ 388 + "core-foundation-sys", 389 + "libc", 390 + ] 391 + 392 + [[package]] 393 name = "core-foundation-sys" 394 version = "0.8.7" 395 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 513 "futures-util", 514 "jetstream", 515 "log", 516 + "metrics-exporter-prometheus", 517 + "metrics-util", 518 + "prometheus", 519 "reqwest", 520 "serde", 521 "serde_json", 522 "sled", 523 "tokio", 524 + "tokio-metrics 0.3.1", 525 + "tokio-metrics-collector", 526 ] 527 528 [[package]] ··· 550 ] 551 552 [[package]] 553 + name = "dunce" 554 + version = "1.0.5" 555 + source = "registry+https://github.com/rust-lang/crates.io-index" 556 + checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" 557 + 558 + [[package]] 559 + name = "either" 560 + version = "1.15.0" 561 + source = "registry+https://github.com/rust-lang/crates.io-index" 562 + checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 563 + 564 + [[package]] 565 name = "encoding_rs" 566 version = "0.8.35" 567 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 571 ] 572 573 [[package]] 574 + name = "endian-type" 575 + version = "0.1.2" 576 + source = "registry+https://github.com/rust-lang/crates.io-index" 577 + checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" 578 + 579 + [[package]] 580 name = "env_filter" 581 version = "0.1.4" 582 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 646 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 647 648 [[package]] 649 + name = "foldhash" 650 + version = "0.1.5" 651 + source = "registry+https://github.com/rust-lang/crates.io-index" 652 + checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" 653 + 654 + [[package]] 655 name = "foreign-types" 656 version = "0.3.2" 657 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 686 ] 687 688 [[package]] 689 + name = "fs_extra" 690 + version = "1.3.0" 691 + source = "registry+https://github.com/rust-lang/crates.io-index" 692 + checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" 693 + 694 + [[package]] 695 name = "futures-channel" 696 version = "0.3.31" 697 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 789 ] 790 791 [[package]] 792 + name = "glob" 793 + version = "0.3.3" 794 + source = "registry+https://github.com/rust-lang/crates.io-index" 795 + checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" 796 + 797 + [[package]] 798 name = "h2" 799 version = "0.4.12" 800 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 815 816 [[package]] 817 name = "hashbrown" 818 + version = "0.15.5" 819 + source = "registry+https://github.com/rust-lang/crates.io-index" 820 + checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" 821 + dependencies = [ 822 + "foldhash", 823 + ] 824 + 825 + [[package]] 826 + name = "hashbrown" 827 version = "0.16.0" 828 source = "registry+https://github.com/rust-lang/crates.io-index" 829 checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" ··· 887 checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" 888 889 [[package]] 890 + name = "httpdate" 891 + version = "1.0.3" 892 + source = "registry+https://github.com/rust-lang/crates.io-index" 893 + checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" 894 + 895 + [[package]] 896 name = "hyper" 897 version = "1.7.0" 898 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 906 "http", 907 "http-body", 908 "httparse", 909 + "httpdate", 910 "itoa", 911 "pin-project-lite", 912 "pin-utils", ··· 925 "hyper", 926 "hyper-util", 927 "rustls", 928 + "rustls-native-certs", 929 "rustls-pki-types", 930 "tokio", 931 "tokio-rustls", ··· 1112 checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" 1113 dependencies = [ 1114 "equivalent", 1115 + "hashbrown 0.16.0", 1116 ] 1117 1118 [[package]] ··· 1158 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 1159 1160 [[package]] 1161 + name = "itertools" 1162 + version = "0.13.0" 1163 + source = "registry+https://github.com/rust-lang/crates.io-index" 1164 + checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" 1165 + dependencies = [ 1166 + "either", 1167 + ] 1168 + 1169 + [[package]] 1170 name = "itoa" 1171 version = "1.0.15" 1172 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1256 checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" 1257 1258 [[package]] 1259 + name = "libloading" 1260 + version = "0.8.9" 1261 + source = "registry+https://github.com/rust-lang/crates.io-index" 1262 + checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" 1263 + dependencies = [ 1264 + "cfg-if", 1265 + "windows-link 0.2.1", 1266 + ] 1267 + 1268 + [[package]] 1269 name = "linux-raw-sys" 1270 version = "0.11.0" 1271 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1310 checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" 1311 1312 [[package]] 1313 + name = "metrics" 1314 + version = "0.24.2" 1315 + source = "registry+https://github.com/rust-lang/crates.io-index" 1316 + checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5" 1317 + dependencies = [ 1318 + "ahash", 1319 + "portable-atomic", 1320 + ] 1321 + 1322 + [[package]] 1323 + name = "metrics-exporter-prometheus" 1324 + version = "0.17.2" 1325 + source = "registry+https://github.com/rust-lang/crates.io-index" 1326 + checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15" 1327 + dependencies = [ 1328 + "base64", 1329 + "http-body-util", 1330 + "hyper", 1331 + "hyper-rustls", 1332 + "hyper-util", 1333 + "indexmap", 1334 + "ipnet", 1335 + "metrics", 1336 + "metrics-util", 1337 + "quanta", 1338 + "thiserror 2.0.17", 1339 + "tokio", 1340 + "tracing", 1341 + ] 1342 + 1343 + [[package]] 1344 + name = "metrics-util" 1345 + version = "0.20.0" 1346 + source = "registry+https://github.com/rust-lang/crates.io-index" 1347 + checksum = "fe8db7a05415d0f919ffb905afa37784f71901c9a773188876984b4f769ab986" 1348 + dependencies = [ 1349 + "aho-corasick", 1350 + "crossbeam-epoch", 1351 + "crossbeam-utils", 1352 + "hashbrown 0.15.5", 1353 + "indexmap", 1354 + "metrics", 1355 + "ordered-float", 1356 + "quanta", 1357 + "radix_trie", 1358 + "rand 0.9.2", 1359 + "rand_xoshiro", 1360 + "sketches-ddsketch", 1361 + ] 1362 + 1363 + [[package]] 1364 name = "mime" 1365 version = "0.3.17" 1366 source = "registry+https://github.com/rust-lang/crates.io-index" 1367 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 1368 1369 [[package]] 1370 + name = "minimal-lexical" 1371 + version = "0.2.1" 1372 + source = "registry+https://github.com/rust-lang/crates.io-index" 1373 + checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" 1374 + 1375 + [[package]] 1376 name = "mio" 1377 version = "1.1.0" 1378 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1427 "openssl-probe", 1428 "openssl-sys", 1429 "schannel", 1430 + "security-framework 2.11.1", 1431 "security-framework-sys", 1432 "tempfile", 1433 ] 1434 1435 [[package]] 1436 + name = "nibble_vec" 1437 + version = "0.1.0" 1438 + source = "registry+https://github.com/rust-lang/crates.io-index" 1439 + checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" 1440 + dependencies = [ 1441 + "smallvec", 1442 + ] 1443 + 1444 + [[package]] 1445 + name = "nom" 1446 + version = "7.1.3" 1447 + source = "registry+https://github.com/rust-lang/crates.io-index" 1448 + checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" 1449 + dependencies = [ 1450 + "memchr", 1451 + "minimal-lexical", 1452 + ] 1453 + 1454 + [[package]] 1455 name = "num-traits" 1456 version = "0.2.19" 1457 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1517 ] 1518 1519 [[package]] 1520 + name = "ordered-float" 1521 + version = "4.6.0" 1522 + source = "registry+https://github.com/rust-lang/crates.io-index" 1523 + checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" 1524 + dependencies = [ 1525 + "num-traits", 1526 + ] 1527 + 1528 + [[package]] 1529 name = "parking_lot" 1530 version = "0.11.2" 1531 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1631 ] 1632 1633 [[package]] 1634 + name = "prettyplease" 1635 + version = "0.2.37" 1636 + source = "registry+https://github.com/rust-lang/crates.io-index" 1637 + checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" 1638 + dependencies = [ 1639 + "proc-macro2", 1640 + "syn 2.0.106", 1641 + ] 1642 + 1643 + [[package]] 1644 name = "proc-macro-error" 1645 version = "1.0.4" 1646 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1674 ] 1675 1676 [[package]] 1677 + name = "prometheus" 1678 + version = "0.14.0" 1679 + source = "registry+https://github.com/rust-lang/crates.io-index" 1680 + checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" 1681 + dependencies = [ 1682 + "cfg-if", 1683 + "fnv", 1684 + "lazy_static", 1685 + "memchr", 1686 + "parking_lot 0.12.5", 1687 + "protobuf", 1688 + "thiserror 2.0.17", 1689 + ] 1690 + 1691 + [[package]] 1692 + name = "protobuf" 1693 + version = "3.7.2" 1694 + source = "registry+https://github.com/rust-lang/crates.io-index" 1695 + checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" 1696 + dependencies = [ 1697 + "once_cell", 1698 + "protobuf-support", 1699 + "thiserror 1.0.69", 1700 + ] 1701 + 1702 + [[package]] 1703 + name = "protobuf-support" 1704 + version = "3.7.2" 1705 + source = "registry+https://github.com/rust-lang/crates.io-index" 1706 + checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" 1707 + dependencies = [ 1708 + "thiserror 1.0.69", 1709 + ] 1710 + 1711 + [[package]] 1712 + name = "quanta" 1713 + version = "0.12.6" 1714 + source = "registry+https://github.com/rust-lang/crates.io-index" 1715 + checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" 1716 + dependencies = [ 1717 + "crossbeam-utils", 1718 + "libc", 1719 + "once_cell", 1720 + "raw-cpuid", 1721 + "wasi", 1722 + "web-sys", 1723 + "winapi", 1724 + ] 1725 + 1726 + [[package]] 1727 name = "quote" 1728 version = "1.0.41" 1729 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1739 checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" 1740 1741 [[package]] 1742 + name = "radix_trie" 1743 + version = "0.2.1" 1744 + source = "registry+https://github.com/rust-lang/crates.io-index" 1745 + checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" 1746 + dependencies = [ 1747 + "endian-type", 1748 + "nibble_vec", 1749 + ] 1750 + 1751 + [[package]] 1752 name = "rand" 1753 version = "0.8.5" 1754 source = "registry+https://github.com/rust-lang/crates.io-index" 1755 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 1756 dependencies = [ 1757 "libc", 1758 + "rand_chacha 0.3.1", 1759 + "rand_core 0.6.4", 1760 + ] 1761 + 1762 + [[package]] 1763 + name = "rand" 1764 + version = "0.9.2" 1765 + source = "registry+https://github.com/rust-lang/crates.io-index" 1766 + checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" 1767 + dependencies = [ 1768 + "rand_chacha 0.9.0", 1769 + "rand_core 0.9.3", 1770 ] 1771 1772 [[package]] ··· 1776 checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 1777 dependencies = [ 1778 "ppv-lite86", 1779 + "rand_core 0.6.4", 1780 + ] 1781 + 1782 + [[package]] 1783 + name = "rand_chacha" 1784 + version = "0.9.0" 1785 + source = "registry+https://github.com/rust-lang/crates.io-index" 1786 + checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" 1787 + dependencies = [ 1788 + "ppv-lite86", 1789 + "rand_core 0.9.3", 1790 ] 1791 1792 [[package]] ··· 1796 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 1797 dependencies = [ 1798 "getrandom 0.2.16", 1799 + ] 1800 + 1801 + [[package]] 1802 + name = "rand_core" 1803 + version = "0.9.3" 1804 + source = "registry+https://github.com/rust-lang/crates.io-index" 1805 + checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" 1806 + dependencies = [ 1807 + "getrandom 0.3.4", 1808 + ] 1809 + 1810 + [[package]] 1811 + name = "rand_xoshiro" 1812 + version = "0.7.0" 1813 + source = "registry+https://github.com/rust-lang/crates.io-index" 1814 + checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" 1815 + dependencies = [ 1816 + "rand_core 0.9.3", 1817 + ] 1818 + 1819 + [[package]] 1820 + name = "raw-cpuid" 1821 + version = "11.6.0" 1822 + source = "registry+https://github.com/rust-lang/crates.io-index" 1823 + checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" 1824 + dependencies = [ 1825 + "bitflags 2.9.4", 1826 ] 1827 1828 [[package]] ··· 1927 ] 1928 1929 [[package]] 1930 + name = "rustc-hash" 1931 + version = "2.1.1" 1932 + source = "registry+https://github.com/rust-lang/crates.io-index" 1933 + checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" 1934 + 1935 + [[package]] 1936 name = "rustix" 1937 version = "1.1.2" 1938 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1951 source = "registry+https://github.com/rust-lang/crates.io-index" 1952 checksum = "751e04a496ca00bb97a5e043158d23d66b5aabf2e1d5aa2a0aaebb1aafe6f82c" 1953 dependencies = [ 1954 + "aws-lc-rs", 1955 "once_cell", 1956 "rustls-pki-types", 1957 "rustls-webpki", ··· 1960 ] 1961 1962 [[package]] 1963 + name = "rustls-native-certs" 1964 + version = "0.8.2" 1965 + source = "registry+https://github.com/rust-lang/crates.io-index" 1966 + checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" 1967 + dependencies = [ 1968 + "openssl-probe", 1969 + "rustls-pki-types", 1970 + "schannel", 1971 + "security-framework 3.5.1", 1972 + ] 1973 + 1974 + [[package]] 1975 name = "rustls-pki-types" 1976 version = "1.12.0" 1977 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1986 source = "registry+https://github.com/rust-lang/crates.io-index" 1987 checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf" 1988 dependencies = [ 1989 + "aws-lc-rs", 1990 "ring", 1991 "rustls-pki-types", 1992 "untrusted", ··· 2026 checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" 2027 dependencies = [ 2028 "bitflags 2.9.4", 2029 + "core-foundation 0.9.4", 2030 + "core-foundation-sys", 2031 + "libc", 2032 + "security-framework-sys", 2033 + ] 2034 + 2035 + [[package]] 2036 + name = "security-framework" 2037 + version = "3.5.1" 2038 + source = "registry+https://github.com/rust-lang/crates.io-index" 2039 + checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" 2040 + dependencies = [ 2041 + "bitflags 2.9.4", 2042 + "core-foundation 0.10.1", 2043 "core-foundation-sys", 2044 "libc", 2045 "security-framework-sys", ··· 2158 dependencies = [ 2159 "libc", 2160 ] 2161 + 2162 + [[package]] 2163 + name = "sketches-ddsketch" 2164 + version = "0.3.0" 2165 + source = "registry+https://github.com/rust-lang/crates.io-index" 2166 + checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" 2167 2168 [[package]] 2169 name = "slab" ··· 2303 checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" 2304 dependencies = [ 2305 "bitflags 2.9.4", 2306 + "core-foundation 0.9.4", 2307 "system-configuration-sys", 2308 ] 2309 ··· 2427 "pin-project-lite", 2428 "tokio", 2429 "tokio-stream", 2430 + ] 2431 + 2432 + [[package]] 2433 + name = "tokio-metrics" 2434 + version = "0.4.5" 2435 + source = "registry+https://github.com/rust-lang/crates.io-index" 2436 + checksum = "a01bbf7db0b3f5eee8930a119fe99bfa1438de1095fe3d26b25e652a5933ef3f" 2437 + dependencies = [ 2438 + "futures-util", 2439 + "pin-project-lite", 2440 + "tokio", 2441 + "tokio-stream", 2442 + ] 2443 + 2444 + [[package]] 2445 + name = "tokio-metrics-collector" 2446 + version = "0.3.1" 2447 + source = "registry+https://github.com/rust-lang/crates.io-index" 2448 + checksum = "f360c6fa7fd188e65904979ea07f1f44c57f0759ab18ecca88551c26acdf4cd6" 2449 + dependencies = [ 2450 + "lazy_static", 2451 + "parking_lot 0.12.5", 2452 + "prometheus", 2453 + "tokio", 2454 + "tokio-metrics 0.4.5", 2455 ] 2456 2457 [[package]] ··· 2606 "httparse", 2607 "log", 2608 "native-tls", 2609 + "rand 0.8.5", 2610 "sha1", 2611 "thiserror 1.0.69", 2612 "utf-8",
+1 -1
packages-rs/drainpipe-store/src/lib.rs
··· 4 use serde::{Deserialize, Serialize}; 5 use sled::Tree; 6 7 pub struct Store { 8 db: sled::Db, 9 cursor_tree: Tree, ··· 11 } 12 13 #[derive(Serialize, Deserialize, Debug)] 14 - 15 enum CursorInner { 16 V1(u64), 17 }
··· 4 use serde::{Deserialize, Serialize}; 5 use sled::Tree; 6 7 + #[derive(Debug, Clone)] 8 pub struct Store { 9 db: sled::Db, 10 cursor_tree: Tree, ··· 12 } 13 14 #[derive(Serialize, Deserialize, Debug)] 15 enum CursorInner { 16 V1(u64), 17 }
+6
packages-rs/drainpipe/Cargo.toml
··· 13 futures-util = "0.3" 14 jetstream = { path = "../jetstream" } 15 log = "0.4.22" 16 reqwest = { version = "0.12.4", features = ["json"] } 17 serde = { version = "1.0.215", features = ["derive"] } 18 serde_json = "1.0.132" 19 sled = "0.34.7" 20 tokio = { version = "1.38.0", features = ["full"] } 21 tokio-metrics = "0.3.1"
··· 13 futures-util = "0.3" 14 jetstream = { path = "../jetstream" } 15 log = "0.4.22" 16 + metrics-exporter-prometheus = { version = "0.17.2", features = [ 17 + "http-listener", 18 + ] } 19 + metrics-util = "0.20.0" 20 + prometheus = "0.14.0" 21 reqwest = { version = "0.12.4", features = ["json"] } 22 serde = { version = "1.0.215", features = ["derive"] } 23 serde_json = "1.0.132" 24 sled = "0.34.7" 25 tokio = { version = "1.38.0", features = ["full"] } 26 tokio-metrics = "0.3.1" 27 + tokio-metrics-collector = "0.3.1"
+27 -4
packages-rs/drainpipe/src/main.rs
··· 8 JetstreamReceiver, 9 }; 10 use serde_json::json; 11 use std::path::PathBuf; 12 use std::time::Duration; 13 use std::vec; ··· 25 26 env_logger::init(); 27 28 - let monitor = tokio_metrics::TaskMonitor::new(); 29 30 let config = Config::from_env()?; 31 let store = drainpipe_store::Store::open(&config.store_location)?; ··· 58 .await?; 59 60 let metric_logs_abort_handler = { 61 - let metrics_monitor = monitor.clone(); 62 tokio::spawn(async move { 63 for interval in metrics_monitor.intervals() { 64 - log::info!("{:?} per second", interval.instrumented_count as f64 / 5.0,); 65 tokio::time::sleep(Duration::from_millis(5000)).await; 66 } 67 }) ··· 71 loop { 72 match receiver.recv_async().await { 73 Ok(Ok(event)) => { 74 - monitor 75 .instrument(async { 76 if let JetstreamEvent::Commit(ref commit) = event { 77 println!("Received commit: {:?}", commit);
··· 8 JetstreamReceiver, 9 }; 10 use serde_json::json; 11 + use std::net::SocketAddr; 12 use std::path::PathBuf; 13 use std::time::Duration; 14 use std::vec; ··· 26 27 env_logger::init(); 28 29 + metrics_exporter_prometheus::PrometheusBuilder::new() 30 + .with_http_listener(SocketAddr::from(([0, 0, 0, 0], 9898))) 31 + .install()?; 32 + let task_collector = tokio_metrics_collector::default_task_collector(); 33 + prometheus::default_registry().register(Box::new(task_collector))?; 34 + 35 + let jetstream_events_monitor = tokio_metrics_collector::TaskMonitor::new(); 36 + task_collector.add("jetstream_events", jetstream_events_monitor.clone())?; 37 38 let config = Config::from_env()?; 39 let store = drainpipe_store::Store::open(&config.store_location)?; ··· 66 .await?; 67 68 let metric_logs_abort_handler = { 69 + let metrics_monitor = jetstream_events_monitor.clone(); 70 + let store = store.clone(); 71 tokio::spawn(async move { 72 for interval in metrics_monitor.intervals() { 73 + let lag_output = store.get_cursor().unwrap_or(None).and_then(|c| { 74 + let cursor_time = Utc.timestamp_micros(c as i64).earliest()?; 75 + let now = Utc::now(); 76 + let lag = now.signed_duration_since(cursor_time); 77 + Some(format!("Cursor lag: {}ms", lag.num_milliseconds())) 78 + }); 79 + if let Some(lag) = lag_output { 80 + log::info!( 81 + "{:?} per second; {}", 82 + interval.instrumented_count as f64 / 5.0, 83 + lag 84 + ); 85 + } else { 86 + log::info!("{:?} per second", interval.instrumented_count as f64 / 5.0); 87 + } 88 tokio::time::sleep(Duration::from_millis(5000)).await; 89 } 90 }) ··· 94 loop { 95 match receiver.recv_async().await { 96 Ok(Ok(event)) => { 97 + jetstream_events_monitor 98 .instrument(async { 99 if let JetstreamEvent::Commit(ref commit) = event { 100 println!("Received commit: {:?}", commit);