+332 -127

Cargo.lock

··· 70 71 [[package]] 72 name = "anyhow" 73 - version = "1.0.101" 74 source = "registry+https://github.com/rust-lang/crates.io-index" 75 - checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" 76 77 [[package]] 78 name = "arc-swap" ··· 91 92 [[package]] 93 name = "async-compression" 94 - version = "0.4.37" 95 source = "registry+https://github.com/rust-lang/crates.io-index" 96 - checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" 97 dependencies = [ 98 "compression-codecs", 99 "compression-core", ··· 156 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" 157 158 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 159 name = "axum" 160 version = "0.8.8" 161 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 288 289 [[package]] 290 name = "bitflags" 291 - version = "2.10.0" 292 source = "registry+https://github.com/rust-lang/crates.io-index" 293 - checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" 294 295 [[package]] 296 name = "block-buffer" ··· 303 304 [[package]] 305 name = "bon" 306 - version = "3.8.2" 307 source = "registry+https://github.com/rust-lang/crates.io-index" 308 - checksum = "234655ec178edd82b891e262ea7cf71f6584bcd09eff94db786be23f1821825c" 309 dependencies = [ 310 "bon-macros", 311 "rustversion", ··· 313 314 [[package]] 315 name = "bon-macros" 316 - version = "3.8.2" 317 source = "registry+https://github.com/rust-lang/crates.io-index" 318 - checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365" 319 dependencies = [ 320 "darling 0.23.0", 321 "ident_case", ··· 389 390 [[package]] 391 name = "bumpalo" 392 - version = "3.19.1" 393 source = "registry+https://github.com/rust-lang/crates.io-index" 394 - checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" 395 396 [[package]] 397 name = "byteorder" ··· 431 432 [[package]] 433 name = "cc" 434 - version = "1.2.55" 435 source = "registry+https://github.com/rust-lang/crates.io-index" 436 - checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" 437 dependencies = [ 438 "find-msvc-tools", 439 "jobserver", ··· 472 473 [[package]] 474 name = "chrono" 475 - version = "0.4.43" 476 source = "registry+https://github.com/rust-lang/crates.io-index" 477 - checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" 478 dependencies = [ 479 "iana-time-zone", 480 "js-sys", ··· 532 ] 533 534 [[package]] 0 0 0 0 0 0 0 0 0 535 name = "cobs" 536 version = "0.3.0" 537 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 558 559 [[package]] 560 name = "compression-codecs" 561 - version = "0.4.36" 562 source = "registry+https://github.com/rust-lang/crates.io-index" 563 - checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" 564 dependencies = [ 565 "brotli 8.0.2", 566 "compression-core", ··· 896 897 [[package]] 898 name = "deranged" 899 - version = "0.5.5" 900 source = "registry+https://github.com/rust-lang/crates.io-index" 901 - checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" 902 dependencies = [ 903 "powerfmt", 904 ] ··· 977 ] 978 979 [[package]] 0 0 0 0 0 0 980 name = "ecdsa" 981 version = "0.16.9" 982 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1137 1138 [[package]] 1139 name = "fjall" 1140 - version = "3.0.1" 1141 source = "registry+https://github.com/rust-lang/crates.io-index" 1142 - checksum = "4f69637c02d38ad1b0f003101d0195a60368130aa17d9ef78b1557d265a22093" 1143 dependencies = [ 1144 "byteorder-lite", 1145 "byteview", ··· 1191 dependencies = [ 1192 "percent-encoding", 1193 ] 0 0 0 0 0 0 1194 1195 [[package]] 1196 name = "futf" ··· 1204 1205 [[package]] 1206 name = "futures" 1207 - version = "0.3.31" 1208 source = "registry+https://github.com/rust-lang/crates.io-index" 1209 - checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" 1210 dependencies = [ 1211 "futures-channel", 1212 "futures-core", ··· 1219 1220 [[package]] 1221 name = "futures-buffered" 1222 - version = "0.2.12" 1223 source = "registry+https://github.com/rust-lang/crates.io-index" 1224 - checksum = "a8e0e1f38ec07ba4abbde21eed377082f17ccb988be9d988a5adbf4bafc118fd" 1225 dependencies = [ 1226 "cordyceps", 1227 "diatomic-waker", ··· 1232 1233 [[package]] 1234 name = "futures-channel" 1235 - version = "0.3.31" 1236 source = "registry+https://github.com/rust-lang/crates.io-index" 1237 - checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" 1238 dependencies = [ 1239 "futures-core", 1240 "futures-sink", ··· 1242 1243 [[package]] 1244 name = "futures-core" 1245 - version = "0.3.31" 1246 source = "registry+https://github.com/rust-lang/crates.io-index" 1247 - checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" 1248 1249 [[package]] 1250 name = "futures-executor" 1251 - version = "0.3.31" 1252 source = "registry+https://github.com/rust-lang/crates.io-index" 1253 - checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" 1254 dependencies = [ 1255 "futures-core", 1256 "futures-task", ··· 1259 1260 [[package]] 1261 name = "futures-io" 1262 - version = "0.3.31" 1263 source = "registry+https://github.com/rust-lang/crates.io-index" 1264 - checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" 1265 1266 [[package]] 1267 name = "futures-lite" ··· 1278 1279 [[package]] 1280 name = "futures-macro" 1281 - version = "0.3.31" 1282 source = "registry+https://github.com/rust-lang/crates.io-index" 1283 - checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" 1284 dependencies = [ 1285 "proc-macro2", 1286 "quote", ··· 1289 1290 [[package]] 1291 name = "futures-sink" 1292 - version = "0.3.31" 1293 source = "registry+https://github.com/rust-lang/crates.io-index" 1294 - checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" 1295 1296 [[package]] 1297 name = "futures-task" 1298 - version = "0.3.31" 1299 source = "registry+https://github.com/rust-lang/crates.io-index" 1300 - checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" 1301 1302 [[package]] 1303 name = "futures-util" 1304 - version = "0.3.31" 1305 source = "registry+https://github.com/rust-lang/crates.io-index" 1306 - checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" 1307 dependencies = [ 1308 "futures-channel", 1309 "futures-core", ··· 1313 "futures-task", 1314 "memchr", 1315 "pin-project-lite", 1316 - "pin-utils", 1317 "slab", 1318 ] 1319 ··· 1690 "n0-future 0.3.2", 1691 "ordermap", 1692 "rand 0.10.0", 1693 - "reqwest", 0 0 0 1694 "rmp-serde", 0 1695 "scc", 1696 "serde", 1697 "serde_bytes", ··· 1935 1936 [[package]] 1937 name = "inventory" 1938 - version = "0.3.21" 1939 source = "registry+https://github.com/rust-lang/crates.io-index" 1940 - checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e" 1941 dependencies = [ 1942 "rustversion", 1943 ] ··· 1956 1957 [[package]] 1958 name = "ipld-core" 1959 - version = "0.4.2" 1960 source = "registry+https://github.com/rust-lang/crates.io-index" 1961 - checksum = "104718b1cc124d92a6d01ca9c9258a7df311405debb3408c445a36452f9bf8db" 1962 dependencies = [ 1963 "cid", 1964 "serde", ··· 2012 [[package]] 2013 name = "jacquard" 2014 version = "0.9.5" 2015 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2016 dependencies = [ 2017 "bytes", 2018 "getrandom 0.2.17", ··· 2028 "n0-future 0.1.3", 2029 "regex", 2030 "regex-lite", 2031 - "reqwest", 2032 "serde", 2033 "serde_html_form", 2034 "serde_json", ··· 2043 [[package]] 2044 name = "jacquard-api" 2045 version = "0.9.5" 2046 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2047 dependencies = [ 2048 "bon", 2049 "bytes", ··· 2062 [[package]] 2063 name = "jacquard-axum" 2064 version = "0.9.6" 2065 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2066 dependencies = [ 2067 "axum", 2068 "bytes", ··· 2084 [[package]] 2085 name = "jacquard-common" 2086 version = "0.9.5" 2087 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2088 dependencies = [ 2089 "base64 0.22.1", 2090 "bon", ··· 2114 "regex", 2115 "regex-automata", 2116 "regex-lite", 2117 - "reqwest", 2118 "serde", 2119 "serde_bytes", 2120 "serde_html_form", ··· 2134 [[package]] 2135 name = "jacquard-derive" 2136 version = "0.9.5" 2137 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2138 dependencies = [ 2139 "heck 0.5.0", 2140 "jacquard-lexicon", ··· 2146 [[package]] 2147 name = "jacquard-identity" 2148 version = "0.9.5" 2149 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2150 dependencies = [ 2151 "bon", 2152 "bytes", ··· 2159 "mini-moka-wasm", 2160 "n0-future 0.1.3", 2161 "percent-encoding", 2162 - "reqwest", 2163 "serde", 2164 "serde_html_form", 2165 "serde_json", ··· 2173 [[package]] 2174 name = "jacquard-lexicon" 2175 version = "0.9.5" 2176 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2177 dependencies = [ 2178 "cid", 2179 "dashmap", ··· 2200 [[package]] 2201 name = "jacquard-oauth" 2202 version = "0.9.6" 2203 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2204 dependencies = [ 2205 "base64 0.22.1", 2206 "bytes", ··· 2232 [[package]] 2233 name = "jacquard-repo" 2234 version = "0.9.6" 2235 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2236 dependencies = [ 2237 "bytes", 2238 "cid", ··· 2325 2326 [[package]] 2327 name = "js-sys" 2328 - version = "0.3.85" 2329 source = "registry+https://github.com/rust-lang/crates.io-index" 2330 - checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" 2331 dependencies = [ 2332 "once_cell", 2333 "wasm-bindgen", ··· 2386 2387 [[package]] 2388 name = "libredox" 2389 - version = "0.1.12" 2390 source = "registry+https://github.com/rust-lang/crates.io-index" 2391 - checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" 2392 dependencies = [ 2393 "bitflags", 2394 "libc", 2395 - "redox_syscall 0.7.0", 0 2396 ] 2397 2398 [[package]] ··· 2458 2459 [[package]] 2460 name = "lsm-tree" 2461 - version = "3.0.1" 2462 source = "registry+https://github.com/rust-lang/crates.io-index" 2463 - checksum = "b875f1dfe14f557f805b167fb9b0fc54c5560c7a4bd6ae02535b2846f276a8cb" 2464 dependencies = [ 2465 "byteorder-lite", 2466 "byteview", ··· 2560 2561 [[package]] 2562 name = "memchr" 2563 - version = "2.7.6" 2564 source = "registry+https://github.com/rust-lang/crates.io-index" 2565 - checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" 2566 2567 [[package]] 2568 name = "miette" ··· 2622 [[package]] 2623 name = "mini-moka-wasm" 2624 version = "0.10.99" 2625 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2626 dependencies = [ 2627 "crossbeam-channel", 2628 "crossbeam-utils", ··· 2837 2838 [[package]] 2839 name = "objc2" 2840 - version = "0.6.3" 2841 source = "registry+https://github.com/rust-lang/crates.io-index" 2842 - checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" 2843 dependencies = [ 2844 "objc2-encode", 2845 ] ··· 2918 2919 [[package]] 2920 name = "owo-colors" 2921 - version = "4.2.3" 2922 source = "registry+https://github.com/rust-lang/crates.io-index" 2923 - checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" 2924 2925 [[package]] 2926 name = "oxilangtag" ··· 3037 3038 [[package]] 3039 name = "pin-project" 3040 - version = "1.1.10" 3041 source = "registry+https://github.com/rust-lang/crates.io-index" 3042 - checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" 3043 dependencies = [ 3044 "pin-project-internal", 3045 ] 3046 3047 [[package]] 3048 name = "pin-project-internal" 3049 - version = "1.1.10" 3050 source = "registry+https://github.com/rust-lang/crates.io-index" 3051 - checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" 3052 dependencies = [ 3053 "proc-macro2", 3054 "quote", ··· 3057 3058 [[package]] 3059 name = "pin-project-lite" 3060 - version = "0.2.16" 3061 source = "registry+https://github.com/rust-lang/crates.io-index" 3062 - checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" 3063 3064 [[package]] 3065 name = "pin-utils" ··· 3095 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" 3096 3097 [[package]] 0 0 0 0 0 0 3098 name = "portable-atomic" 3099 version = "1.13.1" 3100 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3226 source = "registry+https://github.com/rust-lang/crates.io-index" 3227 checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" 3228 dependencies = [ 0 3229 "bytes", 3230 "getrandom 0.3.4", 3231 "lru-slab", ··· 3357 3358 [[package]] 3359 name = "redox_syscall" 3360 - version = "0.7.0" 3361 source = "registry+https://github.com/rust-lang/crates.io-index" 3362 - checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" 3363 dependencies = [ 3364 "bitflags", 3365 ] ··· 3395 3396 [[package]] 3397 name = "regex-syntax" 3398 - version = "0.8.9" 3399 source = "registry+https://github.com/rust-lang/crates.io-index" 3400 - checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" 3401 3402 [[package]] 3403 name = "reqwest" ··· 3438 "url", 3439 "wasm-bindgen", 3440 "wasm-bindgen-futures", 3441 - "wasm-streams", 3442 "web-sys", 3443 "webpki-roots", 3444 ] 3445 3446 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3447 name = "resolv-conf" 3448 version = "0.7.6" 3449 source = "registry+https://github.com/rust-lang/crates.io-index" 3450 checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" 0 0 0 0 0 0 0 0 0 3451 3452 [[package]] 3453 name = "rfc6979" ··· 3570 3571 [[package]] 3572 name = "rustls" 3573 - version = "0.23.36" 3574 source = "registry+https://github.com/rust-lang/crates.io-index" 3575 - checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" 3576 dependencies = [ 0 0 3577 "once_cell", 3578 "ring", 3579 "rustls-pki-types", ··· 3605 ] 3606 3607 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3608 name = "rustls-webpki" 3609 version = "0.103.9" 3610 source = "registry+https://github.com/rust-lang/crates.io-index" 3611 checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" 3612 dependencies = [ 0 3613 "ring", 3614 "rustls-pki-types", 3615 "untrusted", ··· 3623 3624 [[package]] 3625 name = "ryu" 3626 - version = "1.0.22" 3627 source = "registry+https://github.com/rust-lang/crates.io-index" 3628 - checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" 3629 3630 [[package]] 3631 name = "saa" 3632 - version = "5.4.9" 3633 source = "registry+https://github.com/rust-lang/crates.io-index" 3634 - checksum = "da0ba8adb63e0deebd0744d8fc5bea394c08029159deaf680513fec1a3949144" 3635 3636 [[package]] 3637 name = "safemem" ··· 3650 3651 [[package]] 3652 name = "scc" 3653 - version = "3.5.6" 3654 source = "registry+https://github.com/rust-lang/crates.io-index" 3655 - checksum = "e4bd9d1727de391b6982925d830baad51692fa2aa6e337733c03d95121ca2793" 3656 dependencies = [ 3657 "saa", 3658 "sdd", ··· 3681 3682 [[package]] 3683 name = "sdd" 3684 - version = "4.6.2" 3685 source = "registry+https://github.com/rust-lang/crates.io-index" 3686 - checksum = "c25da4ae64b24edfcb0b0d30b96b2b0dbc64ec63aefeb6ec35bfc5ef167e5c9e" 3687 3688 [[package]] 3689 name = "sec1" ··· 3701 3702 [[package]] 3703 name = "security-framework" 3704 - version = "3.5.1" 3705 source = "registry+https://github.com/rust-lang/crates.io-index" 3706 - checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" 3707 dependencies = [ 3708 "bitflags", 3709 "core-foundation 0.10.1", ··· 3714 3715 [[package]] 3716 name = "security-framework-sys" 3717 - version = "2.15.0" 3718 source = "registry+https://github.com/rust-lang/crates.io-index" 3719 - checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" 3720 dependencies = [ 3721 "core-foundation-sys", 3722 "libc", ··· 3853 3854 [[package]] 3855 name = "serde_with" 3856 - version = "3.16.1" 3857 source = "registry+https://github.com/rust-lang/crates.io-index" 3858 - checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" 3859 dependencies = [ 3860 "base64 0.22.1", 3861 "chrono", ··· 3868 3869 [[package]] 3870 name = "serde_with_macros" 3871 - version = "3.16.1" 3872 source = "registry+https://github.com/rust-lang/crates.io-index" 3873 - checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" 3874 dependencies = [ 3875 "darling 0.21.3", 3876 "proc-macro2", ··· 4109 4110 [[package]] 4111 name = "syn" 4112 - version = "2.0.114" 4113 source = "registry+https://github.com/rust-lang/crates.io-index" 4114 - checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" 4115 dependencies = [ 4116 "proc-macro2", 4117 "quote", ··· 4638 4639 [[package]] 4640 name = "unicode-ident" 4641 - version = "1.0.22" 4642 source = "registry+https://github.com/rust-lang/crates.io-index" 4643 - checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" 4644 4645 [[package]] 4646 name = "unicode-linebreak" ··· 4784 4785 [[package]] 4786 name = "wasm-bindgen" 4787 - version = "0.2.108" 4788 source = "registry+https://github.com/rust-lang/crates.io-index" 4789 - checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" 4790 dependencies = [ 4791 "cfg-if", 4792 "once_cell", ··· 4797 4798 [[package]] 4799 name = "wasm-bindgen-futures" 4800 - version = "0.4.58" 4801 source = "registry+https://github.com/rust-lang/crates.io-index" 4802 - checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" 4803 dependencies = [ 4804 "cfg-if", 4805 "futures-util", ··· 4811 4812 [[package]] 4813 name = "wasm-bindgen-macro" 4814 - version = "0.2.108" 4815 source = "registry+https://github.com/rust-lang/crates.io-index" 4816 - checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" 4817 dependencies = [ 4818 "quote", 4819 "wasm-bindgen-macro-support", ··· 4821 4822 [[package]] 4823 name = "wasm-bindgen-macro-support" 4824 - version = "0.2.108" 4825 source = "registry+https://github.com/rust-lang/crates.io-index" 4826 - checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" 4827 dependencies = [ 4828 "bumpalo", 4829 "proc-macro2", ··· 4834 4835 [[package]] 4836 name = "wasm-bindgen-shared" 4837 - version = "0.2.108" 4838 source = "registry+https://github.com/rust-lang/crates.io-index" 4839 - checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" 4840 dependencies = [ 4841 "unicode-ident", 4842 ] ··· 4877 ] 4878 4879 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 4880 name = "wasmparser" 4881 version = "0.244.0" 4882 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4889 ] 4890 4891 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4892 name = "web-sys" 4893 - version = "0.3.85" 4894 source = "registry+https://github.com/rust-lang/crates.io-index" 4895 - checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" 4896 dependencies = [ 4897 "js-sys", 4898 "wasm-bindgen", ··· 4910 4911 [[package]] 4912 name = "webbrowser" 4913 - version = "1.0.6" 4914 source = "registry+https://github.com/rust-lang/crates.io-index" 4915 - checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97" 4916 dependencies = [ 4917 "core-foundation 0.10.1", 4918 "jni", ··· 4934 "markup5ever_rcdom", 4935 "serde_json", 4936 "url", 0 0 0 0 0 0 0 0 0 4937 ] 4938 4939 [[package]] ··· 5470 5471 [[package]] 5472 name = "zerocopy" 5473 - version = "0.8.39" 5474 source = "registry+https://github.com/rust-lang/crates.io-index" 5475 - checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" 5476 dependencies = [ 5477 "zerocopy-derive", 5478 ] 5479 5480 [[package]] 5481 name = "zerocopy-derive" 5482 - version = "0.8.39" 5483 source = "registry+https://github.com/rust-lang/crates.io-index" 5484 - checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" 5485 dependencies = [ 5486 "proc-macro2", 5487 "quote", ··· 5553 5554 [[package]] 5555 name = "zmij" 5556 - version = "1.0.19" 5557 source = "registry+https://github.com/rust-lang/crates.io-index" 5558 - checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" 5559 5560 [[package]] 5561 name = "zstd"

··· 70 71 [[package]] 72 name = "anyhow" 73 + version = "1.0.102" 74 source = "registry+https://github.com/rust-lang/crates.io-index" 75 + checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" 76 77 [[package]] 78 name = "arc-swap" ··· 91 92 [[package]] 93 name = "async-compression" 94 + version = "0.4.41" 95 source = "registry+https://github.com/rust-lang/crates.io-index" 96 + checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" 97 dependencies = [ 98 "compression-codecs", 99 "compression-core", ··· 156 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" 157 158 [[package]] 159 + name = "aws-lc-rs" 160 + version = "1.16.0" 161 + source = "registry+https://github.com/rust-lang/crates.io-index" 162 + checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" 163 + dependencies = [ 164 + "aws-lc-sys", 165 + "zeroize", 166 + ] 167 + 168 + [[package]] 169 + name = "aws-lc-sys" 170 + version = "0.37.1" 171 + source = "registry+https://github.com/rust-lang/crates.io-index" 172 + checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" 173 + dependencies = [ 174 + "cc", 175 + "cmake", 176 + "dunce", 177 + "fs_extra", 178 + ] 179 + 180 + [[package]] 181 name = "axum" 182 version = "0.8.8" 183 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 310 311 [[package]] 312 name = "bitflags" 313 + version = "2.11.0" 314 source = "registry+https://github.com/rust-lang/crates.io-index" 315 + checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" 316 317 [[package]] 318 name = "block-buffer" ··· 325 326 [[package]] 327 name = "bon" 328 + version = "3.9.0" 329 source = "registry+https://github.com/rust-lang/crates.io-index" 330 + checksum = "2d13a61f2963b88eef9c1be03df65d42f6996dfeac1054870d950fcf66686f83" 331 dependencies = [ 332 "bon-macros", 333 "rustversion", ··· 335 336 [[package]] 337 name = "bon-macros" 338 + version = "3.9.0" 339 source = "registry+https://github.com/rust-lang/crates.io-index" 340 + checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0" 341 dependencies = [ 342 "darling 0.23.0", 343 "ident_case", ··· 411 412 [[package]] 413 name = "bumpalo" 414 + version = "3.20.2" 415 source = "registry+https://github.com/rust-lang/crates.io-index" 416 + checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" 417 418 [[package]] 419 name = "byteorder" ··· 453 454 [[package]] 455 name = "cc" 456 + version = "1.2.56" 457 source = "registry+https://github.com/rust-lang/crates.io-index" 458 + checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" 459 dependencies = [ 460 "find-msvc-tools", 461 "jobserver", ··· 494 495 [[package]] 496 name = "chrono" 497 + version = "0.4.44" 498 source = "registry+https://github.com/rust-lang/crates.io-index" 499 + checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" 500 dependencies = [ 501 "iana-time-zone", 502 "js-sys", ··· 554 ] 555 556 [[package]] 557 + name = "cmake" 558 + version = "0.1.57" 559 + source = "registry+https://github.com/rust-lang/crates.io-index" 560 + checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" 561 + dependencies = [ 562 + "cc", 563 + ] 564 + 565 + [[package]] 566 name = "cobs" 567 version = "0.3.0" 568 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 589 590 [[package]] 591 name = "compression-codecs" 592 + version = "0.4.37" 593 source = "registry+https://github.com/rust-lang/crates.io-index" 594 + checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" 595 dependencies = [ 596 "brotli 8.0.2", 597 "compression-core", ··· 927 928 [[package]] 929 name = "deranged" 930 + version = "0.5.8" 931 source = "registry+https://github.com/rust-lang/crates.io-index" 932 + checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" 933 dependencies = [ 934 "powerfmt", 935 ] ··· 1008 ] 1009 1010 [[package]] 1011 + name = "dunce" 1012 + version = "1.0.5" 1013 + source = "registry+https://github.com/rust-lang/crates.io-index" 1014 + checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" 1015 + 1016 + [[package]] 1017 name = "ecdsa" 1018 version = "0.16.9" 1019 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1174 1175 [[package]] 1176 name = "fjall" 1177 + version = "3.0.3" 1178 source = "registry+https://github.com/rust-lang/crates.io-index" 1179 + checksum = "5ce4b4c3a755bae832077657db2727b29b9acc535bb471c6d91ab4fe408fd805" 1180 dependencies = [ 1181 "byteorder-lite", 1182 "byteview", ··· 1228 dependencies = [ 1229 "percent-encoding", 1230 ] 1231 + 1232 + [[package]] 1233 + name = "fs_extra" 1234 + version = "1.3.0" 1235 + source = "registry+https://github.com/rust-lang/crates.io-index" 1236 + checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" 1237 1238 [[package]] 1239 name = "futf" ··· 1247 1248 [[package]] 1249 name = "futures" 1250 + version = "0.3.32" 1251 source = "registry+https://github.com/rust-lang/crates.io-index" 1252 + checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" 1253 dependencies = [ 1254 "futures-channel", 1255 "futures-core", ··· 1262 1263 [[package]] 1264 name = "futures-buffered" 1265 + version = "0.2.13" 1266 source = "registry+https://github.com/rust-lang/crates.io-index" 1267 + checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" 1268 dependencies = [ 1269 "cordyceps", 1270 "diatomic-waker", ··· 1275 1276 [[package]] 1277 name = "futures-channel" 1278 + version = "0.3.32" 1279 source = "registry+https://github.com/rust-lang/crates.io-index" 1280 + checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" 1281 dependencies = [ 1282 "futures-core", 1283 "futures-sink", ··· 1285 1286 [[package]] 1287 name = "futures-core" 1288 + version = "0.3.32" 1289 source = "registry+https://github.com/rust-lang/crates.io-index" 1290 + checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" 1291 1292 [[package]] 1293 name = "futures-executor" 1294 + version = "0.3.32" 1295 source = "registry+https://github.com/rust-lang/crates.io-index" 1296 + checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" 1297 dependencies = [ 1298 "futures-core", 1299 "futures-task", ··· 1302 1303 [[package]] 1304 name = "futures-io" 1305 + version = "0.3.32" 1306 source = "registry+https://github.com/rust-lang/crates.io-index" 1307 + checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" 1308 1309 [[package]] 1310 name = "futures-lite" ··· 1321 1322 [[package]] 1323 name = "futures-macro" 1324 + version = "0.3.32" 1325 source = "registry+https://github.com/rust-lang/crates.io-index" 1326 + checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" 1327 dependencies = [ 1328 "proc-macro2", 1329 "quote", ··· 1332 1333 [[package]] 1334 name = "futures-sink" 1335 + version = "0.3.32" 1336 source = "registry+https://github.com/rust-lang/crates.io-index" 1337 + checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" 1338 1339 [[package]] 1340 name = "futures-task" 1341 + version = "0.3.32" 1342 source = "registry+https://github.com/rust-lang/crates.io-index" 1343 + checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" 1344 1345 [[package]] 1346 name = "futures-util" 1347 + version = "0.3.32" 1348 source = "registry+https://github.com/rust-lang/crates.io-index" 1349 + checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" 1350 dependencies = [ 1351 "futures-channel", 1352 "futures-core", ··· 1356 "futures-task", 1357 "memchr", 1358 "pin-project-lite", 0 1359 "slab", 1360 ] 1361 ··· 1732 "n0-future 0.3.2", 1733 "ordermap", 1734 "rand 0.10.0", 1735 + "reqwest 0.12.28", 1736 + "reqwest 0.13.2", 1737 + "reqwest-middleware", 1738 + "reqwest-retry", 1739 "rmp-serde", 1740 + "rustls", 1741 "scc", 1742 "serde", 1743 "serde_bytes", ··· 1981 1982 [[package]] 1983 name = "inventory" 1984 + version = "0.3.22" 1985 source = "registry+https://github.com/rust-lang/crates.io-index" 1986 + checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" 1987 dependencies = [ 1988 "rustversion", 1989 ] ··· 2002 2003 [[package]] 2004 name = "ipld-core" 2005 + version = "0.4.3" 2006 source = "registry+https://github.com/rust-lang/crates.io-index" 2007 + checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" 2008 dependencies = [ 2009 "cid", 2010 "serde", ··· 2058 [[package]] 2059 name = "jacquard" 2060 version = "0.9.5" 2061 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2062 dependencies = [ 2063 "bytes", 2064 "getrandom 0.2.17", ··· 2074 "n0-future 0.1.3", 2075 "regex", 2076 "regex-lite", 2077 + "reqwest 0.12.28", 2078 "serde", 2079 "serde_html_form", 2080 "serde_json", ··· 2089 [[package]] 2090 name = "jacquard-api" 2091 version = "0.9.5" 2092 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2093 dependencies = [ 2094 "bon", 2095 "bytes", ··· 2108 [[package]] 2109 name = "jacquard-axum" 2110 version = "0.9.6" 2111 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2112 dependencies = [ 2113 "axum", 2114 "bytes", ··· 2130 [[package]] 2131 name = "jacquard-common" 2132 version = "0.9.5" 2133 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2134 dependencies = [ 2135 "base64 0.22.1", 2136 "bon", ··· 2160 "regex", 2161 "regex-automata", 2162 "regex-lite", 2163 + "reqwest 0.12.28", 2164 "serde", 2165 "serde_bytes", 2166 "serde_html_form", ··· 2180 [[package]] 2181 name = "jacquard-derive" 2182 version = "0.9.5" 2183 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2184 dependencies = [ 2185 "heck 0.5.0", 2186 "jacquard-lexicon", ··· 2192 [[package]] 2193 name = "jacquard-identity" 2194 version = "0.9.5" 2195 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2196 dependencies = [ 2197 "bon", 2198 "bytes", ··· 2205 "mini-moka-wasm", 2206 "n0-future 0.1.3", 2207 "percent-encoding", 2208 + "reqwest 0.12.28", 2209 "serde", 2210 "serde_html_form", 2211 "serde_json", ··· 2219 [[package]] 2220 name = "jacquard-lexicon" 2221 version = "0.9.5" 2222 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2223 dependencies = [ 2224 "cid", 2225 "dashmap", ··· 2246 [[package]] 2247 name = "jacquard-oauth" 2248 version = "0.9.6" 2249 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2250 dependencies = [ 2251 "base64 0.22.1", 2252 "bytes", ··· 2278 [[package]] 2279 name = "jacquard-repo" 2280 version = "0.9.6" 2281 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2282 dependencies = [ 2283 "bytes", 2284 "cid", ··· 2371 2372 [[package]] 2373 name = "js-sys" 2374 + version = "0.3.91" 2375 source = "registry+https://github.com/rust-lang/crates.io-index" 2376 + checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" 2377 dependencies = [ 2378 "once_cell", 2379 "wasm-bindgen", ··· 2432 2433 [[package]] 2434 name = "libredox" 2435 + version = "0.1.14" 2436 source = "registry+https://github.com/rust-lang/crates.io-index" 2437 + checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" 2438 dependencies = [ 2439 "bitflags", 2440 "libc", 2441 + "plain", 2442 + "redox_syscall 0.7.3", 2443 ] 2444 2445 [[package]] ··· 2505 2506 [[package]] 2507 name = "lsm-tree" 2508 + version = "3.0.3" 2509 source = "registry+https://github.com/rust-lang/crates.io-index" 2510 + checksum = "341e13fa389d2e24023c2c2c110ae556b29fa59a2e734872de295863e11a6741" 2511 dependencies = [ 2512 "byteorder-lite", 2513 "byteview", ··· 2607 2608 [[package]] 2609 name = "memchr" 2610 + version = "2.8.0" 2611 source = "registry+https://github.com/rust-lang/crates.io-index" 2612 + checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" 2613 2614 [[package]] 2615 name = "miette" ··· 2669 [[package]] 2670 name = "mini-moka-wasm" 2671 version = "0.10.99" 2672 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2673 dependencies = [ 2674 "crossbeam-channel", 2675 "crossbeam-utils", ··· 2884 2885 [[package]] 2886 name = "objc2" 2887 + version = "0.6.4" 2888 source = "registry+https://github.com/rust-lang/crates.io-index" 2889 + checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" 2890 dependencies = [ 2891 "objc2-encode", 2892 ] ··· 2965 2966 [[package]] 2967 name = "owo-colors" 2968 + version = "4.3.0" 2969 source = "registry+https://github.com/rust-lang/crates.io-index" 2970 + checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" 2971 2972 [[package]] 2973 name = "oxilangtag" ··· 3084 3085 [[package]] 3086 name = "pin-project" 3087 + version = "1.1.11" 3088 source = "registry+https://github.com/rust-lang/crates.io-index" 3089 + checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" 3090 dependencies = [ 3091 "pin-project-internal", 3092 ] 3093 3094 [[package]] 3095 name = "pin-project-internal" 3096 + version = "1.1.11" 3097 source = "registry+https://github.com/rust-lang/crates.io-index" 3098 + checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" 3099 dependencies = [ 3100 "proc-macro2", 3101 "quote", ··· 3104 3105 [[package]] 3106 name = "pin-project-lite" 3107 + version = "0.2.17" 3108 source = "registry+https://github.com/rust-lang/crates.io-index" 3109 + checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" 3110 3111 [[package]] 3112 name = "pin-utils" ··· 3142 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" 3143 3144 [[package]] 3145 + name = "plain" 3146 + version = "0.2.3" 3147 + source = "registry+https://github.com/rust-lang/crates.io-index" 3148 + checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" 3149 + 3150 + [[package]] 3151 name = "portable-atomic" 3152 version = "1.13.1" 3153 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3279 source = "registry+https://github.com/rust-lang/crates.io-index" 3280 checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" 3281 dependencies = [ 3282 + "aws-lc-rs", 3283 "bytes", 3284 "getrandom 0.3.4", 3285 "lru-slab", ··· 3411 3412 [[package]] 3413 name = "redox_syscall" 3414 + version = "0.7.3" 3415 source = "registry+https://github.com/rust-lang/crates.io-index" 3416 + checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" 3417 dependencies = [ 3418 "bitflags", 3419 ] ··· 3449 3450 [[package]] 3451 name = "regex-syntax" 3452 + version = "0.8.10" 3453 source = "registry+https://github.com/rust-lang/crates.io-index" 3454 + checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" 3455 3456 [[package]] 3457 name = "reqwest" ··· 3492 "url", 3493 "wasm-bindgen", 3494 "wasm-bindgen-futures", 3495 + "wasm-streams 0.4.2", 3496 "web-sys", 3497 "webpki-roots", 3498 ] 3499 3500 [[package]] 3501 + name = "reqwest" 3502 + version = "0.13.2" 3503 + source = "registry+https://github.com/rust-lang/crates.io-index" 3504 + checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" 3505 + dependencies = [ 3506 + "base64 0.22.1", 3507 + "bytes", 3508 + "futures-core", 3509 + "futures-util", 3510 + "h2", 3511 + "http", 3512 + "http-body", 3513 + "http-body-util", 3514 + "hyper", 3515 + "hyper-rustls", 3516 + "hyper-util", 3517 + "js-sys", 3518 + "log", 3519 + "percent-encoding", 3520 + "pin-project-lite", 3521 + "quinn", 3522 + "rustls", 3523 + "rustls-pki-types", 3524 + "rustls-platform-verifier", 3525 + "serde", 3526 + "serde_json", 3527 + "sync_wrapper", 3528 + "tokio", 3529 + "tokio-rustls", 3530 + "tokio-util", 3531 + "tower", 3532 + "tower-http", 3533 + "tower-service", 3534 + "url", 3535 + "wasm-bindgen", 3536 + "wasm-bindgen-futures", 3537 + "wasm-streams 0.5.0", 3538 + "web-sys", 3539 + ] 3540 + 3541 + [[package]] 3542 + name = "reqwest-middleware" 3543 + version = "0.5.1" 3544 + source = "registry+https://github.com/rust-lang/crates.io-index" 3545 + checksum = "199dda04a536b532d0cc04d7979e39b1c763ea749bf91507017069c00b96056f" 3546 + dependencies = [ 3547 + "anyhow", 3548 + "async-trait", 3549 + "http", 3550 + "reqwest 0.13.2", 3551 + "thiserror 2.0.18", 3552 + "tower-service", 3553 + ] 3554 + 3555 + [[package]] 3556 + name = "reqwest-retry" 3557 + version = "0.9.1" 3558 + source = "registry+https://github.com/rust-lang/crates.io-index" 3559 + checksum = "fe2412db2af7d2268e7a5406be0431f37d9eb67ff390f35b395716f5f06c2eaa" 3560 + dependencies = [ 3561 + "anyhow", 3562 + "async-trait", 3563 + "futures", 3564 + "getrandom 0.2.17", 3565 + "http", 3566 + "hyper", 3567 + "reqwest 0.13.2", 3568 + "reqwest-middleware", 3569 + "retry-policies", 3570 + "thiserror 2.0.18", 3571 + "tokio", 3572 + "tracing", 3573 + "wasmtimer", 3574 + ] 3575 + 3576 + [[package]] 3577 name = "resolv-conf" 3578 version = "0.7.6" 3579 source = "registry+https://github.com/rust-lang/crates.io-index" 3580 checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" 3581 + 3582 + [[package]] 3583 + name = "retry-policies" 3584 + version = "0.5.1" 3585 + source = "registry+https://github.com/rust-lang/crates.io-index" 3586 + checksum = "46a4bd6027df676bcb752d3724db0ea3c0c5fc1dd0376fec51ac7dcaf9cc69be" 3587 + dependencies = [ 3588 + "rand 0.9.2", 3589 + ] 3590 3591 [[package]] 3592 name = "rfc6979" ··· 3709 3710 [[package]] 3711 name = "rustls" 3712 + version = "0.23.37" 3713 source = "registry+https://github.com/rust-lang/crates.io-index" 3714 + checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" 3715 dependencies = [ 3716 + "aws-lc-rs", 3717 + "log", 3718 "once_cell", 3719 "ring", 3720 "rustls-pki-types", ··· 3746 ] 3747 3748 [[package]] 3749 + name = "rustls-platform-verifier" 3750 + version = "0.6.2" 3751 + source = "registry+https://github.com/rust-lang/crates.io-index" 3752 + checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" 3753 + dependencies = [ 3754 + "core-foundation 0.10.1", 3755 + "core-foundation-sys", 3756 + "jni", 3757 + "log", 3758 + "once_cell", 3759 + "rustls", 3760 + "rustls-native-certs", 3761 + "rustls-platform-verifier-android", 3762 + "rustls-webpki", 3763 + "security-framework", 3764 + "security-framework-sys", 3765 + "webpki-root-certs", 3766 + "windows-sys 0.61.2", 3767 + ] 3768 + 3769 + [[package]] 3770 + name = "rustls-platform-verifier-android" 3771 + version = "0.1.1" 3772 + source = "registry+https://github.com/rust-lang/crates.io-index" 3773 + checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" 3774 + 3775 + [[package]] 3776 name = "rustls-webpki" 3777 version = "0.103.9" 3778 source = "registry+https://github.com/rust-lang/crates.io-index" 3779 checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" 3780 dependencies = [ 3781 + "aws-lc-rs", 3782 "ring", 3783 "rustls-pki-types", 3784 "untrusted", ··· 3792 3793 [[package]] 3794 name = "ryu" 3795 + version = "1.0.23" 3796 source = "registry+https://github.com/rust-lang/crates.io-index" 3797 + checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" 3798 3799 [[package]] 3800 name = "saa" 3801 + version = "5.5.0" 3802 source = "registry+https://github.com/rust-lang/crates.io-index" 3803 + checksum = "16c7f49c9d5caa3bf4b3106900484b447b9253fe99670ceb81cb6cb5027855e1" 3804 3805 [[package]] 3806 name = "safemem" ··· 3819 3820 [[package]] 3821 name = "scc" 3822 + version = "3.6.6" 3823 source = "registry+https://github.com/rust-lang/crates.io-index" 3824 + checksum = "6012e652611b2fdcb557a7b4be8cee00d8be19397c70011906a68aa4dac2fe37" 3825 dependencies = [ 3826 "saa", 3827 "sdd", ··· 3850 3851 [[package]] 3852 name = "sdd" 3853 + version = "4.6.5" 3854 source = "registry+https://github.com/rust-lang/crates.io-index" 3855 + checksum = "4becc2f27bd39aafb78a8a1b4e1d6877ed0a6b5bf096722ed538dc028367a9b5" 3856 3857 [[package]] 3858 name = "sec1" ··· 3870 3871 [[package]] 3872 name = "security-framework" 3873 + version = "3.7.0" 3874 source = "registry+https://github.com/rust-lang/crates.io-index" 3875 + checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" 3876 dependencies = [ 3877 "bitflags", 3878 "core-foundation 0.10.1", ··· 3883 3884 [[package]] 3885 name = "security-framework-sys" 3886 + version = "2.17.0" 3887 source = "registry+https://github.com/rust-lang/crates.io-index" 3888 + checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" 3889 dependencies = [ 3890 "core-foundation-sys", 3891 "libc", ··· 4022 4023 [[package]] 4024 name = "serde_with" 4025 + version = "3.17.0" 4026 source = "registry+https://github.com/rust-lang/crates.io-index" 4027 + checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" 4028 dependencies = [ 4029 "base64 0.22.1", 4030 "chrono", ··· 4037 4038 [[package]] 4039 name = "serde_with_macros" 4040 + version = "3.17.0" 4041 source = "registry+https://github.com/rust-lang/crates.io-index" 4042 + checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" 4043 dependencies = [ 4044 "darling 0.21.3", 4045 "proc-macro2", ··· 4278 4279 [[package]] 4280 name = "syn" 4281 + version = "2.0.117" 4282 source = "registry+https://github.com/rust-lang/crates.io-index" 4283 + checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" 4284 dependencies = [ 4285 "proc-macro2", 4286 "quote", ··· 4807 4808 [[package]] 4809 name = "unicode-ident" 4810 + version = "1.0.24" 4811 source = "registry+https://github.com/rust-lang/crates.io-index" 4812 + checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" 4813 4814 [[package]] 4815 name = "unicode-linebreak" ··· 4953 4954 [[package]] 4955 name = "wasm-bindgen" 4956 + version = "0.2.114" 4957 source = "registry+https://github.com/rust-lang/crates.io-index" 4958 + checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" 4959 dependencies = [ 4960 "cfg-if", 4961 "once_cell", ··· 4966 4967 [[package]] 4968 name = "wasm-bindgen-futures" 4969 + version = "0.4.64" 4970 source = "registry+https://github.com/rust-lang/crates.io-index" 4971 + checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" 4972 dependencies = [ 4973 "cfg-if", 4974 "futures-util", ··· 4980 4981 [[package]] 4982 name = "wasm-bindgen-macro" 4983 + version = "0.2.114" 4984 source = "registry+https://github.com/rust-lang/crates.io-index" 4985 + checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" 4986 dependencies = [ 4987 "quote", 4988 "wasm-bindgen-macro-support", ··· 4990 4991 [[package]] 4992 name = "wasm-bindgen-macro-support" 4993 + version = "0.2.114" 4994 source = "registry+https://github.com/rust-lang/crates.io-index" 4995 + checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" 4996 dependencies = [ 4997 "bumpalo", 4998 "proc-macro2", ··· 5003 5004 [[package]] 5005 name = "wasm-bindgen-shared" 5006 + version = "0.2.114" 5007 source = "registry+https://github.com/rust-lang/crates.io-index" 5008 + checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" 5009 dependencies = [ 5010 "unicode-ident", 5011 ] ··· 5046 ] 5047 5048 [[package]] 5049 + name = "wasm-streams" 5050 + version = "0.5.0" 5051 + source = "registry+https://github.com/rust-lang/crates.io-index" 5052 + checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" 5053 + dependencies = [ 5054 + "futures-util", 5055 + "js-sys", 5056 + "wasm-bindgen", 5057 + "wasm-bindgen-futures", 5058 + "web-sys", 5059 + ] 5060 + 5061 + [[package]] 5062 name = "wasmparser" 5063 version = "0.244.0" 5064 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5071 ] 5072 5073 [[package]] 5074 + name = "wasmtimer" 5075 + version = "0.4.3" 5076 + source = "registry+https://github.com/rust-lang/crates.io-index" 5077 + checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" 5078 + dependencies = [ 5079 + "futures", 5080 + "js-sys", 5081 + "parking_lot", 5082 + "pin-utils", 5083 + "slab", 5084 + "wasm-bindgen", 5085 + ] 5086 + 5087 + [[package]] 5088 name = "web-sys" 5089 + version = "0.3.91" 5090 source = "registry+https://github.com/rust-lang/crates.io-index" 5091 + checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" 5092 dependencies = [ 5093 "js-sys", 5094 "wasm-bindgen", ··· 5106 5107 [[package]] 5108 name = "webbrowser" 5109 + version = "1.1.0" 5110 source = "registry+https://github.com/rust-lang/crates.io-index" 5111 + checksum = "3f00bb839c1cf1e3036066614cbdcd035ecf215206691ea646aa3c60a24f68f2" 5112 dependencies = [ 5113 "core-foundation 0.10.1", 5114 "jni", ··· 5130 "markup5ever_rcdom", 5131 "serde_json", 5132 "url", 5133 + ] 5134 + 5135 + [[package]] 5136 + name = "webpki-root-certs" 5137 + version = "1.0.6" 5138 + source = "registry+https://github.com/rust-lang/crates.io-index" 5139 + checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" 5140 + dependencies = [ 5141 + "rustls-pki-types", 5142 ] 5143 5144 [[package]] ··· 5675 5676 [[package]] 5677 name = "zerocopy" 5678 + version = "0.8.40" 5679 source = "registry+https://github.com/rust-lang/crates.io-index" 5680 + checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" 5681 dependencies = [ 5682 "zerocopy-derive", 5683 ] 5684 5685 [[package]] 5686 name = "zerocopy-derive" 5687 + version = "0.8.40" 5688 source = "registry+https://github.com/rust-lang/crates.io-index" 5689 + checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" 5690 dependencies = [ 5691 "proc-macro2", 5692 "quote", ··· 5758 5759 [[package]] 5760 name = "zmij" 5761 + version = "1.0.21" 5762 source = "registry+https://github.com/rust-lang/crates.io-index" 5763 + checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" 5764 5765 [[package]] 5766 name = "zstd"

+4

Cargo.toml

··· 23 smol_str = "0.3" 24 futures = "0.3" 25 reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream", "gzip", "brotli", "zstd", "http2"], default-features = false } 0 0 0 26 axum = { version = "0.8.8", features = ["ws", "macros"] } 27 tower-http = { version = "0.6.6", features = ["cors", "trace"] } 28 tokio-stream = "0.1" ··· 49 glob = "0.3" 50 ordermap = { version = "1.1.0", features = ["serde"] } 51 arc-swap = "1.8.2" 0 52 53 [dev-dependencies] 54 tempfile = "3.26.0"

··· 23 smol_str = "0.3" 24 futures = "0.3" 25 reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream", "gzip", "brotli", "zstd", "http2"], default-features = false } 26 + reqwest-client = { package = "reqwest", version = "0.13.2", features = ["json", "rustls", "stream", "gzip", "brotli", "zstd", "http2"], default-features = false } 27 + reqwest-middleware = { version = "0.5.1", default-features = false, features = ["http2", "rustls"] } 28 + reqwest-retry = { version = "0.9.1" } 29 axum = { version = "0.8.8", features = ["ws", "macros"] } 30 tower-http = { version = "0.6.6", features = ["cors", "trace"] } 31 tokio-stream = "0.1" ··· 52 glob = "0.3" 53 ordermap = { version = "1.1.0", features = ["serde"] } 54 arc-swap = "1.8.2" 55 + rustls = { version = "0.23", features = ["ring"] } 56 57 [dev-dependencies] 58 tempfile = "3.26.0"

+1 -1

README.md

··· 42 | `NO_LZ4_COMPRESSION` | `false` | disable lz4 compression for storage. | 43 | `ENABLE_FIREHOSE` | `true` | whether to ingest relay subscriptions. | 44 | `ENABLE_BACKFILL` | `true` | whether to backfill from PDS instances. | 45 - | `ENABLE_CRAWLER` | `false` (if Filter), `true` (if Full) | whether to actively query the network for unknown repositories. | 46 | `DB_WORKER_THREADS` | `4` (`8` if full network) | database worker threads. | 47 | `DB_MAX_JOURNALING_SIZE_MB` | `512` (`1024` if full network) | max database journaling size in MB. | 48 | `DB_PENDING_MEMTABLE_SIZE_MB` | `64` (`192` if full network) | pending memtable size in MB. |

··· 42 | `NO_LZ4_COMPRESSION` | `false` | disable lz4 compression for storage. | 43 | `ENABLE_FIREHOSE` | `true` | whether to ingest relay subscriptions. | 44 | `ENABLE_BACKFILL` | `true` | whether to backfill from PDS instances. | 45 + | `ENABLE_CRAWLER` | `false` (if Filter), `true` (if Full) | whether to actively query the network for unknown repositories. when in `Filter` mode without wildcard (`*`) signals, the crawler uses `com.atproto.repo.listRecords` to verify if a discovered repository has matching records before queuing it for backfill, this will be a lot faster usually since most repos will get filtered out faster. | 46 | `DB_WORKER_THREADS` | `4` (`8` if full network) | database worker threads. | 47 | `DB_MAX_JOURNALING_SIZE_MB` | `512` (`1024` if full network) | max database journaling size in MB. | 48 | `DB_PENDING_MEMTABLE_SIZE_MB` | `64` (`192` if full network) | pending memtable size in MB. |

+151 -36

src/crawler/mod.rs

··· 1 use crate::db::{Db, keys, ser_repo_state}; 2 use crate::state::AppState; 3 use crate::types::RepoState; 4 - use jacquard::api::com_atproto::sync::list_repos::{ListRepos, ListReposOutput}; 5 - use jacquard::prelude::*; 6 - use jacquard_common::CowStr; 0 7 use miette::{IntoDiagnostic, Result}; 8 use rand::Rng; 9 use rand::rngs::SmallRng; 0 0 0 10 use smol_str::SmolStr; 11 use std::sync::Arc; 12 use std::time::Duration; ··· 16 pub struct Crawler { 17 state: Arc<AppState>, 18 relay_host: Url, 19 - http: reqwest::Client, 20 max_pending: usize, 21 resume_pending: usize, 22 } ··· 28 max_pending: usize, 29 resume_pending: usize, 30 ) -> Self { 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 31 Self { 32 state, 33 relay_host, 34 - http: reqwest::Client::new(), 35 max_pending, 36 resume_pending, 37 } ··· 40 pub async fn run(self) -> Result<()> { 41 info!("crawler started"); 42 0 0 0 0 0 0 0 0 0 0 0 43 let mut rng: SmallRng = rand::make_rng(); 44 45 let db = &self.state.db; 46 47 // 1. load cursor 48 let cursor_key = b"crawler_cursor"; 49 - let mut cursor: Option<SmolStr> = 50 - if let Ok(Some(bytes)) = Db::get(db.cursors.clone(), cursor_key.to_vec()).await { 0 51 let s = String::from_utf8_lossy(&bytes); 52 info!("resuming crawler from cursor: {}", s); 53 - Some(s.into()) 54 - } else { 55 - None 56 - }; 57 let mut was_throttled = false; 58 59 loop { ··· 97 } 98 99 // 2. fetch listrepos 100 - let req = ListRepos::new() 101 - .limit(1000) 102 - .maybe_cursor(cursor.clone().map(|c| CowStr::from(c.to_string()))) 103 - .build(); 104 - 105 - let mut url = self.relay_host.clone(); 106 - if url.scheme() == "wss" { 107 - url.set_scheme("https") 108 - .map_err(|_| miette::miette!("invalid url: {url}"))?; 109 - } else if url.scheme() == "ws" { 110 - url.set_scheme("http") 111 - .map_err(|_| miette::miette!("invalid url: {url}"))?; 112 } 113 - let res_result = self.http.xrpc(url).send(&req).await; 114 115 - let output: ListReposOutput = match res_result { 116 - Ok(res) => res.into_output().into_diagnostic()?, 0 0 0 0 0 0 0 0 0 0 117 Err(e) => { 118 - let e = e 119 - .source_err() 120 - .map(|e| e.to_string()) 121 - .unwrap_or_else(|| e.to_string()); 122 error!("crawler failed to list repos: {e}. retrying in 30s..."); 123 tokio::time::sleep(Duration::from_secs(30)).await; 124 continue; 125 } 126 }; 0 0 0 127 128 if output.repos.is_empty() { 129 info!("crawler finished enumeration (or empty page). sleeping for 1 hour."); ··· 135 136 let mut batch = db.inner.batch(); 137 let mut to_queue = Vec::new(); 0 0 0 0 0 0 0 138 139 // 3. process repos 0 140 for repo in output.repos { 141 - let did_key = keys::repo_key(&repo.did); 0 142 143 let excl_key = crate::db::filter::exclude_key(repo.did.as_str())?; 144 if db.filter.contains_key(&excl_key).into_diagnostic()? { ··· 147 148 // check if known 149 if !Db::contains_key(db.repos.clone(), &did_key).await? { 150 - trace!("crawler found new repo: {}", repo.did); 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 151 152 - let state = RepoState::backfilling(rng.next_u64()); 153 - batch.insert(&db.repos, &did_key, ser_repo_state(&state)?); 154 - batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key); 155 - to_queue.push(repo.did.clone()); 0 0 0 0 0 156 } 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 157 } 158 159 // 4. update cursor

··· 1 use crate::db::{Db, keys, ser_repo_state}; 2 use crate::state::AppState; 3 use crate::types::RepoState; 4 + use jacquard::IntoStatic; 5 + use jacquard_api::com_atproto::repo::list_records::ListRecordsOutput; 6 + use jacquard_api::com_atproto::sync::list_repos::ListReposOutput; 7 + use jacquard_common::types::string::Did; 8 use miette::{IntoDiagnostic, Result}; 9 use rand::Rng; 10 use rand::rngs::SmallRng; 11 + use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; 12 + use reqwest_retry::Jitter; 13 + use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff}; 14 use smol_str::SmolStr; 15 use std::sync::Arc; 16 use std::time::Duration; ··· 20 pub struct Crawler { 21 state: Arc<AppState>, 22 relay_host: Url, 23 + http: Arc<ClientWithMiddleware>, 24 max_pending: usize, 25 resume_pending: usize, 26 } ··· 32 max_pending: usize, 33 resume_pending: usize, 34 ) -> Self { 35 + let retry_policy = ExponentialBackoff::builder() 36 + .jitter(Jitter::Bounded) 37 + .build_with_max_retries(8); 38 + let reqwest_client = reqwest_client::Client::builder() 39 + .user_agent(concat!( 40 + env!("CARGO_PKG_NAME"), 41 + "/", 42 + env!("CARGO_PKG_VERSION") 43 + )) 44 + .gzip(true) 45 + .build() 46 + .expect("that reqwest will build"); 47 + 48 + let http = ClientBuilder::new(reqwest_client) 49 + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) 50 + .build(); 51 + let http = Arc::new(http); 52 + 53 Self { 54 state, 55 relay_host, 56 + http, 57 max_pending, 58 resume_pending, 59 } ··· 62 pub async fn run(self) -> Result<()> { 63 info!("crawler started"); 64 65 + let mut api_url = self.relay_host.clone(); 66 + if api_url.scheme() == "wss" { 67 + api_url 68 + .set_scheme("https") 69 + .map_err(|_| miette::miette!("invalid url: {api_url}"))?; 70 + } else if api_url.scheme() == "ws" { 71 + api_url 72 + .set_scheme("http") 73 + .map_err(|_| miette::miette!("invalid url: {api_url}"))?; 74 + } 75 + 76 let mut rng: SmallRng = rand::make_rng(); 77 78 let db = &self.state.db; 79 80 // 1. load cursor 81 let cursor_key = b"crawler_cursor"; 82 + let mut cursor: Option<SmolStr> = Db::get(db.cursors.clone(), cursor_key.to_vec()) 83 + .await? 84 + .map(|bytes| { 85 let s = String::from_utf8_lossy(&bytes); 86 info!("resuming crawler from cursor: {}", s); 87 + s.into() 88 + }); 0 0 89 let mut was_throttled = false; 90 91 loop { ··· 129 } 130 131 // 2. fetch listrepos 132 + let mut list_repos_url = api_url 133 + .join("/xrpc/com.atproto.sync.listRepos") 134 + .into_diagnostic()?; 135 + list_repos_url 136 + .query_pairs_mut() 137 + .append_pair("limit", "1000"); 138 + if let Some(c) = &cursor { 139 + list_repos_url 140 + .query_pairs_mut() 141 + .append_pair("cursor", c.as_str()); 0 0 142 } 0 143 144 + let res_result = self.http.get(list_repos_url.clone()).send().await; 145 + let bytes = match res_result { 146 + Ok(res) => match res.bytes().await { 147 + Ok(b) => b, 148 + Err(e) => { 149 + error!( 150 + "crawler failed to parse list repos response: {e}. retrying in 30s..." 151 + ); 152 + tokio::time::sleep(Duration::from_secs(30)).await; 153 + continue; 154 + } 155 + }, 156 Err(e) => { 0 0 0 0 157 error!("crawler failed to list repos: {e}. retrying in 30s..."); 158 tokio::time::sleep(Duration::from_secs(30)).await; 159 continue; 160 } 161 }; 162 + let output = serde_json::from_slice::<ListReposOutput>(&bytes) 163 + .into_diagnostic()? 164 + .into_static(); 165 166 if output.repos.is_empty() { 167 info!("crawler finished enumeration (or empty page). sleeping for 1 hour."); ··· 173 174 let mut batch = db.inner.batch(); 175 let mut to_queue = Vec::new(); 176 + let filter = self.state.filter.load(); 177 + // we can check whether or not to backfill repos faster if we only have to check 178 + // certain known signals, since we can just listRecords for those signals 179 + // if we have glob signals we cant do this since we dont know what signals to check 180 + let check_signals = filter.mode == crate::filter::FilterMode::Filter 181 + && !filter.signals.is_empty() 182 + && !filter.has_glob_signals(); 183 184 // 3. process repos 185 + let mut unknown_repos = Vec::new(); 186 for repo in output.repos { 187 + let parsed_did: Did = repo.did.parse().unwrap(); 188 + let did_key = keys::repo_key(&parsed_did); 189 190 let excl_key = crate::db::filter::exclude_key(repo.did.as_str())?; 191 if db.filter.contains_key(&excl_key).into_diagnostic()? { ··· 194 195 // check if known 196 if !Db::contains_key(db.repos.clone(), &did_key).await? { 197 + unknown_repos.push(repo); 198 + } 199 + } 200 + 201 + let mut valid_repos = Vec::new(); 202 + if check_signals && !unknown_repos.is_empty() { 203 + let mut set = tokio::task::JoinSet::new(); 204 + for repo in unknown_repos { 205 + let http = self.http.clone(); 206 + let api_url = api_url.clone(); 207 + let filter = filter.clone(); 208 + set.spawn(async move { 209 + let mut found_signal = false; 210 + for signal in filter.signals.iter() { 211 + let mut list_records_url = 212 + api_url.join("/xrpc/com.atproto.repo.listRecords").unwrap(); 213 + list_records_url 214 + .query_pairs_mut() 215 + .append_pair("repo", &repo.did) 216 + .append_pair("collection", signal) 217 + .append_pair("limit", "1"); 218 + 219 + match http.get(list_records_url).send().await { 220 + Ok(res) => { 221 + let Ok(bytes) = res.bytes().await else { 222 + error!("failed to read bytes from listRecords response for repo {}, signal {signal}", repo.did); 223 + continue; 224 + }; 225 + if let Ok(out) = serde_json::from_slice::<ListRecordsOutput>(&bytes) { 226 + if !out.records.is_empty() { 227 + found_signal = true; 228 + break; 229 + } 230 + } 231 + } 232 + Err(e) => { 233 + error!( 234 + "failed to listRecords for repo {}, signal {signal}: {e}", 235 + repo.did 236 + ); 237 + continue; 238 + } 239 + } 240 + } 241 242 + if !found_signal { 243 + trace!( 244 + "crawler skipped repo {}: no records match signals", 245 + repo.did 246 + ); 247 + } 248 + 249 + (repo, found_signal) 250 + }); 251 } 252 + 253 + while let Some(res) = set.join_next().await { 254 + let (repo, found_signal) = res.into_diagnostic()?; 255 + if found_signal { 256 + valid_repos.push(repo); 257 + } 258 + } 259 + } else { 260 + valid_repos = unknown_repos; 261 + } 262 + 263 + for repo in valid_repos { 264 + let parsed_did: Did = repo.did.parse().unwrap(); 265 + let did_key = keys::repo_key(&parsed_did); 266 + trace!("crawler found new repo: {}", repo.did); 267 + 268 + let state = RepoState::backfilling(rng.next_u64()); 269 + batch.insert(&db.repos, &did_key, ser_repo_state(&state)?); 270 + batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key); 271 + to_queue.push(repo.did.clone()); 272 } 273 274 // 4. update cursor

+12 -4

src/db/filter.rs

··· 1 use fjall::{Keyspace, OwnedWriteBatch}; 0 0 2 use miette::{IntoDiagnostic, Result}; 3 4 use crate::db::types::TrimmedDid; ··· 111 for guard in ks.prefix(signal_prefix) { 112 let (k, _) = guard.into_inner().into_diagnostic()?; 113 let val = std::str::from_utf8(&k[signal_prefix.len()..]).into_diagnostic()?; 114 - config.signals.push(smol_str::SmolStr::new(val)); 115 } 116 117 let col_prefix = [COLLECTION_PREFIX, SEP]; 118 for guard in ks.prefix(col_prefix) { 119 let (k, _) = guard.into_inner().into_diagnostic()?; 120 let val = std::str::from_utf8(&k[col_prefix.len()..]).into_diagnostic()?; 121 - config.collections.push(smol_str::SmolStr::new(val)); 122 } 123 124 Ok(config) ··· 196 197 let config = load(&ks)?; 198 assert_eq!(config.mode, FilterMode::Filter); 199 - assert_eq!(config.signals, vec!["a.b.c"]); 200 - assert_eq!(config.collections, vec!["d.e.f"]); 0 0 0 0 0 0 201 202 let excludes = read_set(&ks, EXCLUDE_PREFIX)?; 203 assert_eq!(excludes, vec!["did:plc:yk4q3id7id6p5z3bypvshc64"]);

··· 1 use fjall::{Keyspace, OwnedWriteBatch}; 2 + use jacquard::IntoStatic; 3 + use jacquard::types::nsid::Nsid; 4 use miette::{IntoDiagnostic, Result}; 5 6 use crate::db::types::TrimmedDid; ··· 113 for guard in ks.prefix(signal_prefix) { 114 let (k, _) = guard.into_inner().into_diagnostic()?; 115 let val = std::str::from_utf8(&k[signal_prefix.len()..]).into_diagnostic()?; 116 + config.signals.push(Nsid::new(val)?.into_static()); 117 } 118 119 let col_prefix = [COLLECTION_PREFIX, SEP]; 120 for guard in ks.prefix(col_prefix) { 121 let (k, _) = guard.into_inner().into_diagnostic()?; 122 let val = std::str::from_utf8(&k[col_prefix.len()..]).into_diagnostic()?; 123 + config.collections.push(Nsid::new(val)?.into_static()); 124 } 125 126 Ok(config) ··· 198 199 let config = load(&ks)?; 200 assert_eq!(config.mode, FilterMode::Filter); 201 + assert_eq!( 202 + config.signals, 203 + vec![Nsid::new("a.b.c").unwrap().into_static()] 204 + ); 205 + assert_eq!( 206 + config.collections, 207 + vec![Nsid::new("d.e.f").unwrap().into_static()] 208 + ); 209 210 let excludes = read_set(&ks, EXCLUDE_PREFIX)?; 211 assert_eq!(excludes, vec!["did:plc:yk4q3id7id6p5z3bypvshc64"]);

+7 -8

src/filter.rs

··· 0 1 use serde::{Deserialize, Serialize}; 2 - use smol_str::SmolStr; 3 use std::sync::Arc; 4 5 pub type FilterHandle = Arc<arc_swap::ArcSwap<FilterConfig>>; ··· 25 Full = 2, 26 } 27 28 - /// hot-path in-memory config: only the small fields needed on every event. 29 - /// dids and excludes are large sets kept in the filter keyspace only. 30 #[derive(Debug, Clone, Serialize)] 31 pub struct FilterConfig { 32 pub mode: FilterMode, 33 - pub signals: Vec<SmolStr>, 34 - pub collections: Vec<SmolStr>, 35 } 36 37 impl FilterConfig { ··· 43 } 44 } 45 46 - /// returns true if the collection matches the content filter. 47 - /// if collections is empty, all collections match. 48 pub fn matches_collection(&self, collection: &str) -> bool { 49 if self.collections.is_empty() { 50 return true; ··· 52 self.collections.iter().any(|p| nsid_matches(p, collection)) 53 } 54 55 - /// returns true if the commit touches a collection covered by a signal. 56 pub fn matches_signal(&self, collection: &str) -> bool { 57 self.signals.iter().any(|p| nsid_matches(p, collection)) 0 0 0 0 58 } 59 } 60

··· 1 + use jacquard::types::nsid::Nsid; 2 use serde::{Deserialize, Serialize}; 0 3 use std::sync::Arc; 4 5 pub type FilterHandle = Arc<arc_swap::ArcSwap<FilterConfig>>; ··· 25 Full = 2, 26 } 27 0 0 28 #[derive(Debug, Clone, Serialize)] 29 pub struct FilterConfig { 30 pub mode: FilterMode, 31 + pub signals: Vec<Nsid<'static>>, 32 + pub collections: Vec<Nsid<'static>>, 33 } 34 35 impl FilterConfig { ··· 41 } 42 } 43 0 0 44 pub fn matches_collection(&self, collection: &str) -> bool { 45 if self.collections.is_empty() { 46 return true; ··· 48 self.collections.iter().any(|p| nsid_matches(p, collection)) 49 } 50 0 51 pub fn matches_signal(&self, collection: &str) -> bool { 52 self.signals.iter().any(|p| nsid_matches(p, collection)) 53 + } 54 + 55 + pub fn has_glob_signals(&self) -> bool { 56 + self.signals.iter().any(|s| s.ends_with(".*")) 57 } 58 } 59