at protocol indexer with flexible filtering, xrpc queries, and a cursor-backed event stream, built on fjall
at-protocol atproto indexer rust fjall

[crawler] check for signals using listRecords if we know the signals

ptr.pet f89fd624 53ac9ddd

verified
+507 -176
+332 -127
Cargo.lock
··· 70 70 71 71 [[package]] 72 72 name = "anyhow" 73 - version = "1.0.101" 73 + version = "1.0.102" 74 74 source = "registry+https://github.com/rust-lang/crates.io-index" 75 - checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" 75 + checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" 76 76 77 77 [[package]] 78 78 name = "arc-swap" ··· 91 91 92 92 [[package]] 93 93 name = "async-compression" 94 - version = "0.4.37" 94 + version = "0.4.41" 95 95 source = "registry+https://github.com/rust-lang/crates.io-index" 96 - checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" 96 + checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" 97 97 dependencies = [ 98 98 "compression-codecs", 99 99 "compression-core", ··· 156 156 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" 157 157 158 158 [[package]] 159 + name = "aws-lc-rs" 160 + version = "1.16.0" 161 + source = "registry+https://github.com/rust-lang/crates.io-index" 162 + checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" 163 + dependencies = [ 164 + "aws-lc-sys", 165 + "zeroize", 166 + ] 167 + 168 + [[package]] 169 + name = "aws-lc-sys" 170 + version = "0.37.1" 171 + source = "registry+https://github.com/rust-lang/crates.io-index" 172 + checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" 173 + dependencies = [ 174 + "cc", 175 + "cmake", 176 + "dunce", 177 + "fs_extra", 178 + ] 179 + 180 + [[package]] 159 181 name = "axum" 160 182 version = "0.8.8" 161 183 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 288 310 289 311 [[package]] 290 312 name = "bitflags" 291 - version = "2.10.0" 313 + version = "2.11.0" 292 314 source = "registry+https://github.com/rust-lang/crates.io-index" 293 - checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" 315 + checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" 294 316 295 317 [[package]] 296 318 name = "block-buffer" ··· 303 325 304 326 [[package]] 305 327 name = "bon" 306 - version = "3.8.2" 328 + version = "3.9.0" 307 329 source = "registry+https://github.com/rust-lang/crates.io-index" 308 - checksum = "234655ec178edd82b891e262ea7cf71f6584bcd09eff94db786be23f1821825c" 330 + checksum = "2d13a61f2963b88eef9c1be03df65d42f6996dfeac1054870d950fcf66686f83" 309 331 dependencies = [ 310 332 "bon-macros", 311 333 "rustversion", ··· 313 335 314 336 [[package]] 315 337 name = "bon-macros" 316 - version = "3.8.2" 338 + version = "3.9.0" 317 339 source = "registry+https://github.com/rust-lang/crates.io-index" 318 - checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365" 340 + checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0" 319 341 dependencies = [ 320 342 "darling 0.23.0", 321 343 "ident_case", ··· 389 411 390 412 [[package]] 391 413 name = "bumpalo" 392 - version = "3.19.1" 414 + version = "3.20.2" 393 415 source = "registry+https://github.com/rust-lang/crates.io-index" 394 - checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" 416 + checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" 395 417 396 418 [[package]] 397 419 name = "byteorder" ··· 431 453 432 454 [[package]] 433 455 name = "cc" 434 - version = "1.2.55" 456 + version = "1.2.56" 435 457 source = "registry+https://github.com/rust-lang/crates.io-index" 436 - checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" 458 + checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" 437 459 dependencies = [ 438 460 "find-msvc-tools", 439 461 "jobserver", ··· 472 494 473 495 [[package]] 474 496 name = "chrono" 475 - version = "0.4.43" 497 + version = "0.4.44" 476 498 source = "registry+https://github.com/rust-lang/crates.io-index" 477 - checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" 499 + checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" 478 500 dependencies = [ 479 501 "iana-time-zone", 480 502 "js-sys", ··· 532 554 ] 533 555 534 556 [[package]] 557 + name = "cmake" 558 + version = "0.1.57" 559 + source = "registry+https://github.com/rust-lang/crates.io-index" 560 + checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" 561 + dependencies = [ 562 + "cc", 563 + ] 564 + 565 + [[package]] 535 566 name = "cobs" 536 567 version = "0.3.0" 537 568 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 558 589 559 590 [[package]] 560 591 name = "compression-codecs" 561 - version = "0.4.36" 592 + version = "0.4.37" 562 593 source = "registry+https://github.com/rust-lang/crates.io-index" 563 - checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" 594 + checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" 564 595 dependencies = [ 565 596 "brotli 8.0.2", 566 597 "compression-core", ··· 896 927 897 928 [[package]] 898 929 name = "deranged" 899 - version = "0.5.5" 930 + version = "0.5.8" 900 931 source = "registry+https://github.com/rust-lang/crates.io-index" 901 - checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" 932 + checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" 902 933 dependencies = [ 903 934 "powerfmt", 904 935 ] ··· 977 1008 ] 978 1009 979 1010 [[package]] 1011 + name = "dunce" 1012 + version = "1.0.5" 1013 + source = "registry+https://github.com/rust-lang/crates.io-index" 1014 + checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" 1015 + 1016 + [[package]] 980 1017 name = "ecdsa" 981 1018 version = "0.16.9" 982 1019 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1137 1174 1138 1175 [[package]] 1139 1176 name = "fjall" 1140 - version = "3.0.1" 1177 + version = "3.0.3" 1141 1178 source = "registry+https://github.com/rust-lang/crates.io-index" 1142 - checksum = "4f69637c02d38ad1b0f003101d0195a60368130aa17d9ef78b1557d265a22093" 1179 + checksum = "5ce4b4c3a755bae832077657db2727b29b9acc535bb471c6d91ab4fe408fd805" 1143 1180 dependencies = [ 1144 1181 "byteorder-lite", 1145 1182 "byteview", ··· 1191 1228 dependencies = [ 1192 1229 "percent-encoding", 1193 1230 ] 1231 + 1232 + [[package]] 1233 + name = "fs_extra" 1234 + version = "1.3.0" 1235 + source = "registry+https://github.com/rust-lang/crates.io-index" 1236 + checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" 1194 1237 1195 1238 [[package]] 1196 1239 name = "futf" ··· 1204 1247 1205 1248 [[package]] 1206 1249 name = "futures" 1207 - version = "0.3.31" 1250 + version = "0.3.32" 1208 1251 source = "registry+https://github.com/rust-lang/crates.io-index" 1209 - checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" 1252 + checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" 1210 1253 dependencies = [ 1211 1254 "futures-channel", 1212 1255 "futures-core", ··· 1219 1262 1220 1263 [[package]] 1221 1264 name = "futures-buffered" 1222 - version = "0.2.12" 1265 + version = "0.2.13" 1223 1266 source = "registry+https://github.com/rust-lang/crates.io-index" 1224 - checksum = "a8e0e1f38ec07ba4abbde21eed377082f17ccb988be9d988a5adbf4bafc118fd" 1267 + checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" 1225 1268 dependencies = [ 1226 1269 "cordyceps", 1227 1270 "diatomic-waker", ··· 1232 1275 1233 1276 [[package]] 1234 1277 name = "futures-channel" 1235 - version = "0.3.31" 1278 + version = "0.3.32" 1236 1279 source = "registry+https://github.com/rust-lang/crates.io-index" 1237 - checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" 1280 + checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" 1238 1281 dependencies = [ 1239 1282 "futures-core", 1240 1283 "futures-sink", ··· 1242 1285 1243 1286 [[package]] 1244 1287 name = "futures-core" 1245 - version = "0.3.31" 1288 + version = "0.3.32" 1246 1289 source = "registry+https://github.com/rust-lang/crates.io-index" 1247 - checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" 1290 + checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" 1248 1291 1249 1292 [[package]] 1250 1293 name = "futures-executor" 1251 - version = "0.3.31" 1294 + version = "0.3.32" 1252 1295 source = "registry+https://github.com/rust-lang/crates.io-index" 1253 - checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" 1296 + checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" 1254 1297 dependencies = [ 1255 1298 "futures-core", 1256 1299 "futures-task", ··· 1259 1302 1260 1303 [[package]] 1261 1304 name = "futures-io" 1262 - version = "0.3.31" 1305 + version = "0.3.32" 1263 1306 source = "registry+https://github.com/rust-lang/crates.io-index" 1264 - checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" 1307 + checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" 1265 1308 1266 1309 [[package]] 1267 1310 name = "futures-lite" ··· 1278 1321 1279 1322 [[package]] 1280 1323 name = "futures-macro" 1281 - version = "0.3.31" 1324 + version = "0.3.32" 1282 1325 source = "registry+https://github.com/rust-lang/crates.io-index" 1283 - checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" 1326 + checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" 1284 1327 dependencies = [ 1285 1328 "proc-macro2", 1286 1329 "quote", ··· 1289 1332 1290 1333 [[package]] 1291 1334 name = "futures-sink" 1292 - version = "0.3.31" 1335 + version = "0.3.32" 1293 1336 source = "registry+https://github.com/rust-lang/crates.io-index" 1294 - checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" 1337 + checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" 1295 1338 1296 1339 [[package]] 1297 1340 name = "futures-task" 1298 - version = "0.3.31" 1341 + version = "0.3.32" 1299 1342 source = "registry+https://github.com/rust-lang/crates.io-index" 1300 - checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" 1343 + checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" 1301 1344 1302 1345 [[package]] 1303 1346 name = "futures-util" 1304 - version = "0.3.31" 1347 + version = "0.3.32" 1305 1348 source = "registry+https://github.com/rust-lang/crates.io-index" 1306 - checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" 1349 + checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" 1307 1350 dependencies = [ 1308 1351 "futures-channel", 1309 1352 "futures-core", ··· 1313 1356 "futures-task", 1314 1357 "memchr", 1315 1358 "pin-project-lite", 1316 - "pin-utils", 1317 1359 "slab", 1318 1360 ] 1319 1361 ··· 1690 1732 "n0-future 0.3.2", 1691 1733 "ordermap", 1692 1734 "rand 0.10.0", 1693 - "reqwest", 1735 + "reqwest 0.12.28", 1736 + "reqwest 0.13.2", 1737 + "reqwest-middleware", 1738 + "reqwest-retry", 1694 1739 "rmp-serde", 1740 + "rustls", 1695 1741 "scc", 1696 1742 "serde", 1697 1743 "serde_bytes", ··· 1935 1981 1936 1982 [[package]] 1937 1983 name = "inventory" 1938 - version = "0.3.21" 1984 + version = "0.3.22" 1939 1985 source = "registry+https://github.com/rust-lang/crates.io-index" 1940 - checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e" 1986 + checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" 1941 1987 dependencies = [ 1942 1988 "rustversion", 1943 1989 ] ··· 1956 2002 1957 2003 [[package]] 1958 2004 name = "ipld-core" 1959 - version = "0.4.2" 2005 + version = "0.4.3" 1960 2006 source = "registry+https://github.com/rust-lang/crates.io-index" 1961 - checksum = "104718b1cc124d92a6d01ca9c9258a7df311405debb3408c445a36452f9bf8db" 2007 + checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" 1962 2008 dependencies = [ 1963 2009 "cid", 1964 2010 "serde", ··· 2012 2058 [[package]] 2013 2059 name = "jacquard" 2014 2060 version = "0.9.5" 2015 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2061 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2016 2062 dependencies = [ 2017 2063 "bytes", 2018 2064 "getrandom 0.2.17", ··· 2028 2074 "n0-future 0.1.3", 2029 2075 "regex", 2030 2076 "regex-lite", 2031 - "reqwest", 2077 + "reqwest 0.12.28", 2032 2078 "serde", 2033 2079 "serde_html_form", 2034 2080 "serde_json", ··· 2043 2089 [[package]] 2044 2090 name = "jacquard-api" 2045 2091 version = "0.9.5" 2046 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2092 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2047 2093 dependencies = [ 2048 2094 "bon", 2049 2095 "bytes", ··· 2062 2108 [[package]] 2063 2109 name = "jacquard-axum" 2064 2110 version = "0.9.6" 2065 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2111 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2066 2112 dependencies = [ 2067 2113 "axum", 2068 2114 "bytes", ··· 2084 2130 [[package]] 2085 2131 name = "jacquard-common" 2086 2132 version = "0.9.5" 2087 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2133 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2088 2134 dependencies = [ 2089 2135 "base64 0.22.1", 2090 2136 "bon", ··· 2114 2160 "regex", 2115 2161 "regex-automata", 2116 2162 "regex-lite", 2117 - "reqwest", 2163 + "reqwest 0.12.28", 2118 2164 "serde", 2119 2165 "serde_bytes", 2120 2166 "serde_html_form", ··· 2134 2180 [[package]] 2135 2181 name = "jacquard-derive" 2136 2182 version = "0.9.5" 2137 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2183 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2138 2184 dependencies = [ 2139 2185 "heck 0.5.0", 2140 2186 "jacquard-lexicon", ··· 2146 2192 [[package]] 2147 2193 name = "jacquard-identity" 2148 2194 version = "0.9.5" 2149 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2195 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2150 2196 dependencies = [ 2151 2197 "bon", 2152 2198 "bytes", ··· 2159 2205 "mini-moka-wasm", 2160 2206 "n0-future 0.1.3", 2161 2207 "percent-encoding", 2162 - "reqwest", 2208 + "reqwest 0.12.28", 2163 2209 "serde", 2164 2210 "serde_html_form", 2165 2211 "serde_json", ··· 2173 2219 [[package]] 2174 2220 name = "jacquard-lexicon" 2175 2221 version = "0.9.5" 2176 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2222 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2177 2223 dependencies = [ 2178 2224 "cid", 2179 2225 "dashmap", ··· 2200 2246 [[package]] 2201 2247 name = "jacquard-oauth" 2202 2248 version = "0.9.6" 2203 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2249 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2204 2250 dependencies = [ 2205 2251 "base64 0.22.1", 2206 2252 "bytes", ··· 2232 2278 [[package]] 2233 2279 name = "jacquard-repo" 2234 2280 version = "0.9.6" 2235 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2281 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2236 2282 dependencies = [ 2237 2283 "bytes", 2238 2284 "cid", ··· 2325 2371 2326 2372 [[package]] 2327 2373 name = "js-sys" 2328 - version = "0.3.85" 2374 + version = "0.3.91" 2329 2375 source = "registry+https://github.com/rust-lang/crates.io-index" 2330 - checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" 2376 + checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" 2331 2377 dependencies = [ 2332 2378 "once_cell", 2333 2379 "wasm-bindgen", ··· 2386 2432 2387 2433 [[package]] 2388 2434 name = "libredox" 2389 - version = "0.1.12" 2435 + version = "0.1.14" 2390 2436 source = "registry+https://github.com/rust-lang/crates.io-index" 2391 - checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" 2437 + checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" 2392 2438 dependencies = [ 2393 2439 "bitflags", 2394 2440 "libc", 2395 - "redox_syscall 0.7.0", 2441 + "plain", 2442 + "redox_syscall 0.7.3", 2396 2443 ] 2397 2444 2398 2445 [[package]] ··· 2458 2505 2459 2506 [[package]] 2460 2507 name = "lsm-tree" 2461 - version = "3.0.1" 2508 + version = "3.0.3" 2462 2509 source = "registry+https://github.com/rust-lang/crates.io-index" 2463 - checksum = "b875f1dfe14f557f805b167fb9b0fc54c5560c7a4bd6ae02535b2846f276a8cb" 2510 + checksum = "341e13fa389d2e24023c2c2c110ae556b29fa59a2e734872de295863e11a6741" 2464 2511 dependencies = [ 2465 2512 "byteorder-lite", 2466 2513 "byteview", ··· 2560 2607 2561 2608 [[package]] 2562 2609 name = "memchr" 2563 - version = "2.7.6" 2610 + version = "2.8.0" 2564 2611 source = "registry+https://github.com/rust-lang/crates.io-index" 2565 - checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" 2612 + checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" 2566 2613 2567 2614 [[package]] 2568 2615 name = "miette" ··· 2622 2669 [[package]] 2623 2670 name = "mini-moka-wasm" 2624 2671 version = "0.10.99" 2625 - source = "git+https://tangled.org/nonbinary.computer/jacquard#bfb72e29f20b0683e939db0140fa44cabde162d2" 2672 + source = "git+https://tangled.org/nonbinary.computer/jacquard#bc1391738bd8aa8f1cfe49aca8d9dae469db9344" 2626 2673 dependencies = [ 2627 2674 "crossbeam-channel", 2628 2675 "crossbeam-utils", ··· 2837 2884 2838 2885 [[package]] 2839 2886 name = "objc2" 2840 - version = "0.6.3" 2887 + version = "0.6.4" 2841 2888 source = "registry+https://github.com/rust-lang/crates.io-index" 2842 - checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" 2889 + checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" 2843 2890 dependencies = [ 2844 2891 "objc2-encode", 2845 2892 ] ··· 2918 2965 2919 2966 [[package]] 2920 2967 name = "owo-colors" 2921 - version = "4.2.3" 2968 + version = "4.3.0" 2922 2969 source = "registry+https://github.com/rust-lang/crates.io-index" 2923 - checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" 2970 + checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" 2924 2971 2925 2972 [[package]] 2926 2973 name = "oxilangtag" ··· 3037 3084 3038 3085 [[package]] 3039 3086 name = "pin-project" 3040 - version = "1.1.10" 3087 + version = "1.1.11" 3041 3088 source = "registry+https://github.com/rust-lang/crates.io-index" 3042 - checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" 3089 + checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" 3043 3090 dependencies = [ 3044 3091 "pin-project-internal", 3045 3092 ] 3046 3093 3047 3094 [[package]] 3048 3095 name = "pin-project-internal" 3049 - version = "1.1.10" 3096 + version = "1.1.11" 3050 3097 source = "registry+https://github.com/rust-lang/crates.io-index" 3051 - checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" 3098 + checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" 3052 3099 dependencies = [ 3053 3100 "proc-macro2", 3054 3101 "quote", ··· 3057 3104 3058 3105 [[package]] 3059 3106 name = "pin-project-lite" 3060 - version = "0.2.16" 3107 + version = "0.2.17" 3061 3108 source = "registry+https://github.com/rust-lang/crates.io-index" 3062 - checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" 3109 + checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" 3063 3110 3064 3111 [[package]] 3065 3112 name = "pin-utils" ··· 3095 3142 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" 3096 3143 3097 3144 [[package]] 3145 + name = "plain" 3146 + version = "0.2.3" 3147 + source = "registry+https://github.com/rust-lang/crates.io-index" 3148 + checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" 3149 + 3150 + [[package]] 3098 3151 name = "portable-atomic" 3099 3152 version = "1.13.1" 3100 3153 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3226 3279 source = "registry+https://github.com/rust-lang/crates.io-index" 3227 3280 checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" 3228 3281 dependencies = [ 3282 + "aws-lc-rs", 3229 3283 "bytes", 3230 3284 "getrandom 0.3.4", 3231 3285 "lru-slab", ··· 3357 3411 3358 3412 [[package]] 3359 3413 name = "redox_syscall" 3360 - version = "0.7.0" 3414 + version = "0.7.3" 3361 3415 source = "registry+https://github.com/rust-lang/crates.io-index" 3362 - checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" 3416 + checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" 3363 3417 dependencies = [ 3364 3418 "bitflags", 3365 3419 ] ··· 3395 3449 3396 3450 [[package]] 3397 3451 name = "regex-syntax" 3398 - version = "0.8.9" 3452 + version = "0.8.10" 3399 3453 source = "registry+https://github.com/rust-lang/crates.io-index" 3400 - checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" 3454 + checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" 3401 3455 3402 3456 [[package]] 3403 3457 name = "reqwest" ··· 3438 3492 "url", 3439 3493 "wasm-bindgen", 3440 3494 "wasm-bindgen-futures", 3441 - "wasm-streams", 3495 + "wasm-streams 0.4.2", 3442 3496 "web-sys", 3443 3497 "webpki-roots", 3444 3498 ] 3445 3499 3446 3500 [[package]] 3501 + name = "reqwest" 3502 + version = "0.13.2" 3503 + source = "registry+https://github.com/rust-lang/crates.io-index" 3504 + checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" 3505 + dependencies = [ 3506 + "base64 0.22.1", 3507 + "bytes", 3508 + "futures-core", 3509 + "futures-util", 3510 + "h2", 3511 + "http", 3512 + "http-body", 3513 + "http-body-util", 3514 + "hyper", 3515 + "hyper-rustls", 3516 + "hyper-util", 3517 + "js-sys", 3518 + "log", 3519 + "percent-encoding", 3520 + "pin-project-lite", 3521 + "quinn", 3522 + "rustls", 3523 + "rustls-pki-types", 3524 + "rustls-platform-verifier", 3525 + "serde", 3526 + "serde_json", 3527 + "sync_wrapper", 3528 + "tokio", 3529 + "tokio-rustls", 3530 + "tokio-util", 3531 + "tower", 3532 + "tower-http", 3533 + "tower-service", 3534 + "url", 3535 + "wasm-bindgen", 3536 + "wasm-bindgen-futures", 3537 + "wasm-streams 0.5.0", 3538 + "web-sys", 3539 + ] 3540 + 3541 + [[package]] 3542 + name = "reqwest-middleware" 3543 + version = "0.5.1" 3544 + source = "registry+https://github.com/rust-lang/crates.io-index" 3545 + checksum = "199dda04a536b532d0cc04d7979e39b1c763ea749bf91507017069c00b96056f" 3546 + dependencies = [ 3547 + "anyhow", 3548 + "async-trait", 3549 + "http", 3550 + "reqwest 0.13.2", 3551 + "thiserror 2.0.18", 3552 + "tower-service", 3553 + ] 3554 + 3555 + [[package]] 3556 + name = "reqwest-retry" 3557 + version = "0.9.1" 3558 + source = "registry+https://github.com/rust-lang/crates.io-index" 3559 + checksum = "fe2412db2af7d2268e7a5406be0431f37d9eb67ff390f35b395716f5f06c2eaa" 3560 + dependencies = [ 3561 + "anyhow", 3562 + "async-trait", 3563 + "futures", 3564 + "getrandom 0.2.17", 3565 + "http", 3566 + "hyper", 3567 + "reqwest 0.13.2", 3568 + "reqwest-middleware", 3569 + "retry-policies", 3570 + "thiserror 2.0.18", 3571 + "tokio", 3572 + "tracing", 3573 + "wasmtimer", 3574 + ] 3575 + 3576 + [[package]] 3447 3577 name = "resolv-conf" 3448 3578 version = "0.7.6" 3449 3579 source = "registry+https://github.com/rust-lang/crates.io-index" 3450 3580 checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" 3581 + 3582 + [[package]] 3583 + name = "retry-policies" 3584 + version = "0.5.1" 3585 + source = "registry+https://github.com/rust-lang/crates.io-index" 3586 + checksum = "46a4bd6027df676bcb752d3724db0ea3c0c5fc1dd0376fec51ac7dcaf9cc69be" 3587 + dependencies = [ 3588 + "rand 0.9.2", 3589 + ] 3451 3590 3452 3591 [[package]] 3453 3592 name = "rfc6979" ··· 3570 3709 3571 3710 [[package]] 3572 3711 name = "rustls" 3573 - version = "0.23.36" 3712 + version = "0.23.37" 3574 3713 source = "registry+https://github.com/rust-lang/crates.io-index" 3575 - checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" 3714 + checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" 3576 3715 dependencies = [ 3716 + "aws-lc-rs", 3717 + "log", 3577 3718 "once_cell", 3578 3719 "ring", 3579 3720 "rustls-pki-types", ··· 3605 3746 ] 3606 3747 3607 3748 [[package]] 3749 + name = "rustls-platform-verifier" 3750 + version = "0.6.2" 3751 + source = "registry+https://github.com/rust-lang/crates.io-index" 3752 + checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" 3753 + dependencies = [ 3754 + "core-foundation 0.10.1", 3755 + "core-foundation-sys", 3756 + "jni", 3757 + "log", 3758 + "once_cell", 3759 + "rustls", 3760 + "rustls-native-certs", 3761 + "rustls-platform-verifier-android", 3762 + "rustls-webpki", 3763 + "security-framework", 3764 + "security-framework-sys", 3765 + "webpki-root-certs", 3766 + "windows-sys 0.61.2", 3767 + ] 3768 + 3769 + [[package]] 3770 + name = "rustls-platform-verifier-android" 3771 + version = "0.1.1" 3772 + source = "registry+https://github.com/rust-lang/crates.io-index" 3773 + checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" 3774 + 3775 + [[package]] 3608 3776 name = "rustls-webpki" 3609 3777 version = "0.103.9" 3610 3778 source = "registry+https://github.com/rust-lang/crates.io-index" 3611 3779 checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" 3612 3780 dependencies = [ 3781 + "aws-lc-rs", 3613 3782 "ring", 3614 3783 "rustls-pki-types", 3615 3784 "untrusted", ··· 3623 3792 3624 3793 [[package]] 3625 3794 name = "ryu" 3626 - version = "1.0.22" 3795 + version = "1.0.23" 3627 3796 source = "registry+https://github.com/rust-lang/crates.io-index" 3628 - checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" 3797 + checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" 3629 3798 3630 3799 [[package]] 3631 3800 name = "saa" 3632 - version = "5.4.9" 3801 + version = "5.5.0" 3633 3802 source = "registry+https://github.com/rust-lang/crates.io-index" 3634 - checksum = "da0ba8adb63e0deebd0744d8fc5bea394c08029159deaf680513fec1a3949144" 3803 + checksum = "16c7f49c9d5caa3bf4b3106900484b447b9253fe99670ceb81cb6cb5027855e1" 3635 3804 3636 3805 [[package]] 3637 3806 name = "safemem" ··· 3650 3819 3651 3820 [[package]] 3652 3821 name = "scc" 3653 - version = "3.5.6" 3822 + version = "3.6.6" 3654 3823 source = "registry+https://github.com/rust-lang/crates.io-index" 3655 - checksum = "e4bd9d1727de391b6982925d830baad51692fa2aa6e337733c03d95121ca2793" 3824 + checksum = "6012e652611b2fdcb557a7b4be8cee00d8be19397c70011906a68aa4dac2fe37" 3656 3825 dependencies = [ 3657 3826 "saa", 3658 3827 "sdd", ··· 3681 3850 3682 3851 [[package]] 3683 3852 name = "sdd" 3684 - version = "4.6.2" 3853 + version = "4.6.5" 3685 3854 source = "registry+https://github.com/rust-lang/crates.io-index" 3686 - checksum = "c25da4ae64b24edfcb0b0d30b96b2b0dbc64ec63aefeb6ec35bfc5ef167e5c9e" 3855 + checksum = "4becc2f27bd39aafb78a8a1b4e1d6877ed0a6b5bf096722ed538dc028367a9b5" 3687 3856 3688 3857 [[package]] 3689 3858 name = "sec1" ··· 3701 3870 3702 3871 [[package]] 3703 3872 name = "security-framework" 3704 - version = "3.5.1" 3873 + version = "3.7.0" 3705 3874 source = "registry+https://github.com/rust-lang/crates.io-index" 3706 - checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" 3875 + checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" 3707 3876 dependencies = [ 3708 3877 "bitflags", 3709 3878 "core-foundation 0.10.1", ··· 3714 3883 3715 3884 [[package]] 3716 3885 name = "security-framework-sys" 3717 - version = "2.15.0" 3886 + version = "2.17.0" 3718 3887 source = "registry+https://github.com/rust-lang/crates.io-index" 3719 - checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" 3888 + checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" 3720 3889 dependencies = [ 3721 3890 "core-foundation-sys", 3722 3891 "libc", ··· 3853 4022 3854 4023 [[package]] 3855 4024 name = "serde_with" 3856 - version = "3.16.1" 4025 + version = "3.17.0" 3857 4026 source = "registry+https://github.com/rust-lang/crates.io-index" 3858 - checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" 4027 + checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" 3859 4028 dependencies = [ 3860 4029 "base64 0.22.1", 3861 4030 "chrono", ··· 3868 4037 3869 4038 [[package]] 3870 4039 name = "serde_with_macros" 3871 - version = "3.16.1" 4040 + version = "3.17.0" 3872 4041 source = "registry+https://github.com/rust-lang/crates.io-index" 3873 - checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" 4042 + checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" 3874 4043 dependencies = [ 3875 4044 "darling 0.21.3", 3876 4045 "proc-macro2", ··· 4109 4278 4110 4279 [[package]] 4111 4280 name = "syn" 4112 - version = "2.0.114" 4281 + version = "2.0.117" 4113 4282 source = "registry+https://github.com/rust-lang/crates.io-index" 4114 - checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" 4283 + checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" 4115 4284 dependencies = [ 4116 4285 "proc-macro2", 4117 4286 "quote", ··· 4638 4807 4639 4808 [[package]] 4640 4809 name = "unicode-ident" 4641 - version = "1.0.22" 4810 + version = "1.0.24" 4642 4811 source = "registry+https://github.com/rust-lang/crates.io-index" 4643 - checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" 4812 + checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" 4644 4813 4645 4814 [[package]] 4646 4815 name = "unicode-linebreak" ··· 4784 4953 4785 4954 [[package]] 4786 4955 name = "wasm-bindgen" 4787 - version = "0.2.108" 4956 + version = "0.2.114" 4788 4957 source = "registry+https://github.com/rust-lang/crates.io-index" 4789 - checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" 4958 + checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" 4790 4959 dependencies = [ 4791 4960 "cfg-if", 4792 4961 "once_cell", ··· 4797 4966 4798 4967 [[package]] 4799 4968 name = "wasm-bindgen-futures" 4800 - version = "0.4.58" 4969 + version = "0.4.64" 4801 4970 source = "registry+https://github.com/rust-lang/crates.io-index" 4802 - checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" 4971 + checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" 4803 4972 dependencies = [ 4804 4973 "cfg-if", 4805 4974 "futures-util", ··· 4811 4980 4812 4981 [[package]] 4813 4982 name = "wasm-bindgen-macro" 4814 - version = "0.2.108" 4983 + version = "0.2.114" 4815 4984 source = "registry+https://github.com/rust-lang/crates.io-index" 4816 - checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" 4985 + checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" 4817 4986 dependencies = [ 4818 4987 "quote", 4819 4988 "wasm-bindgen-macro-support", ··· 4821 4990 4822 4991 [[package]] 4823 4992 name = "wasm-bindgen-macro-support" 4824 - version = "0.2.108" 4993 + version = "0.2.114" 4825 4994 source = "registry+https://github.com/rust-lang/crates.io-index" 4826 - checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" 4995 + checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" 4827 4996 dependencies = [ 4828 4997 "bumpalo", 4829 4998 "proc-macro2", ··· 4834 5003 4835 5004 [[package]] 4836 5005 name = "wasm-bindgen-shared" 4837 - version = "0.2.108" 5006 + version = "0.2.114" 4838 5007 source = "registry+https://github.com/rust-lang/crates.io-index" 4839 - checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" 5008 + checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" 4840 5009 dependencies = [ 4841 5010 "unicode-ident", 4842 5011 ] ··· 4877 5046 ] 4878 5047 4879 5048 [[package]] 5049 + name = "wasm-streams" 5050 + version = "0.5.0" 5051 + source = "registry+https://github.com/rust-lang/crates.io-index" 5052 + checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" 5053 + dependencies = [ 5054 + "futures-util", 5055 + "js-sys", 5056 + "wasm-bindgen", 5057 + "wasm-bindgen-futures", 5058 + "web-sys", 5059 + ] 5060 + 5061 + [[package]] 4880 5062 name = "wasmparser" 4881 5063 version = "0.244.0" 4882 5064 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4889 5071 ] 4890 5072 4891 5073 [[package]] 5074 + name = "wasmtimer" 5075 + version = "0.4.3" 5076 + source = "registry+https://github.com/rust-lang/crates.io-index" 5077 + checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" 5078 + dependencies = [ 5079 + "futures", 5080 + "js-sys", 5081 + "parking_lot", 5082 + "pin-utils", 5083 + "slab", 5084 + "wasm-bindgen", 5085 + ] 5086 + 5087 + [[package]] 4892 5088 name = "web-sys" 4893 - version = "0.3.85" 5089 + version = "0.3.91" 4894 5090 source = "registry+https://github.com/rust-lang/crates.io-index" 4895 - checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" 5091 + checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" 4896 5092 dependencies = [ 4897 5093 "js-sys", 4898 5094 "wasm-bindgen", ··· 4910 5106 4911 5107 [[package]] 4912 5108 name = "webbrowser" 4913 - version = "1.0.6" 5109 + version = "1.1.0" 4914 5110 source = "registry+https://github.com/rust-lang/crates.io-index" 4915 - checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97" 5111 + checksum = "3f00bb839c1cf1e3036066614cbdcd035ecf215206691ea646aa3c60a24f68f2" 4916 5112 dependencies = [ 4917 5113 "core-foundation 0.10.1", 4918 5114 "jni", ··· 4934 5130 "markup5ever_rcdom", 4935 5131 "serde_json", 4936 5132 "url", 5133 + ] 5134 + 5135 + [[package]] 5136 + name = "webpki-root-certs" 5137 + version = "1.0.6" 5138 + source = "registry+https://github.com/rust-lang/crates.io-index" 5139 + checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" 5140 + dependencies = [ 5141 + "rustls-pki-types", 4937 5142 ] 4938 5143 4939 5144 [[package]] ··· 5470 5675 5471 5676 [[package]] 5472 5677 name = "zerocopy" 5473 - version = "0.8.39" 5678 + version = "0.8.40" 5474 5679 source = "registry+https://github.com/rust-lang/crates.io-index" 5475 - checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" 5680 + checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" 5476 5681 dependencies = [ 5477 5682 "zerocopy-derive", 5478 5683 ] 5479 5684 5480 5685 [[package]] 5481 5686 name = "zerocopy-derive" 5482 - version = "0.8.39" 5687 + version = "0.8.40" 5483 5688 source = "registry+https://github.com/rust-lang/crates.io-index" 5484 - checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" 5689 + checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" 5485 5690 dependencies = [ 5486 5691 "proc-macro2", 5487 5692 "quote", ··· 5553 5758 5554 5759 [[package]] 5555 5760 name = "zmij" 5556 - version = "1.0.19" 5761 + version = "1.0.21" 5557 5762 source = "registry+https://github.com/rust-lang/crates.io-index" 5558 - checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" 5763 + checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" 5559 5764 5560 5765 [[package]] 5561 5766 name = "zstd"
+4
Cargo.toml
··· 23 23 smol_str = "0.3" 24 24 futures = "0.3" 25 25 reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream", "gzip", "brotli", "zstd", "http2"], default-features = false } 26 + reqwest-client = { package = "reqwest", version = "0.13.2", features = ["json", "rustls", "stream", "gzip", "brotli", "zstd", "http2"], default-features = false } 27 + reqwest-middleware = { version = "0.5.1", default-features = false, features = ["http2", "rustls"] } 28 + reqwest-retry = { version = "0.9.1" } 26 29 axum = { version = "0.8.8", features = ["ws", "macros"] } 27 30 tower-http = { version = "0.6.6", features = ["cors", "trace"] } 28 31 tokio-stream = "0.1" ··· 49 52 glob = "0.3" 50 53 ordermap = { version = "1.1.0", features = ["serde"] } 51 54 arc-swap = "1.8.2" 55 + rustls = { version = "0.23", features = ["ring"] } 52 56 53 57 [dev-dependencies] 54 58 tempfile = "3.26.0"
+1 -1
README.md
··· 42 42 | `NO_LZ4_COMPRESSION` | `false` | disable lz4 compression for storage. | 43 43 | `ENABLE_FIREHOSE` | `true` | whether to ingest relay subscriptions. | 44 44 | `ENABLE_BACKFILL` | `true` | whether to backfill from PDS instances. | 45 - | `ENABLE_CRAWLER` | `false` (if Filter), `true` (if Full) | whether to actively query the network for unknown repositories. | 45 + | `ENABLE_CRAWLER` | `false` (if Filter), `true` (if Full) | whether to actively query the network for unknown repositories. when in `Filter` mode without wildcard (`*`) signals, the crawler uses `com.atproto.repo.listRecords` to verify if a discovered repository has matching records before queuing it for backfill, this will be a lot faster usually since most repos will get filtered out faster. | 46 46 | `DB_WORKER_THREADS` | `4` (`8` if full network) | database worker threads. | 47 47 | `DB_MAX_JOURNALING_SIZE_MB` | `512` (`1024` if full network) | max database journaling size in MB. | 48 48 | `DB_PENDING_MEMTABLE_SIZE_MB` | `64` (`192` if full network) | pending memtable size in MB. |
+151 -36
src/crawler/mod.rs
··· 1 1 use crate::db::{Db, keys, ser_repo_state}; 2 2 use crate::state::AppState; 3 3 use crate::types::RepoState; 4 - use jacquard::api::com_atproto::sync::list_repos::{ListRepos, ListReposOutput}; 5 - use jacquard::prelude::*; 6 - use jacquard_common::CowStr; 4 + use jacquard::IntoStatic; 5 + use jacquard_api::com_atproto::repo::list_records::ListRecordsOutput; 6 + use jacquard_api::com_atproto::sync::list_repos::ListReposOutput; 7 + use jacquard_common::types::string::Did; 7 8 use miette::{IntoDiagnostic, Result}; 8 9 use rand::Rng; 9 10 use rand::rngs::SmallRng; 11 + use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; 12 + use reqwest_retry::Jitter; 13 + use reqwest_retry::{RetryTransientMiddleware, policies::ExponentialBackoff}; 10 14 use smol_str::SmolStr; 11 15 use std::sync::Arc; 12 16 use std::time::Duration; ··· 16 20 pub struct Crawler { 17 21 state: Arc<AppState>, 18 22 relay_host: Url, 19 - http: reqwest::Client, 23 + http: Arc<ClientWithMiddleware>, 20 24 max_pending: usize, 21 25 resume_pending: usize, 22 26 } ··· 28 32 max_pending: usize, 29 33 resume_pending: usize, 30 34 ) -> Self { 35 + let retry_policy = ExponentialBackoff::builder() 36 + .jitter(Jitter::Bounded) 37 + .build_with_max_retries(8); 38 + let reqwest_client = reqwest_client::Client::builder() 39 + .user_agent(concat!( 40 + env!("CARGO_PKG_NAME"), 41 + "/", 42 + env!("CARGO_PKG_VERSION") 43 + )) 44 + .gzip(true) 45 + .build() 46 + .expect("that reqwest will build"); 47 + 48 + let http = ClientBuilder::new(reqwest_client) 49 + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) 50 + .build(); 51 + let http = Arc::new(http); 52 + 31 53 Self { 32 54 state, 33 55 relay_host, 34 - http: reqwest::Client::new(), 56 + http, 35 57 max_pending, 36 58 resume_pending, 37 59 } ··· 40 62 pub async fn run(self) -> Result<()> { 41 63 info!("crawler started"); 42 64 65 + let mut api_url = self.relay_host.clone(); 66 + if api_url.scheme() == "wss" { 67 + api_url 68 + .set_scheme("https") 69 + .map_err(|_| miette::miette!("invalid url: {api_url}"))?; 70 + } else if api_url.scheme() == "ws" { 71 + api_url 72 + .set_scheme("http") 73 + .map_err(|_| miette::miette!("invalid url: {api_url}"))?; 74 + } 75 + 43 76 let mut rng: SmallRng = rand::make_rng(); 44 77 45 78 let db = &self.state.db; 46 79 47 80 // 1. load cursor 48 81 let cursor_key = b"crawler_cursor"; 49 - let mut cursor: Option<SmolStr> = 50 - if let Ok(Some(bytes)) = Db::get(db.cursors.clone(), cursor_key.to_vec()).await { 82 + let mut cursor: Option<SmolStr> = Db::get(db.cursors.clone(), cursor_key.to_vec()) 83 + .await? 84 + .map(|bytes| { 51 85 let s = String::from_utf8_lossy(&bytes); 52 86 info!("resuming crawler from cursor: {}", s); 53 - Some(s.into()) 54 - } else { 55 - None 56 - }; 87 + s.into() 88 + }); 57 89 let mut was_throttled = false; 58 90 59 91 loop { ··· 97 129 } 98 130 99 131 // 2. fetch listrepos 100 - let req = ListRepos::new() 101 - .limit(1000) 102 - .maybe_cursor(cursor.clone().map(|c| CowStr::from(c.to_string()))) 103 - .build(); 104 - 105 - let mut url = self.relay_host.clone(); 106 - if url.scheme() == "wss" { 107 - url.set_scheme("https") 108 - .map_err(|_| miette::miette!("invalid url: {url}"))?; 109 - } else if url.scheme() == "ws" { 110 - url.set_scheme("http") 111 - .map_err(|_| miette::miette!("invalid url: {url}"))?; 132 + let mut list_repos_url = api_url 133 + .join("/xrpc/com.atproto.sync.listRepos") 134 + .into_diagnostic()?; 135 + list_repos_url 136 + .query_pairs_mut() 137 + .append_pair("limit", "1000"); 138 + if let Some(c) = &cursor { 139 + list_repos_url 140 + .query_pairs_mut() 141 + .append_pair("cursor", c.as_str()); 112 142 } 113 - let res_result = self.http.xrpc(url).send(&req).await; 114 143 115 - let output: ListReposOutput = match res_result { 116 - Ok(res) => res.into_output().into_diagnostic()?, 144 + let res_result = self.http.get(list_repos_url.clone()).send().await; 145 + let bytes = match res_result { 146 + Ok(res) => match res.bytes().await { 147 + Ok(b) => b, 148 + Err(e) => { 149 + error!( 150 + "crawler failed to parse list repos response: {e}. retrying in 30s..." 151 + ); 152 + tokio::time::sleep(Duration::from_secs(30)).await; 153 + continue; 154 + } 155 + }, 117 156 Err(e) => { 118 - let e = e 119 - .source_err() 120 - .map(|e| e.to_string()) 121 - .unwrap_or_else(|| e.to_string()); 122 157 error!("crawler failed to list repos: {e}. retrying in 30s..."); 123 158 tokio::time::sleep(Duration::from_secs(30)).await; 124 159 continue; 125 160 } 126 161 }; 162 + let output = serde_json::from_slice::<ListReposOutput>(&bytes) 163 + .into_diagnostic()? 164 + .into_static(); 127 165 128 166 if output.repos.is_empty() { 129 167 info!("crawler finished enumeration (or empty page). sleeping for 1 hour."); ··· 135 173 136 174 let mut batch = db.inner.batch(); 137 175 let mut to_queue = Vec::new(); 176 + let filter = self.state.filter.load(); 177 + // we can check whether or not to backfill repos faster if we only have to check 178 + // certain known signals, since we can just listRecords for those signals 179 + // if we have glob signals we cant do this since we dont know what signals to check 180 + let check_signals = filter.mode == crate::filter::FilterMode::Filter 181 + && !filter.signals.is_empty() 182 + && !filter.has_glob_signals(); 138 183 139 184 // 3. process repos 185 + let mut unknown_repos = Vec::new(); 140 186 for repo in output.repos { 141 - let did_key = keys::repo_key(&repo.did); 187 + let parsed_did: Did = repo.did.parse().unwrap(); 188 + let did_key = keys::repo_key(&parsed_did); 142 189 143 190 let excl_key = crate::db::filter::exclude_key(repo.did.as_str())?; 144 191 if db.filter.contains_key(&excl_key).into_diagnostic()? { ··· 147 194 148 195 // check if known 149 196 if !Db::contains_key(db.repos.clone(), &did_key).await? { 150 - trace!("crawler found new repo: {}", repo.did); 197 + unknown_repos.push(repo); 198 + } 199 + } 200 + 201 + let mut valid_repos = Vec::new(); 202 + if check_signals && !unknown_repos.is_empty() { 203 + let mut set = tokio::task::JoinSet::new(); 204 + for repo in unknown_repos { 205 + let http = self.http.clone(); 206 + let api_url = api_url.clone(); 207 + let filter = filter.clone(); 208 + set.spawn(async move { 209 + let mut found_signal = false; 210 + for signal in filter.signals.iter() { 211 + let mut list_records_url = 212 + api_url.join("/xrpc/com.atproto.repo.listRecords").unwrap(); 213 + list_records_url 214 + .query_pairs_mut() 215 + .append_pair("repo", &repo.did) 216 + .append_pair("collection", signal) 217 + .append_pair("limit", "1"); 218 + 219 + match http.get(list_records_url).send().await { 220 + Ok(res) => { 221 + let Ok(bytes) = res.bytes().await else { 222 + error!("failed to read bytes from listRecords response for repo {}, signal {signal}", repo.did); 223 + continue; 224 + }; 225 + if let Ok(out) = serde_json::from_slice::<ListRecordsOutput>(&bytes) { 226 + if !out.records.is_empty() { 227 + found_signal = true; 228 + break; 229 + } 230 + } 231 + } 232 + Err(e) => { 233 + error!( 234 + "failed to listRecords for repo {}, signal {signal}: {e}", 235 + repo.did 236 + ); 237 + continue; 238 + } 239 + } 240 + } 151 241 152 - let state = RepoState::backfilling(rng.next_u64()); 153 - batch.insert(&db.repos, &did_key, ser_repo_state(&state)?); 154 - batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key); 155 - to_queue.push(repo.did.clone()); 242 + if !found_signal { 243 + trace!( 244 + "crawler skipped repo {}: no records match signals", 245 + repo.did 246 + ); 247 + } 248 + 249 + (repo, found_signal) 250 + }); 156 251 } 252 + 253 + while let Some(res) = set.join_next().await { 254 + let (repo, found_signal) = res.into_diagnostic()?; 255 + if found_signal { 256 + valid_repos.push(repo); 257 + } 258 + } 259 + } else { 260 + valid_repos = unknown_repos; 261 + } 262 + 263 + for repo in valid_repos { 264 + let parsed_did: Did = repo.did.parse().unwrap(); 265 + let did_key = keys::repo_key(&parsed_did); 266 + trace!("crawler found new repo: {}", repo.did); 267 + 268 + let state = RepoState::backfilling(rng.next_u64()); 269 + batch.insert(&db.repos, &did_key, ser_repo_state(&state)?); 270 + batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key); 271 + to_queue.push(repo.did.clone()); 157 272 } 158 273 159 274 // 4. update cursor
+12 -4
src/db/filter.rs
··· 1 1 use fjall::{Keyspace, OwnedWriteBatch}; 2 + use jacquard::IntoStatic; 3 + use jacquard::types::nsid::Nsid; 2 4 use miette::{IntoDiagnostic, Result}; 3 5 4 6 use crate::db::types::TrimmedDid; ··· 111 113 for guard in ks.prefix(signal_prefix) { 112 114 let (k, _) = guard.into_inner().into_diagnostic()?; 113 115 let val = std::str::from_utf8(&k[signal_prefix.len()..]).into_diagnostic()?; 114 - config.signals.push(smol_str::SmolStr::new(val)); 116 + config.signals.push(Nsid::new(val)?.into_static()); 115 117 } 116 118 117 119 let col_prefix = [COLLECTION_PREFIX, SEP]; 118 120 for guard in ks.prefix(col_prefix) { 119 121 let (k, _) = guard.into_inner().into_diagnostic()?; 120 122 let val = std::str::from_utf8(&k[col_prefix.len()..]).into_diagnostic()?; 121 - config.collections.push(smol_str::SmolStr::new(val)); 123 + config.collections.push(Nsid::new(val)?.into_static()); 122 124 } 123 125 124 126 Ok(config) ··· 196 198 197 199 let config = load(&ks)?; 198 200 assert_eq!(config.mode, FilterMode::Filter); 199 - assert_eq!(config.signals, vec!["a.b.c"]); 200 - assert_eq!(config.collections, vec!["d.e.f"]); 201 + assert_eq!( 202 + config.signals, 203 + vec![Nsid::new("a.b.c").unwrap().into_static()] 204 + ); 205 + assert_eq!( 206 + config.collections, 207 + vec![Nsid::new("d.e.f").unwrap().into_static()] 208 + ); 201 209 202 210 let excludes = read_set(&ks, EXCLUDE_PREFIX)?; 203 211 assert_eq!(excludes, vec!["did:plc:yk4q3id7id6p5z3bypvshc64"]);
+7 -8
src/filter.rs
··· 1 + use jacquard::types::nsid::Nsid; 1 2 use serde::{Deserialize, Serialize}; 2 - use smol_str::SmolStr; 3 3 use std::sync::Arc; 4 4 5 5 pub type FilterHandle = Arc<arc_swap::ArcSwap<FilterConfig>>; ··· 25 25 Full = 2, 26 26 } 27 27 28 - /// hot-path in-memory config: only the small fields needed on every event. 29 - /// dids and excludes are large sets kept in the filter keyspace only. 30 28 #[derive(Debug, Clone, Serialize)] 31 29 pub struct FilterConfig { 32 30 pub mode: FilterMode, 33 - pub signals: Vec<SmolStr>, 34 - pub collections: Vec<SmolStr>, 31 + pub signals: Vec<Nsid<'static>>, 32 + pub collections: Vec<Nsid<'static>>, 35 33 } 36 34 37 35 impl FilterConfig { ··· 43 41 } 44 42 } 45 43 46 - /// returns true if the collection matches the content filter. 47 - /// if collections is empty, all collections match. 48 44 pub fn matches_collection(&self, collection: &str) -> bool { 49 45 if self.collections.is_empty() { 50 46 return true; ··· 52 48 self.collections.iter().any(|p| nsid_matches(p, collection)) 53 49 } 54 50 55 - /// returns true if the commit touches a collection covered by a signal. 56 51 pub fn matches_signal(&self, collection: &str) -> bool { 57 52 self.signals.iter().any(|p| nsid_matches(p, collection)) 53 + } 54 + 55 + pub fn has_glob_signals(&self) -> bool { 56 + self.signals.iter().any(|s| s.ends_with(".*")) 58 57 } 59 58 } 60 59