+1

.gitignore

··· 3 .idea/ 4 .env 5 Config.toml 0

··· 3 .idea/ 4 .env 5 Config.toml 6 + data/

+293 -16

Cargo.lock

··· 401 source = "registry+https://github.com/rust-lang/crates.io-index" 402 checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" 403 dependencies = [ 404 - "bitflags", 405 "cexpr", 406 "clang-sys", 407 "itertools 0.12.1", ··· 417 "syn", 418 "which", 419 ] 0 0 0 0 0 0 420 421 [[package]] 422 name = "bitflags" ··· 656 "metrics", 657 "metrics-exporter-prometheus", 658 "parakeet-db", 0 659 "reqwest", 660 "serde", 661 "serde_bytes", ··· 663 "serde_json", 664 "tokio", 665 "tokio-postgres", 0 666 "tokio-tungstenite", 667 "tracing", 668 "tracing-subscriber", ··· 710 checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" 711 dependencies = [ 712 "libc", 0 0 0 0 0 0 0 0 0 713 ] 714 715 [[package]] ··· 847 source = "registry+https://github.com/rust-lang/crates.io-index" 848 checksum = "ccf1bedf64cdb9643204a36dd15b19a6ce8e7aa7f7b105868e9f1fad5ffa7d12" 849 dependencies = [ 850 - "bitflags", 851 "byteorder", 852 "chrono", 853 "diesel_derives", ··· 1041 ] 1042 1043 [[package]] 0 0 0 0 0 0 1044 name = "flume" 1045 version = "0.11.1" 1046 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1086 checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" 1087 dependencies = [ 1088 "percent-encoding", 0 0 0 0 0 0 0 0 0 0 1089 ] 1090 1091 [[package]] ··· 1197 ] 1198 1199 [[package]] 0 0 0 0 0 0 0 0 0 1200 name = "generic-array" 1201 version = "0.14.7" 1202 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1335 "ipconfig", 1336 "lru-cache", 1337 "once_cell", 1338 - "parking_lot", 1339 "rand", 1340 "resolv-conf", 1341 "smallvec", ··· 1459 ] 1460 1461 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 1462 name = "hyper-tls" 1463 version = "0.6.0" 1464 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1684 checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" 1685 1686 [[package]] 0 0 0 0 0 0 0 0 0 1687 name = "ipconfig" 1688 version = "0.3.2" 1689 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1999 ] 2000 2001 [[package]] 0 0 0 0 0 0 2002 name = "nanorand" 2003 version = "0.7.0" 2004 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2084 source = "registry+https://github.com/rust-lang/crates.io-index" 2085 checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" 2086 dependencies = [ 2087 - "bitflags", 2088 "cfg-if", 2089 "foreign-types", 2090 "libc", ··· 2145 "itertools 0.14.0", 2146 "lexica", 2147 "parakeet-db", 0 2148 "serde", 2149 "serde_json", 2150 "tokio", ··· 2163 ] 2164 2165 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2166 name = "parakeet-lexgen" 2167 version = "0.1.0" 2168 dependencies = [ ··· 2182 2183 [[package]] 2184 name = "parking_lot" 0 0 0 0 0 0 0 0 0 0 0 2185 version = "0.12.3" 2186 source = "registry+https://github.com/rust-lang/crates.io-index" 2187 checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" 2188 dependencies = [ 2189 "lock_api", 2190 - "parking_lot_core", 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2191 ] 2192 2193 [[package]] ··· 2198 dependencies = [ 2199 "cfg-if", 2200 "libc", 2201 - "redox_syscall", 2202 "smallvec", 2203 "windows-targets 0.52.6", 2204 ] ··· 2239 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 2240 2241 [[package]] 0 0 0 0 0 0 0 0 0 0 2242 name = "phf" 2243 version = "0.11.3" 2244 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2254 checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" 2255 dependencies = [ 2256 "siphasher", 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2257 ] 2258 2259 [[package]] ··· 2378 ] 2379 2380 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2381 name = "quanta" 2382 version = "0.12.5" 2383 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2452 source = "registry+https://github.com/rust-lang/crates.io-index" 2453 checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" 2454 dependencies = [ 2455 - "bitflags", 0 0 0 0 0 0 0 0 0 2456 ] 2457 2458 [[package]] ··· 2461 source = "registry+https://github.com/rust-lang/crates.io-index" 2462 checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" 2463 dependencies = [ 2464 - "bitflags", 2465 ] 2466 2467 [[package]] ··· 2580 source = "registry+https://github.com/rust-lang/crates.io-index" 2581 checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" 2582 dependencies = [ 2583 - "bitflags", 2584 "errno", 2585 "libc", 2586 "linux-raw-sys", ··· 2691 source = "registry+https://github.com/rust-lang/crates.io-index" 2692 checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" 2693 dependencies = [ 2694 - "bitflags", 2695 "core-foundation 0.9.4", 2696 "core-foundation-sys", 2697 "libc", ··· 2704 source = "registry+https://github.com/rust-lang/crates.io-index" 2705 checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" 2706 dependencies = [ 2707 - "bitflags", 2708 "core-foundation 0.10.0", 2709 "core-foundation-sys", 2710 "libc", ··· 2886 ] 2887 2888 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2889 name = "smallvec" 2890 version = "1.13.2" 2891 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2976 source = "registry+https://github.com/rust-lang/crates.io-index" 2977 checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" 2978 dependencies = [ 2979 - "bitflags", 2980 "core-foundation 0.9.4", 2981 "system-configuration-sys", 2982 ] ··· 3090 "bytes", 3091 "libc", 3092 "mio", 3093 - "parking_lot", 3094 "pin-project-lite", 3095 "signal-hook-registry", 3096 "socket2", ··· 3132 "futures-channel", 3133 "futures-util", 3134 "log", 3135 - "parking_lot", 3136 "percent-encoding", 3137 "phf", 3138 "pin-project-lite", ··· 3156 ] 3157 3158 [[package]] 0 0 0 0 0 0 0 0 0 0 0 3159 name = "tokio-tungstenite" 3160 version = "0.26.1" 3161 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3217 ] 3218 3219 [[package]] 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 3220 name = "tower" 3221 version = "0.5.2" 3222 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3224 dependencies = [ 3225 "futures-core", 3226 "futures-util", 0 3227 "pin-project-lite", 0 3228 "sync_wrapper", 3229 "tokio", 0 3230 "tower-layer", 3231 "tower-service", 3232 "tracing", ··· 3238 source = "registry+https://github.com/rust-lang/crates.io-index" 3239 checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" 3240 dependencies = [ 3241 - "bitflags", 3242 "bytes", 3243 "http", 3244 "http-body", ··· 3592 source = "registry+https://github.com/rust-lang/crates.io-index" 3593 checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" 3594 dependencies = [ 3595 - "redox_syscall", 3596 "wasite", 3597 "web-sys", 3598 ]

··· 401 source = "registry+https://github.com/rust-lang/crates.io-index" 402 checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" 403 dependencies = [ 404 + "bitflags 2.8.0", 405 "cexpr", 406 "clang-sys", 407 "itertools 0.12.1", ··· 417 "syn", 418 "which", 419 ] 420 + 421 + [[package]] 422 + name = "bitflags" 423 + version = "1.3.2" 424 + source = "registry+https://github.com/rust-lang/crates.io-index" 425 + checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 426 427 [[package]] 428 name = "bitflags" ··· 662 "metrics", 663 "metrics-exporter-prometheus", 664 "parakeet-db", 665 + "parakeet-index", 666 "reqwest", 667 "serde", 668 "serde_bytes", ··· 670 "serde_json", 671 "tokio", 672 "tokio-postgres", 673 + "tokio-stream", 674 "tokio-tungstenite", 675 "tracing", 676 "tracing-subscriber", ··· 718 checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" 719 dependencies = [ 720 "libc", 721 + ] 722 + 723 + [[package]] 724 + name = "crc32fast" 725 + version = "1.4.2" 726 + source = "registry+https://github.com/rust-lang/crates.io-index" 727 + checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" 728 + dependencies = [ 729 + "cfg-if", 730 ] 731 732 [[package]] ··· 864 source = "registry+https://github.com/rust-lang/crates.io-index" 865 checksum = "ccf1bedf64cdb9643204a36dd15b19a6ce8e7aa7f7b105868e9f1fad5ffa7d12" 866 dependencies = [ 867 + "bitflags 2.8.0", 868 "byteorder", 869 "chrono", 870 "diesel_derives", ··· 1058 ] 1059 1060 [[package]] 1061 + name = "fixedbitset" 1062 + version = "0.5.7" 1063 + source = "registry+https://github.com/rust-lang/crates.io-index" 1064 + checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" 1065 + 1066 + [[package]] 1067 name = "flume" 1068 version = "0.11.1" 1069 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1109 checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" 1110 dependencies = [ 1111 "percent-encoding", 1112 + ] 1113 + 1114 + [[package]] 1115 + name = "fs2" 1116 + version = "0.4.3" 1117 + source = "registry+https://github.com/rust-lang/crates.io-index" 1118 + checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" 1119 + dependencies = [ 1120 + "libc", 1121 + "winapi", 1122 ] 1123 1124 [[package]] ··· 1230 ] 1231 1232 [[package]] 1233 + name = "fxhash" 1234 + version = "0.2.1" 1235 + source = "registry+https://github.com/rust-lang/crates.io-index" 1236 + checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" 1237 + dependencies = [ 1238 + "byteorder", 1239 + ] 1240 + 1241 + [[package]] 1242 name = "generic-array" 1243 version = "0.14.7" 1244 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1377 "ipconfig", 1378 "lru-cache", 1379 "once_cell", 1380 + "parking_lot 0.12.3", 1381 "rand", 1382 "resolv-conf", 1383 "smallvec", ··· 1501 ] 1502 1503 [[package]] 1504 + name = "hyper-timeout" 1505 + version = "0.5.2" 1506 + source = "registry+https://github.com/rust-lang/crates.io-index" 1507 + checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" 1508 + dependencies = [ 1509 + "hyper", 1510 + "hyper-util", 1511 + "pin-project-lite", 1512 + "tokio", 1513 + "tower-service", 1514 + ] 1515 + 1516 + [[package]] 1517 name = "hyper-tls" 1518 version = "0.6.0" 1519 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1739 checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" 1740 1741 [[package]] 1742 + name = "instant" 1743 + version = "0.1.13" 1744 + source = "registry+https://github.com/rust-lang/crates.io-index" 1745 + checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" 1746 + dependencies = [ 1747 + "cfg-if", 1748 + ] 1749 + 1750 + [[package]] 1751 name = "ipconfig" 1752 version = "0.3.2" 1753 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2063 ] 2064 2065 [[package]] 2066 + name = "multimap" 2067 + version = "0.10.0" 2068 + source = "registry+https://github.com/rust-lang/crates.io-index" 2069 + checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" 2070 + 2071 + [[package]] 2072 name = "nanorand" 2073 version = "0.7.0" 2074 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2154 source = "registry+https://github.com/rust-lang/crates.io-index" 2155 checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" 2156 dependencies = [ 2157 + "bitflags 2.8.0", 2158 "cfg-if", 2159 "foreign-types", 2160 "libc", ··· 2215 "itertools 0.14.0", 2216 "lexica", 2217 "parakeet-db", 2218 + "parakeet-index", 2219 "serde", 2220 "serde_json", 2221 "tokio", ··· 2234 ] 2235 2236 [[package]] 2237 + name = "parakeet-index" 2238 + version = "0.1.0" 2239 + dependencies = [ 2240 + "eyre", 2241 + "figment", 2242 + "itertools 0.14.0", 2243 + "prost", 2244 + "serde", 2245 + "sled", 2246 + "tokio", 2247 + "tonic", 2248 + "tonic-build", 2249 + "tracing", 2250 + "tracing-subscriber", 2251 + ] 2252 + 2253 + [[package]] 2254 name = "parakeet-lexgen" 2255 version = "0.1.0" 2256 dependencies = [ ··· 2270 2271 [[package]] 2272 name = "parking_lot" 2273 + version = "0.11.2" 2274 + source = "registry+https://github.com/rust-lang/crates.io-index" 2275 + checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" 2276 + dependencies = [ 2277 + "instant", 2278 + "lock_api", 2279 + "parking_lot_core 0.8.6", 2280 + ] 2281 + 2282 + [[package]] 2283 + name = "parking_lot" 2284 version = "0.12.3" 2285 source = "registry+https://github.com/rust-lang/crates.io-index" 2286 checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" 2287 dependencies = [ 2288 "lock_api", 2289 + "parking_lot_core 0.9.10", 2290 + ] 2291 + 2292 + [[package]] 2293 + name = "parking_lot_core" 2294 + version = "0.8.6" 2295 + source = "registry+https://github.com/rust-lang/crates.io-index" 2296 + checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" 2297 + dependencies = [ 2298 + "cfg-if", 2299 + "instant", 2300 + "libc", 2301 + "redox_syscall 0.2.16", 2302 + "smallvec", 2303 + "winapi", 2304 ] 2305 2306 [[package]] ··· 2311 dependencies = [ 2312 "cfg-if", 2313 "libc", 2314 + "redox_syscall 0.5.8", 2315 "smallvec", 2316 "windows-targets 0.52.6", 2317 ] ··· 2352 checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 2353 2354 [[package]] 2355 + name = "petgraph" 2356 + version = "0.7.1" 2357 + source = "registry+https://github.com/rust-lang/crates.io-index" 2358 + checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" 2359 + dependencies = [ 2360 + "fixedbitset", 2361 + "indexmap", 2362 + ] 2363 + 2364 + [[package]] 2365 name = "phf" 2366 version = "0.11.3" 2367 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2377 checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" 2378 dependencies = [ 2379 "siphasher", 2380 + ] 2381 + 2382 + [[package]] 2383 + name = "pin-project" 2384 + version = "1.1.10" 2385 + source = "registry+https://github.com/rust-lang/crates.io-index" 2386 + checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" 2387 + dependencies = [ 2388 + "pin-project-internal", 2389 + ] 2390 + 2391 + [[package]] 2392 + name = "pin-project-internal" 2393 + version = "1.1.10" 2394 + source = "registry+https://github.com/rust-lang/crates.io-index" 2395 + checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" 2396 + dependencies = [ 2397 + "proc-macro2", 2398 + "quote", 2399 + "syn", 2400 ] 2401 2402 [[package]] ··· 2521 ] 2522 2523 [[package]] 2524 + name = "prost" 2525 + version = "0.13.5" 2526 + source = "registry+https://github.com/rust-lang/crates.io-index" 2527 + checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" 2528 + dependencies = [ 2529 + "bytes", 2530 + "prost-derive", 2531 + ] 2532 + 2533 + [[package]] 2534 + name = "prost-build" 2535 + version = "0.13.5" 2536 + source = "registry+https://github.com/rust-lang/crates.io-index" 2537 + checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" 2538 + dependencies = [ 2539 + "heck", 2540 + "itertools 0.14.0", 2541 + "log", 2542 + "multimap", 2543 + "once_cell", 2544 + "petgraph", 2545 + "prettyplease", 2546 + "prost", 2547 + "prost-types", 2548 + "regex", 2549 + "syn", 2550 + "tempfile", 2551 + ] 2552 + 2553 + [[package]] 2554 + name = "prost-derive" 2555 + version = "0.13.5" 2556 + source = "registry+https://github.com/rust-lang/crates.io-index" 2557 + checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" 2558 + dependencies = [ 2559 + "anyhow", 2560 + "itertools 0.14.0", 2561 + "proc-macro2", 2562 + "quote", 2563 + "syn", 2564 + ] 2565 + 2566 + [[package]] 2567 + name = "prost-types" 2568 + version = "0.13.5" 2569 + source = "registry+https://github.com/rust-lang/crates.io-index" 2570 + checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" 2571 + dependencies = [ 2572 + "prost", 2573 + ] 2574 + 2575 + [[package]] 2576 name = "quanta" 2577 version = "0.12.5" 2578 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2647 source = "registry+https://github.com/rust-lang/crates.io-index" 2648 checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" 2649 dependencies = [ 2650 + "bitflags 2.8.0", 2651 + ] 2652 + 2653 + [[package]] 2654 + name = "redox_syscall" 2655 + version = "0.2.16" 2656 + source = "registry+https://github.com/rust-lang/crates.io-index" 2657 + checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" 2658 + dependencies = [ 2659 + "bitflags 1.3.2", 2660 ] 2661 2662 [[package]] ··· 2665 source = "registry+https://github.com/rust-lang/crates.io-index" 2666 checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" 2667 dependencies = [ 2668 + "bitflags 2.8.0", 2669 ] 2670 2671 [[package]] ··· 2784 source = "registry+https://github.com/rust-lang/crates.io-index" 2785 checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" 2786 dependencies = [ 2787 + "bitflags 2.8.0", 2788 "errno", 2789 "libc", 2790 "linux-raw-sys", ··· 2895 source = "registry+https://github.com/rust-lang/crates.io-index" 2896 checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" 2897 dependencies = [ 2898 + "bitflags 2.8.0", 2899 "core-foundation 0.9.4", 2900 "core-foundation-sys", 2901 "libc", ··· 2908 source = "registry+https://github.com/rust-lang/crates.io-index" 2909 checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" 2910 dependencies = [ 2911 + "bitflags 2.8.0", 2912 "core-foundation 0.10.0", 2913 "core-foundation-sys", 2914 "libc", ··· 3090 ] 3091 3092 [[package]] 3093 + name = "sled" 3094 + version = "0.34.7" 3095 + source = "registry+https://github.com/rust-lang/crates.io-index" 3096 + checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" 3097 + dependencies = [ 3098 + "crc32fast", 3099 + "crossbeam-epoch", 3100 + "crossbeam-utils", 3101 + "fs2", 3102 + "fxhash", 3103 + "libc", 3104 + "log", 3105 + "parking_lot 0.11.2", 3106 + ] 3107 + 3108 + [[package]] 3109 name = "smallvec" 3110 version = "1.13.2" 3111 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3196 source = "registry+https://github.com/rust-lang/crates.io-index" 3197 checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" 3198 dependencies = [ 3199 + "bitflags 2.8.0", 3200 "core-foundation 0.9.4", 3201 "system-configuration-sys", 3202 ] ··· 3310 "bytes", 3311 "libc", 3312 "mio", 3313 + "parking_lot 0.12.3", 3314 "pin-project-lite", 3315 "signal-hook-registry", 3316 "socket2", ··· 3352 "futures-channel", 3353 "futures-util", 3354 "log", 3355 + "parking_lot 0.12.3", 3356 "percent-encoding", 3357 "phf", 3358 "pin-project-lite", ··· 3376 ] 3377 3378 [[package]] 3379 + name = "tokio-stream" 3380 + version = "0.1.17" 3381 + source = "registry+https://github.com/rust-lang/crates.io-index" 3382 + checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" 3383 + dependencies = [ 3384 + "futures-core", 3385 + "pin-project-lite", 3386 + "tokio", 3387 + ] 3388 + 3389 + [[package]] 3390 name = "tokio-tungstenite" 3391 version = "0.26.1" 3392 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3448 ] 3449 3450 [[package]] 3451 + name = "tonic" 3452 + version = "0.13.0" 3453 + source = "registry+https://github.com/rust-lang/crates.io-index" 3454 + checksum = "85839f0b32fd242bb3209262371d07feda6d780d16ee9d2bc88581b89da1549b" 3455 + dependencies = [ 3456 + "async-trait", 3457 + "axum", 3458 + "base64", 3459 + "bytes", 3460 + "h2", 3461 + "http", 3462 + "http-body", 3463 + "http-body-util", 3464 + "hyper", 3465 + "hyper-timeout", 3466 + "hyper-util", 3467 + "percent-encoding", 3468 + "pin-project", 3469 + "prost", 3470 + "socket2", 3471 + "tokio", 3472 + "tokio-stream", 3473 + "tower", 3474 + "tower-layer", 3475 + "tower-service", 3476 + "tracing", 3477 + ] 3478 + 3479 + [[package]] 3480 + name = "tonic-build" 3481 + version = "0.13.0" 3482 + source = "registry+https://github.com/rust-lang/crates.io-index" 3483 + checksum = "d85f0383fadd15609306383a90e85eaed44169f931a5d2be1b42c76ceff1825e" 3484 + dependencies = [ 3485 + "prettyplease", 3486 + "proc-macro2", 3487 + "prost-build", 3488 + "prost-types", 3489 + "quote", 3490 + "syn", 3491 + ] 3492 + 3493 + [[package]] 3494 name = "tower" 3495 version = "0.5.2" 3496 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3498 dependencies = [ 3499 "futures-core", 3500 "futures-util", 3501 + "indexmap", 3502 "pin-project-lite", 3503 + "slab", 3504 "sync_wrapper", 3505 "tokio", 3506 + "tokio-util", 3507 "tower-layer", 3508 "tower-service", 3509 "tracing", ··· 3515 source = "registry+https://github.com/rust-lang/crates.io-index" 3516 checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" 3517 dependencies = [ 3518 + "bitflags 2.8.0", 3519 "bytes", 3520 "http", 3521 "http-body", ··· 3869 source = "registry+https://github.com/rust-lang/crates.io-index" 3870 checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" 3871 dependencies = [ 3872 + "redox_syscall 0.5.8", 3873 "wasite", 3874 "web-sys", 3875 ]

+1

Cargo.toml

··· 8 "lexica", 9 "parakeet", 10 "parakeet-db", 0 11 "parakeet-lexgen" 12 ]

··· 8 "lexica", 9 "parakeet", 10 "parakeet-db", 11 + "parakeet-index", 12 "parakeet-lexgen" 13 ]

+2

consumer/Cargo.toml

··· 20 metrics = "0.24.1" 21 metrics-exporter-prometheus = "0.16.2" 22 parakeet-db = { path = "../parakeet-db" } 0 23 reqwest = { version = "0.12.12", features = ["native-tls"] } 24 serde = { version = "1.0.217", features = ["derive"] } 25 serde_bytes = "0.11" ··· 27 serde_json = "1.0.134" 28 tokio = { version = "1.42.0", features = ["full"] } 29 tokio-postgres = { version = "0.7.12", features = ["with-chrono-0_4"] } 0 30 tokio-tungstenite = { version = "0.26.1", features = ["native-tls"] } 31 tracing = "0.1.40" 32 tracing-subscriber = "0.3.18"

··· 20 metrics = "0.24.1" 21 metrics-exporter-prometheus = "0.16.2" 22 parakeet-db = { path = "../parakeet-db" } 23 + parakeet-index = { path = "../parakeet-index" } 24 reqwest = { version = "0.12.12", features = ["native-tls"] } 25 serde = { version = "1.0.217", features = ["derive"] } 26 serde_bytes = "0.11" ··· 28 serde_json = "1.0.134" 29 tokio = { version = "1.42.0", features = ["full"] } 30 tokio-postgres = { version = "0.7.12", features = ["with-chrono-0_4"] } 31 + tokio-stream = "0.1.17" 32 tokio-tungstenite = { version = "0.26.1", features = ["native-tls"] } 33 tracing = "0.1.40" 34 tracing-subscriber = "0.3.18"

+1

consumer/run.sh

··· 0

··· 1 + cargo run

+68 -51

consumer/src/backfill/mod.rs

··· 1 use crate::config::HistoryMode; 2 - use crate::indexer::types::{BackfillItem, BackfillItemInner, CollectionType, RecordTypes}; 3 use crate::indexer::{self, db as indexer_db}; 4 use did_resolver::Resolver; 5 use diesel_async::pooled_connection::deadpool::Pool; ··· 9 use metrics::counter; 10 use parakeet_db::types::{ActorStatus, ActorSyncState}; 11 use reqwest::{Client, StatusCode}; 0 12 use std::str::FromStr; 13 use std::sync::Arc; 14 use tracing::{instrument, Instrument}; ··· 18 mod types; 19 20 const PDS_SERVICE_ID: &str = "#atproto_pds"; 0 0 0 21 22 #[derive(Clone)] 23 pub struct BackfillManagerInner { 24 pool: Pool<AsyncPgConnection>, 25 resolver: Arc<Resolver>, 26 client: Client, 0 27 } 28 29 pub struct BackfillManager { ··· 37 pool: Pool<AsyncPgConnection>, 38 history_mode: HistoryMode, 39 resolver: Arc<Resolver>, 0 40 ) -> eyre::Result<(Self, Sender<String>)> { 41 let client = Client::new(); 42 ··· 47 pool, 48 resolver, 49 client, 0 50 }, 51 rx, 52 do_backfill: history_mode == HistoryMode::BackfillHistory, ··· 61 if self.do_backfill { 62 for idx in 0..threads { 63 let rx = self.rx.clone(); 64 - let inner = self.inner.clone(); 65 66 js.spawn( 67 async move { 68 while let Ok(did) = rx.recv_async().await { 69 tracing::trace!("backfilling {did}"); 70 - if let Err(e) = backfill_actor(&inner, &did).await { 71 tracing::error!(did, "backfill failed: {e}"); 72 counter!("backfill_failure").increment(1); 73 } else { ··· 90 } 91 92 #[instrument(skip(inner))] 93 - async fn backfill_actor(inner: &BackfillManagerInner, did: &str) -> eyre::Result<()> { 94 let mut conn = inner.pool.get().await?; 95 96 let (status, sync_state) = db::get_actor_status(&mut conn, did).await?; ··· 161 162 tracing::trace!("repo pulled - inserting"); 163 164 - conn.transaction::<(), diesel::result::Error, _>(|t| { 165 - Box::pin(async move { 166 - db::defer(t).await?; 0 167 168 - indexer_db::update_repo_version(t, did, &rev, cid).await?; 169 170 - let mut follow_stats = vec![did.to_string()]; 171 172 - for (path, (cid, record)) in records { 173 - let Some((collection, rkey)) = path.split_once("/") else { 174 - tracing::warn!("record contained invalid path {}", path); 175 - return Err(diesel::result::Error::RollbackTransaction); 176 - }; 177 178 - counter!("backfilled_commits", "collection" => collection.to_string()).increment(1); 0 0 0 0 179 180 - let full_path = format!("at://{did}/{path}"); 0 181 182 - match record { 183 - RecordTypes::AppBskyGraphFollow(record) => { 184 - follow_stats.push(record.subject.clone()); 185 - indexer_db::insert_follow(t, did, &full_path, record).await?; 186 - } 187 - _ => indexer::index_op(t, did, cid, record, &full_path, rkey).await?, 188 } 189 - } 190 191 - db::update_repo_sync_state(t, did, ActorSyncState::Synced).await?; 192 193 - handle_backfill_rows(t, &mut follow_stats, did, &rev).await?; 0 194 195 - // on second thought, should this be done after the transaction? 196 - // if we're loading a chunky repo, we might be a few seconds+ out of date? 197 - indexer_db::update_follow_stats(t, &follow_stats).await?; 0 198 199 - tracing::trace!("insertion finished"); 200 - Ok(()) 0 0 0 0 0 201 }) 202 - }) 203 - .await?; 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 204 205 Ok(()) 206 } 207 208 async fn handle_backfill_rows( 209 conn: &mut AsyncPgConnection, 210 - follow_stats: &mut Vec<String>, 211 repo: &str, 212 rev: &str, 213 ) -> diesel::QueryResult<()> { ··· 233 continue; 234 }; 235 236 - match record { 237 - RecordTypes::AppBskyGraphFollow(follow) => { 238 - follow_stats.push(follow.subject.clone()); 239 - indexer_db::insert_follow(conn, repo, &item.at_uri, follow).await?; 240 - } 241 - _ => indexer::index_op(conn, repo, cid, record, &item.at_uri, rkey).await?, 242 - } 243 } 244 - BackfillItemInner::Delete => match item.collection { 245 - CollectionType::BskyFollow => { 246 - if let Some(subject) = indexer_db::delete_follow(conn, &item.at_uri).await? 247 - { 248 - follow_stats.push(subject); 249 - } 250 - } 251 - _ => { 252 - indexer::index_op_delete(conn, repo, item.collection, &item.at_uri).await? 253 - } 254 - }, 255 } 256 } 257 }

··· 1 use crate::config::HistoryMode; 2 + use crate::indexer::types::{AggregateDeltaStore, BackfillItem, BackfillItemInner}; 3 use crate::indexer::{self, db as indexer_db}; 4 use did_resolver::Resolver; 5 use diesel_async::pooled_connection::deadpool::Pool; ··· 9 use metrics::counter; 10 use parakeet_db::types::{ActorStatus, ActorSyncState}; 11 use reqwest::{Client, StatusCode}; 12 + use std::collections::HashMap; 13 use std::str::FromStr; 14 use std::sync::Arc; 15 use tracing::{instrument, Instrument}; ··· 19 mod types; 20 21 const PDS_SERVICE_ID: &str = "#atproto_pds"; 22 + // There's a 4MiB limit on parakeet-index, so break delta batches up if there's loads. 23 + // this should be plenty low enough to not trigger the size limit. (59k did slightly) 24 + const DELTA_BATCH_SIZE: usize = 32 * 1024; 25 26 #[derive(Clone)] 27 pub struct BackfillManagerInner { 28 pool: Pool<AsyncPgConnection>, 29 resolver: Arc<Resolver>, 30 client: Client, 31 + index_client: parakeet_index::Client, 32 } 33 34 pub struct BackfillManager { ··· 42 pool: Pool<AsyncPgConnection>, 43 history_mode: HistoryMode, 44 resolver: Arc<Resolver>, 45 + index_client: parakeet_index::Client, 46 ) -> eyre::Result<(Self, Sender<String>)> { 47 let client = Client::new(); 48 ··· 53 pool, 54 resolver, 55 client, 56 + index_client, 57 }, 58 rx, 59 do_backfill: history_mode == HistoryMode::BackfillHistory, ··· 68 if self.do_backfill { 69 for idx in 0..threads { 70 let rx = self.rx.clone(); 71 + let mut inner = self.inner.clone(); 72 73 js.spawn( 74 async move { 75 while let Ok(did) = rx.recv_async().await { 76 tracing::trace!("backfilling {did}"); 77 + if let Err(e) = backfill_actor(&mut inner, &did).await { 78 tracing::error!(did, "backfill failed: {e}"); 79 counter!("backfill_failure").increment(1); 80 } else { ··· 97 } 98 99 #[instrument(skip(inner))] 100 + async fn backfill_actor(inner: &mut BackfillManagerInner, did: &str) -> eyre::Result<()> { 101 let mut conn = inner.pool.get().await?; 102 103 let (status, sync_state) = db::get_actor_status(&mut conn, did).await?; ··· 168 169 tracing::trace!("repo pulled - inserting"); 170 171 + let delta_store = conn 172 + .transaction::<_, diesel::result::Error, _>(|t| { 173 + Box::pin(async move { 174 + let mut delta_store = HashMap::new(); 175 176 + db::defer(t).await?; 177 178 + indexer_db::update_repo_version(t, did, &rev, cid).await?; 179 180 + // let mut follow_stats = vec![did.to_string()]; 0 0 0 0 181 182 + for (path, (cid, record)) in records { 183 + let Some((collection, rkey)) = path.split_once("/") else { 184 + tracing::warn!("record contained invalid path {}", path); 185 + return Err(diesel::result::Error::RollbackTransaction); 186 + }; 187 188 + counter!("backfilled_commits", "collection" => collection.to_string()) 189 + .increment(1); 190 191 + let full_path = format!("at://{did}/{path}"); 192 + 193 + indexer::index_op(t, &mut delta_store, did, cid, record, &full_path, rkey) 194 + .await? 0 0 195 } 0 196 197 + db::update_repo_sync_state(t, did, ActorSyncState::Synced).await?; 198 199 + handle_backfill_rows(t, &mut delta_store, did, &rev).await?; 200 + tracing::trace!("insertion finished"); 201 202 + Ok(delta_store) 203 + }) 204 + }) 205 + .await?; 206 207 + // submit the deltas 208 + let delta_store = delta_store 209 + .into_iter() 210 + .map(|((uri, typ), delta)| parakeet_index::AggregateDeltaReq { 211 + typ, 212 + uri: uri.to_string(), 213 + delta, 214 }) 215 + .collect::<Vec<_>>(); 216 + 217 + let mut read = 0; 218 + 219 + while read < delta_store.len() { 220 + let rem = delta_store.len() - read; 221 + let take = DELTA_BATCH_SIZE.min(rem); 222 + 223 + tracing::debug!("reading & submitting {take} deltas"); 224 + 225 + let deltas = delta_store[read..read + take].to_vec(); 226 + inner 227 + .index_client 228 + .submit_aggregate_delta_batch(parakeet_index::AggregateDeltaBatchReq { deltas }) 229 + .await?; 230 + 231 + read += take; 232 + tracing::debug!("read {read} of {} deltas", delta_store.len()); 233 + } 234 235 Ok(()) 236 } 237 238 async fn handle_backfill_rows( 239 conn: &mut AsyncPgConnection, 240 + deltas: &mut impl AggregateDeltaStore, 241 repo: &str, 242 rev: &str, 243 ) -> diesel::QueryResult<()> { ··· 263 continue; 264 }; 265 266 + indexer::index_op(conn, deltas, repo, cid, record, &item.at_uri, rkey).await? 267 + } 268 + BackfillItemInner::Delete => { 269 + indexer::index_op_delete(conn, deltas, repo, item.collection, &item.at_uri) 270 + .await? 0 0 271 } 0 0 0 0 0 0 0 0 0 0 0 272 } 273 } 274 }

+1

consumer/src/config.rs

··· 13 14 #[derive(Debug, Deserialize)] 15 pub struct Config { 0 16 pub relay_source: String, 17 pub database_url: String, 18 pub plc_directory: Option<String>,

··· 13 14 #[derive(Debug, Deserialize)] 15 pub struct Config { 16 + pub index_uri: String, 17 pub relay_source: String, 18 pub database_url: String, 19 pub plc_directory: Option<String>,

+33 -4

consumer/src/indexer/db.rs

··· 522 .await 523 } 524 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 525 pub async fn upsert_postgate( 526 conn: &mut AsyncPgConnection, 527 at_uri: &str, ··· 642 .await 643 } 644 645 - pub async fn delete_like(conn: &mut AsyncPgConnection, at_uri: &str) -> QueryResult<usize> { 0 0 0 646 diesel::delete(schema::likes::table) 647 .filter(schema::likes::at_uri.eq(at_uri)) 648 - .execute(conn) 0 649 .await 0 650 } 651 652 pub async fn insert_repost( ··· 669 .await 670 } 671 672 - pub async fn delete_repost(conn: &mut AsyncPgConnection, at_uri: &str) -> QueryResult<usize> { 0 0 0 673 diesel::delete(schema::reposts::table) 674 .filter(schema::reposts::at_uri.eq(at_uri)) 675 - .execute(conn) 0 676 .await 0 677 } 678 679 pub async fn upsert_chat_decl(

··· 522 .await 523 } 524 525 + pub async fn get_post_info_for_delete( 526 + conn: &mut AsyncPgConnection, 527 + at_uri: &str, 528 + ) -> QueryResult<Option<(Option<String>, Option<String>)>> { 529 + schema::posts::table 530 + .left_join( 531 + schema::post_embed_record::table 532 + .on(schema::posts::at_uri.eq(schema::post_embed_record::post_uri)), 533 + ) 534 + .select(( 535 + schema::posts::parent_uri, 536 + schema::post_embed_record::uri.nullable(), 537 + )) 538 + .filter(schema::posts::at_uri.eq(at_uri)) 539 + .get_result(conn) 540 + .await 541 + .optional() 542 + } 543 + 544 pub async fn upsert_postgate( 545 conn: &mut AsyncPgConnection, 546 at_uri: &str, ··· 661 .await 662 } 663 664 + pub async fn delete_like( 665 + conn: &mut AsyncPgConnection, 666 + at_uri: &str, 667 + ) -> QueryResult<Option<String>> { 668 diesel::delete(schema::likes::table) 669 .filter(schema::likes::at_uri.eq(at_uri)) 670 + .returning(schema::likes::subject) 671 + .get_result(conn) 672 .await 673 + .optional() 674 } 675 676 pub async fn insert_repost( ··· 693 .await 694 } 695 696 + pub async fn delete_repost( 697 + conn: &mut AsyncPgConnection, 698 + at_uri: &str, 699 + ) -> QueryResult<Option<String>> { 700 diesel::delete(schema::reposts::table) 701 .filter(schema::reposts::at_uri.eq(at_uri)) 702 + .returning(schema::reposts::post) 703 + .get_result(conn) 704 .await 705 + .optional() 706 } 707 708 pub async fn upsert_chat_decl(

+114 -21

consumer/src/indexer/mod.rs

··· 1 use crate::config::HistoryMode; 2 use crate::firehose::{AtpAccountEvent, AtpCommitEvent, AtpIdentityEvent, CommitOp, FirehoseEvent}; 3 - use crate::indexer::types::{BackfillItem, BackfillItemInner, CollectionType, RecordTypes}; 0 0 4 use did_resolver::Resolver; 5 use diesel_async::pooled_connection::deadpool::Pool; 6 use diesel_async::{AsyncConnection, AsyncPgConnection}; ··· 9 use ipld_core::cid::Cid; 10 use metrics::counter; 11 use parakeet_db::types::{ActorStatus, ActorSyncState}; 0 12 use std::collections::HashMap; 13 use std::hash::BuildHasher; 14 use std::sync::Arc; ··· 22 #[derive(Clone)] 23 struct RelayIndexerState { 24 backfill_tx: flume::Sender<String>, 0 25 resolver: Arc<Resolver>, 26 do_backfill: bool, 27 } ··· 37 pub async fn new( 38 pool: Pool<AsyncPgConnection>, 39 backfill_tx: flume::Sender<String>, 0 40 resolver: Arc<Resolver>, 41 history_mode: HistoryMode, 42 ) -> eyre::Result<(Self, Sender<FirehoseEvent>)> { ··· 48 backfill_tx, 49 resolver, 50 do_backfill: history_mode == HistoryMode::BackfillHistory, 0 51 }, 52 rx, 53 hasher: RandomState::default(), ··· 60 let (submit, _handles) = (0..threads) 61 .map(|idx| { 62 let pool = self.pool.clone(); 63 - let state = self.state.clone(); 64 let (tx, mut rx) = channel(16); 65 66 let handle = tokio::spawn(async move { ··· 76 index_account(&state, &mut conn, account).await 77 } 78 FirehoseEvent::Commit(commit) => { 79 - index_commit(&state, &mut conn, commit).await 80 } 81 FirehoseEvent::Label(_) => unreachable!(), 82 }; ··· 186 187 #[instrument(skip_all, fields(seq = commit.seq, repo = commit.repo, rev = commit.rev))] 188 async fn index_commit( 189 - state: &RelayIndexerState, 190 conn: &mut AsyncPgConnection, 191 commit: AtpCommitEvent, 192 ) -> eyre::Result<()> { ··· 262 } 263 264 for op in &commit.ops { 265 - process_op(t, &commit.repo, op, &blocks).await?; 266 } 267 } else { 268 let items = commit ··· 333 #[inline(always)] 334 async fn process_op( 335 conn: &mut AsyncPgConnection, 0 336 repo: &str, 337 op: &CommitOp, 338 blocks: &HashMap<Cid, Vec<u8>>, ··· 361 return Ok(()); 362 }; 363 364 - index_op(conn, repo, cid, decoded, &full_path, rkey).await?; 365 } else if op.action == "delete" { 366 - index_op_delete(conn, repo, collection, &full_path).await?; 367 } else { 368 tracing::warn!("op contained invalid action {}", op.action); 369 } ··· 388 389 pub async fn index_op( 390 conn: &mut AsyncPgConnection, 0 391 repo: &str, 392 cid: Cid, 393 record: RecordTypes, ··· 407 } 408 RecordTypes::AppBskyFeedGenerator(record) => { 409 let labels = record.labels.clone(); 410 - db::upsert_feedgen(conn, repo, cid, at_uri, record).await?; 411 412 if let Some(labels) = labels { 413 db::maintain_self_labels(conn, repo, Some(cid), at_uri, labels).await?; 414 } 0 0 0 0 415 } 416 RecordTypes::AppBskyFeedLike(record) => { 417 - db::insert_like(conn, repo, at_uri, record).await?; 0 0 0 0 0 418 } 419 RecordTypes::AppBskyFeedPost(record) => { 420 if let Some(records::AppBskyEmbed::RecordWithMedia(embed)) = &record.embed { ··· 423 } 424 } 425 0 0 0 0 0 0 0 426 let labels = record.labels.clone(); 427 db::insert_post(conn, repo, cid, at_uri, record).await?; 428 if let Some(labels) = labels { 429 db::maintain_self_labels(conn, repo, Some(cid), at_uri, labels).await?; 430 } 0 0 0 0 0 0 0 0 431 } 432 RecordTypes::AppBskyFeedPostgate(record) => { 433 let split_aturi = record.post.rsplitn(4, '/').collect::<Vec<_>>(); ··· 452 .await?; 453 } 454 RecordTypes::AppBskyFeedRepost(record) => { 0 0 0 455 db::insert_repost(conn, repo, at_uri, record).await?; 456 } 457 RecordTypes::AppBskyFeedThreadgate(record) => { ··· 467 db::insert_block(conn, repo, at_uri, record).await?; 468 } 469 RecordTypes::AppBskyGraphFollow(record) => { 470 - db::insert_follow(conn, repo, at_uri, record).await?; 0 0 0 0 0 0 0 0 471 } 472 RecordTypes::AppBskyGraphList(record) => { 473 let labels = record.labels.clone(); 474 - db::upsert_list(conn, repo, at_uri, cid, record).await?; 475 476 if let Some(labels) = labels { 477 db::maintain_self_labels(conn, repo, Some(cid), at_uri, labels).await?; 478 } 479 480 - // todo: when we have profile stats, update them. 0 0 481 } 482 RecordTypes::AppBskyGraphListBlock(record) => { 483 db::insert_list_block(conn, repo, at_uri, record).await?; ··· 493 db::insert_list_item(conn, at_uri, record).await?; 494 } 495 RecordTypes::AppBskyGraphStarterPack(record) => { 496 - db::upsert_starterpack(conn, repo, cid, at_uri, record).await?; 0 0 0 497 } 498 RecordTypes::AppBskyGraphVerification(record) => { 499 db::upsert_verification(conn, repo, cid, at_uri, record).await?; ··· 520 521 pub async fn index_op_delete( 522 conn: &mut AsyncPgConnection, 0 523 repo: &str, 524 collection: CollectionType, 525 at_uri: &str, ··· 527 match collection { 528 CollectionType::BskyProfile => db::delete_profile(conn, repo).await?, 529 CollectionType::BskyBlock => db::delete_block(conn, at_uri).await?, 530 - CollectionType::BskyFeedGen => db::delete_feedgen(conn, at_uri).await?, 531 - CollectionType::BskyFeedLike => db::delete_like(conn, at_uri).await?, 532 - CollectionType::BskyFeedPost => db::delete_post(conn, at_uri).await?, 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 533 CollectionType::BskyFeedPostgate => db::delete_postgate(conn, at_uri).await?, 534 - CollectionType::BskyFeedRepost => db::delete_repost(conn, at_uri).await?, 0 0 0 0 0 535 CollectionType::BskyFeedThreadgate => db::delete_threadgate(conn, at_uri).await?, 536 CollectionType::BskyFollow => { 537 - db::delete_follow(conn, at_uri).await?; 0 0 0 538 0 539 } 540 CollectionType::BskyList => { 541 - db::delete_list(conn, at_uri).await? 542 - // todo: when we have profile stats, update them. 0 0 0 543 } 544 CollectionType::BskyListBlock => db::delete_list_block(conn, at_uri).await?, 545 CollectionType::BskyListItem => db::delete_list_item(conn, at_uri).await?, 546 - CollectionType::BskyStarterPack => db::delete_starterpack(conn, at_uri).await?, 0 0 0 0 0 0 547 CollectionType::BskyVerification => db::delete_verification(conn, at_uri).await?, 548 CollectionType::BskyLabelerService => db::delete_label_service(conn, at_uri).await?, 549 CollectionType::ChatActorDecl => db::delete_chat_decl(conn, at_uri).await?,

··· 1 use crate::config::HistoryMode; 2 use crate::firehose::{AtpAccountEvent, AtpCommitEvent, AtpIdentityEvent, CommitOp, FirehoseEvent}; 3 + use crate::indexer::types::{ 4 + AggregateDeltaStore, BackfillItem, BackfillItemInner, CollectionType, RecordTypes, 5 + }; 6 use did_resolver::Resolver; 7 use diesel_async::pooled_connection::deadpool::Pool; 8 use diesel_async::{AsyncConnection, AsyncPgConnection}; ··· 11 use ipld_core::cid::Cid; 12 use metrics::counter; 13 use parakeet_db::types::{ActorStatus, ActorSyncState}; 14 + use parakeet_index::AggregateType; 15 use std::collections::HashMap; 16 use std::hash::BuildHasher; 17 use std::sync::Arc; ··· 25 #[derive(Clone)] 26 struct RelayIndexerState { 27 backfill_tx: flume::Sender<String>, 28 + idxc_tx: Sender<parakeet_index::AggregateDeltaReq>, 29 resolver: Arc<Resolver>, 30 do_backfill: bool, 31 } ··· 41 pub async fn new( 42 pool: Pool<AsyncPgConnection>, 43 backfill_tx: flume::Sender<String>, 44 + idxc_tx: Sender<parakeet_index::AggregateDeltaReq>, 45 resolver: Arc<Resolver>, 46 history_mode: HistoryMode, 47 ) -> eyre::Result<(Self, Sender<FirehoseEvent>)> { ··· 53 backfill_tx, 54 resolver, 55 do_backfill: history_mode == HistoryMode::BackfillHistory, 56 + idxc_tx, 57 }, 58 rx, 59 hasher: RandomState::default(), ··· 66 let (submit, _handles) = (0..threads) 67 .map(|idx| { 68 let pool = self.pool.clone(); 69 + let mut state = self.state.clone(); 70 let (tx, mut rx) = channel(16); 71 72 let handle = tokio::spawn(async move { ··· 82 index_account(&state, &mut conn, account).await 83 } 84 FirehoseEvent::Commit(commit) => { 85 + index_commit(&mut state, &mut conn, commit).await 86 } 87 FirehoseEvent::Label(_) => unreachable!(), 88 }; ··· 192 193 #[instrument(skip_all, fields(seq = commit.seq, repo = commit.repo, rev = commit.rev))] 194 async fn index_commit( 195 + state: &mut RelayIndexerState, 196 conn: &mut AsyncPgConnection, 197 commit: AtpCommitEvent, 198 ) -> eyre::Result<()> { ··· 268 } 269 270 for op in &commit.ops { 271 + process_op(t, &mut state.idxc_tx, &commit.repo, op, &blocks).await?; 272 } 273 } else { 274 let items = commit ··· 339 #[inline(always)] 340 async fn process_op( 341 conn: &mut AsyncPgConnection, 342 + deltas: &mut impl AggregateDeltaStore, 343 repo: &str, 344 op: &CommitOp, 345 blocks: &HashMap<Cid, Vec<u8>>, ··· 368 return Ok(()); 369 }; 370 371 + index_op(conn, deltas, repo, cid, decoded, &full_path, rkey).await?; 372 } else if op.action == "delete" { 373 + index_op_delete(conn, deltas, repo, collection, &full_path).await?; 374 } else { 375 tracing::warn!("op contained invalid action {}", op.action); 376 } ··· 395 396 pub async fn index_op( 397 conn: &mut AsyncPgConnection, 398 + deltas: &mut impl AggregateDeltaStore, 399 repo: &str, 400 cid: Cid, 401 record: RecordTypes, ··· 415 } 416 RecordTypes::AppBskyFeedGenerator(record) => { 417 let labels = record.labels.clone(); 418 + let count = db::upsert_feedgen(conn, repo, cid, at_uri, record).await?; 419 420 if let Some(labels) = labels { 421 db::maintain_self_labels(conn, repo, Some(cid), at_uri, labels).await?; 422 } 423 + 424 + deltas 425 + .add_delta(repo, AggregateType::ProfileFeed, count as i32) 426 + .await; 427 } 428 RecordTypes::AppBskyFeedLike(record) => { 429 + let subject = record.subject.uri.clone(); 430 + let count = db::insert_like(conn, repo, at_uri, record).await?; 431 + 432 + deltas 433 + .add_delta(&subject, AggregateType::Like, count as i32) 434 + .await; 435 } 436 RecordTypes::AppBskyFeedPost(record) => { 437 if let Some(records::AppBskyEmbed::RecordWithMedia(embed)) = &record.embed { ··· 440 } 441 } 442 443 + let maybe_reply = record.reply.as_ref().map(|v| v.parent.uri.clone()); 444 + let maybe_embed = record.embed.as_ref().and_then(|v| match v { 445 + records::AppBskyEmbed::Record(r) => Some(r.record.uri.clone()), 446 + records::AppBskyEmbed::RecordWithMedia(r) => Some(r.record.record.uri.clone()), 447 + _ => None, 448 + }); 449 + 450 let labels = record.labels.clone(); 451 db::insert_post(conn, repo, cid, at_uri, record).await?; 452 if let Some(labels) = labels { 453 db::maintain_self_labels(conn, repo, Some(cid), at_uri, labels).await?; 454 } 455 + 456 + deltas.incr(repo, AggregateType::ProfilePost).await; 457 + if let Some(reply) = maybe_reply { 458 + deltas.incr(&reply, AggregateType::Reply).await; 459 + } 460 + if let Some(embed) = maybe_embed { 461 + deltas.incr(&embed, AggregateType::Embed).await; 462 + } 463 } 464 RecordTypes::AppBskyFeedPostgate(record) => { 465 let split_aturi = record.post.rsplitn(4, '/').collect::<Vec<_>>(); ··· 484 .await?; 485 } 486 RecordTypes::AppBskyFeedRepost(record) => { 487 + deltas 488 + .incr(&record.subject.uri, AggregateType::Repost) 489 + .await; 490 db::insert_repost(conn, repo, at_uri, record).await?; 491 } 492 RecordTypes::AppBskyFeedThreadgate(record) => { ··· 502 db::insert_block(conn, repo, at_uri, record).await?; 503 } 504 RecordTypes::AppBskyGraphFollow(record) => { 505 + let subject = record.subject.clone(); 506 + let count = db::insert_follow(conn, repo, at_uri, record).await?; 507 + 508 + deltas 509 + .add_delta(repo, AggregateType::Follow, count as i32) 510 + .await; 511 + deltas 512 + .add_delta(&subject, AggregateType::Follower, count as i32) 513 + .await; 514 } 515 RecordTypes::AppBskyGraphList(record) => { 516 let labels = record.labels.clone(); 517 + let count = db::upsert_list(conn, repo, at_uri, cid, record).await?; 518 519 if let Some(labels) = labels { 520 db::maintain_self_labels(conn, repo, Some(cid), at_uri, labels).await?; 521 } 522 523 + deltas 524 + .add_delta(repo, AggregateType::ProfileList, count as i32) 525 + .await; 526 } 527 RecordTypes::AppBskyGraphListBlock(record) => { 528 db::insert_list_block(conn, repo, at_uri, record).await?; ··· 538 db::insert_list_item(conn, at_uri, record).await?; 539 } 540 RecordTypes::AppBskyGraphStarterPack(record) => { 541 + let count = db::upsert_starterpack(conn, repo, cid, at_uri, record).await?; 542 + deltas 543 + .add_delta(repo, AggregateType::ProfileStarterpack, count as i32) 544 + .await; 545 } 546 RecordTypes::AppBskyGraphVerification(record) => { 547 db::upsert_verification(conn, repo, cid, at_uri, record).await?; ··· 568 569 pub async fn index_op_delete( 570 conn: &mut AsyncPgConnection, 571 + deltas: &mut impl AggregateDeltaStore, 572 repo: &str, 573 collection: CollectionType, 574 at_uri: &str, ··· 576 match collection { 577 CollectionType::BskyProfile => db::delete_profile(conn, repo).await?, 578 CollectionType::BskyBlock => db::delete_block(conn, at_uri).await?, 579 + CollectionType::BskyFeedGen => { 580 + let count = db::delete_feedgen(conn, at_uri).await?; 581 + deltas 582 + .add_delta(repo, AggregateType::ProfileFeed, -(count as i32)) 583 + .await; 584 + count 585 + } 586 + CollectionType::BskyFeedLike => { 587 + if let Some(subject) = db::delete_like(conn, at_uri).await? { 588 + deltas.decr(&subject, AggregateType::Like).await; 589 + } 590 + 0 591 + } 592 + CollectionType::BskyFeedPost => { 593 + let post_info = db::get_post_info_for_delete(conn, at_uri).await?; 594 + 595 + db::delete_post(conn, at_uri).await?; 596 + 597 + if let Some((reply_to, embed)) = post_info { 598 + deltas.decr(repo, AggregateType::ProfilePost).await; 599 + if let Some(reply_to) = reply_to { 600 + deltas.decr(&reply_to, AggregateType::Reply).await; 601 + } 602 + if let Some(embed) = embed { 603 + deltas.decr(&embed, AggregateType::Embed).await; 604 + } 605 + } 606 + 607 + 0 608 + } 609 CollectionType::BskyFeedPostgate => db::delete_postgate(conn, at_uri).await?, 610 + CollectionType::BskyFeedRepost => { 611 + if let Some(subject) = db::delete_repost(conn, at_uri).await? { 612 + deltas.decr(&subject, AggregateType::Repost).await; 613 + } 614 + 0 615 + } 616 CollectionType::BskyFeedThreadgate => db::delete_threadgate(conn, at_uri).await?, 617 CollectionType::BskyFollow => { 618 + if let Some(followee) = db::delete_follow(conn, at_uri).await? { 619 + deltas.decr(&followee, AggregateType::Follower).await; 620 + deltas.decr(repo, AggregateType::Follow).await; 621 + } 622 0 623 } 624 CollectionType::BskyList => { 625 + let count = db::delete_list(conn, at_uri).await?; 626 + deltas 627 + .add_delta(repo, AggregateType::ProfileList, -(count as i32)) 628 + .await; 629 + count 630 } 631 CollectionType::BskyListBlock => db::delete_list_block(conn, at_uri).await?, 632 CollectionType::BskyListItem => db::delete_list_item(conn, at_uri).await?, 633 + CollectionType::BskyStarterPack => { 634 + let count = db::delete_starterpack(conn, at_uri).await?; 635 + deltas 636 + .add_delta(repo, AggregateType::ProfileStarterpack, -(count as i32)) 637 + .await; 638 + count 639 + } 640 CollectionType::BskyVerification => db::delete_verification(conn, at_uri).await?, 641 CollectionType::BskyLabelerService => db::delete_label_service(conn, at_uri).await?, 642 CollectionType::ChatActorDecl => db::delete_chat_decl(conn, at_uri).await?,

+33

consumer/src/indexer/types.rs

··· 121 Update(RecordTypes), 122 Delete, 123 } 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 121 Update(RecordTypes), 122 Delete, 123 } 124 + 125 + pub trait AggregateDeltaStore { 126 + async fn add_delta(&mut self, uri: &str, typ: parakeet_index::AggregateType, delta: i32); 127 + async fn incr(&mut self, uri: &str, typ: parakeet_index::AggregateType) { 128 + self.add_delta(uri, typ, 1).await 129 + } 130 + async fn decr(&mut self, uri: &str, typ: parakeet_index::AggregateType) { 131 + self.add_delta(uri, typ, -1).await 132 + } 133 + } 134 + 135 + impl AggregateDeltaStore for tokio::sync::mpsc::Sender<parakeet_index::AggregateDeltaReq> { 136 + async fn add_delta(&mut self, uri: &str, typ: parakeet_index::AggregateType, delta: i32) { 137 + let res = self 138 + .send(parakeet_index::AggregateDeltaReq { 139 + typ: typ.into(), 140 + uri: uri.to_string(), 141 + delta, 142 + }) 143 + .await; 144 + 145 + if let Err(e) = res { 146 + tracing::error!("failed to send aggregate delta: {e}"); 147 + } 148 + } 149 + } 150 + 151 + impl AggregateDeltaStore for std::collections::HashMap<(String, i32), i32> { 152 + async fn add_delta(&mut self, uri: &str, typ: parakeet_index::AggregateType, delta: i32) { 153 + let key = (uri.to_string(), typ.into()); 154 + self.entry(key).and_modify(|v| *v += delta).or_insert(delta); 155 + } 156 + }

+16 -1

consumer/src/main.rs

··· 31 ..Default::default() 32 })?); 33 0 0 34 let (label_mgr, label_svc_tx) = label_indexer::LabelServiceManager::new( 35 &conf.database_url, 36 resolver.clone(), ··· 42 let (backfiller, backfill_tx) = 43 backfill::BackfillManager::new(pool.clone(), conf.history_mode, resolver.clone()).await?; 44 0 0 45 let (relay_indexer, tx) = indexer::RelayIndexer::new( 46 pool.clone(), 47 backfill_tx, 0 48 resolver.clone(), 49 conf.history_mode, 50 ) 51 .await?; 52 53 - let (firehose_res, indexer_res, backfill_res, label_res) = tokio::try_join! { 54 tokio::spawn(relay_consumer(relay_firehose, tx)), 55 tokio::spawn(relay_indexer.run(conf.indexer_workers)), 56 tokio::spawn(backfiller.run(conf.backfill_workers)), 57 tokio::spawn(label_mgr.run(conf.initial_label_services)), 0 58 }?; 59 60 firehose_res 61 .and(indexer_res) 62 .and(backfill_res) 63 .and(label_res) 0 64 } 65 66 async fn relay_consumer( ··· 83 } 84 } 85 } 0 0 0 0 0 0 0 0 86 87 Ok(()) 88 }

··· 31 ..Default::default() 32 })?); 33 34 + let index_client = parakeet_index::Client::connect(conf.index_uri).await?; 35 + 36 let (label_mgr, label_svc_tx) = label_indexer::LabelServiceManager::new( 37 &conf.database_url, 38 resolver.clone(), ··· 44 let (backfiller, backfill_tx) = 45 backfill::BackfillManager::new(pool.clone(), conf.history_mode, resolver.clone()).await?; 46 47 + let (idxc_tx, idxc_rx) = tokio::sync::mpsc::channel(128); 48 + 49 let (relay_indexer, tx) = indexer::RelayIndexer::new( 50 pool.clone(), 51 backfill_tx, 52 + idxc_tx, 53 resolver.clone(), 54 conf.history_mode, 55 ) 56 .await?; 57 58 + let (firehose_res, indexer_res, backfill_res, label_res, idxt_res) = tokio::try_join! { 59 tokio::spawn(relay_consumer(relay_firehose, tx)), 60 tokio::spawn(relay_indexer.run(conf.indexer_workers)), 61 tokio::spawn(backfiller.run(conf.backfill_workers)), 62 tokio::spawn(label_mgr.run(conf.initial_label_services)), 63 + tokio::spawn(index_transport(index_client, idxc_rx)), 64 }?; 65 66 firehose_res 67 .and(indexer_res) 68 .and(backfill_res) 69 .and(label_res) 70 + .and(idxt_res) 71 } 72 73 async fn relay_consumer( ··· 90 } 91 } 92 } 93 + 94 + Ok(()) 95 + } 96 + 97 + async fn index_transport(mut idxc: parakeet_index::Client, rx: tokio::sync::mpsc::Receiver<parakeet_index::AggregateDeltaReq>) -> eyre::Result<()> { 98 + use tokio_stream::wrappers::ReceiverStream; 99 + 100 + idxc.submit_aggregate_delta_stream(ReceiverStream::new(rx)).await?; 101 102 Ok(()) 103 }

+27

parakeet-index/Cargo.toml

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + [package] 2 + name = "parakeet-index" 3 + version = "0.1.0" 4 + edition = "2024" 5 + 6 + [[bin]] 7 + name = "parakeet-index" 8 + required-features = ["server"] 9 + 10 + [dependencies] 11 + tonic = "0.13.0" 12 + prost = "0.13.5" 13 + 14 + eyre = { version = "0.6.12", optional = true } 15 + figment = { version = "0.10.19", features = ["env", "toml"], optional = true } 16 + itertools = { version = "0.14.0", optional = true } 17 + serde = { version = "1.0.217", features = ["derive"], optional = true } 18 + sled = { version = "0.34.7", optional = true } 19 + tokio = { version = "1.42.0", features = ["full"], optional = true } 20 + tracing = { version = "0.1.40", optional = true } 21 + tracing-subscriber = { version = "0.3.18", optional = true } 22 + 23 + [build-dependencies] 24 + tonic-build = "0.13.0" 25 + 26 + [features] 27 + server = ["dep:eyre", "dep:figment", "dep:itertools", "dep:serde", "dep:sled", "dep:tokio", "dep:tracing", "dep:tracing-subscriber"]

+5

parakeet-index/build.rs

··· 0 0 0 0 0

··· 1 + fn main() -> Result<(), Box<dyn std::error::Error>> { 2 + tonic_build::configure().compile_protos(&["proto/parakeet.proto"], &[""])?; 3 + 4 + Ok(()) 5 + }

+96

parakeet-index/proto/parakeet.proto

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + syntax = "proto3"; 2 + package parakeet; 3 + 4 + service Index { 5 + rpc SubmitAggregateDelta(AggregateDeltaReq) returns (AggregateDeltaRes); 6 + rpc SubmitAggregateDeltaBatch(AggregateDeltaBatchReq) returns (AggregateDeltaRes); 7 + rpc SubmitAggregateDeltaStream(stream AggregateDeltaReq) returns (AggregateDeltaRes); 8 + 9 + rpc GetProfileStats(GetStatsReq) returns (GetProfileStatsRes); 10 + rpc GetProfileStatsMany(GetStatsManyReq) returns (GetProfileStatsManyRes); 11 + rpc GetPostStats(GetStatsReq) returns (GetPostStatsRes); 12 + rpc GetPostStatsMany(GetStatsManyReq) returns (GetPostStatsManyRes); 13 + rpc GetLikeCount(GetStatsReq) returns (GetLikeCountRes); 14 + rpc GetLikeCountMany(GetStatsManyReq) returns (GetLikeCountManyRes); 15 + } 16 + 17 + enum AggregateType { 18 + UNKNOWN = 0; 19 + FOLLOW = 1; 20 + FOLLOWER = 2; 21 + LIKE = 3; 22 + REPLY = 4; 23 + REPOST = 5; 24 + // aka Quotes (in the context of posts) 25 + EMBED = 6; 26 + PROFILE_POST = 7; 27 + PROFILE_LIST = 8; 28 + PROFILE_FEED = 9; 29 + PROFILE_STARTERPACK = 10; 30 + } 31 + 32 + message AggregateDeltaReq { 33 + // The type of aggregate to change 34 + AggregateType typ = 1; 35 + // The entry to change. Can be a full at:// uri for items or a did for actors/profiles 36 + string uri = 2; 37 + sint32 delta = 3; 38 + } 39 + 40 + message AggregateDeltaBatchReq { 41 + repeated AggregateDeltaReq deltas = 1; 42 + } 43 + 44 + message AggregateDeltaRes {} 45 + 46 + message GetStatsReq { 47 + string uri = 1; 48 + } 49 + 50 + message GetStatsManyReq { 51 + repeated string uris = 1; 52 + } 53 + 54 + message ProfileStats { 55 + int32 followers = 1; 56 + int32 following = 2; 57 + int32 posts = 3; 58 + int32 lists = 4; 59 + int32 feeds = 5; 60 + int32 starterpacks = 6; 61 + } 62 + 63 + message GetProfileStatsRes { 64 + optional ProfileStats stats = 1; 65 + } 66 + 67 + message GetProfileStatsManyRes { 68 + map<string, ProfileStats> entries = 1; 69 + } 70 + 71 + message PostStats { 72 + int32 replies = 1; 73 + int32 likes = 2; 74 + int32 reposts = 3; 75 + int32 quotes = 4; 76 + } 77 + 78 + message GetPostStatsRes { 79 + optional PostStats stats = 1; 80 + } 81 + 82 + message GetPostStatsManyRes { 83 + map<string, PostStats> entries = 1; 84 + } 85 + 86 + message LikeCount { 87 + int32 likes = 1; 88 + } 89 + 90 + message GetLikeCountRes { 91 + optional LikeCount likes = 1; 92 + } 93 + 94 + message GetLikeCountManyRes { 95 + map<string, LikeCount> entries = 1; 96 + }

+1

parakeet-index/run.sh

··· 0

··· 1 + cargo run --features server

+10

parakeet-index/src/lib.rs

··· 0 0 0 0 0 0 0 0 0 0

··· 1 + #[allow(clippy::all)] 2 + pub mod index { 3 + tonic::include_proto!("parakeet"); 4 + } 5 + 6 + pub use index::*; 7 + pub type Client = index_client::IndexClient<tonic::transport::Channel>; 8 + 9 + #[cfg(feature = "server")] 10 + pub mod server;

+24

parakeet-index/src/main.rs

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + use parakeet_index::index_server::IndexServer; 2 + use parakeet_index::server::service::Service; 3 + use parakeet_index::server::{GlobalState, config}; 4 + use std::sync::Arc; 5 + use tonic::transport::Server; 6 + 7 + #[tokio::main] 8 + async fn main() -> eyre::Result<()> { 9 + tracing_subscriber::fmt::init(); 10 + 11 + let conf = config::load_config()?; 12 + 13 + let db_root = conf.index_db_path.parse()?; 14 + let addr = std::net::SocketAddr::new(conf.server.bind_address.parse()?, conf.server.port); 15 + let state = Arc::new(GlobalState::new(db_root)?); 16 + 17 + let service = Service::new(state.clone()); 18 + Server::builder() 19 + .add_service(IndexServer::new(service)) 20 + .serve(addr) 21 + .await?; 22 + 23 + Ok(()) 24 + }

+45

parakeet-index/src/server/config.rs

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + use figment::Figment; 2 + use figment::providers::{Env, Format, Toml}; 3 + use serde::Deserialize; 4 + 5 + pub fn load_config() -> eyre::Result<Config> { 6 + let conf = Figment::new() 7 + .merge(Toml::file("Config.toml")) 8 + .merge(Env::prefixed("PKI_")) 9 + .extract()?; 10 + 11 + Ok(conf) 12 + } 13 + 14 + #[derive(Debug, Deserialize)] 15 + pub struct Config { 16 + pub database_url: String, 17 + pub index_db_path: String, 18 + #[serde(default)] 19 + pub server: ConfigServer, 20 + } 21 + 22 + #[derive(Debug, Deserialize)] 23 + pub struct ConfigServer { 24 + #[serde(default = "default_bind_address")] 25 + pub bind_address: String, 26 + #[serde(default = "default_port")] 27 + pub port: u16, 28 + } 29 + 30 + impl Default for ConfigServer { 31 + fn default() -> Self { 32 + ConfigServer { 33 + bind_address: default_bind_address(), 34 + port: default_port(), 35 + } 36 + } 37 + } 38 + 39 + fn default_bind_address() -> String { 40 + "0.0.0.0".to_string() 41 + } 42 + 43 + fn default_port() -> u16 { 44 + 6001 45 + }

+103

parakeet-index/src/server/db.rs

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + use crate::all_none; 2 + use crate::server::utils::{ToIntExt, TreeExt, slice_as_i32}; 3 + use sled::{Db, MergeOperator, Tree}; 4 + use std::path::PathBuf; 5 + 6 + pub struct DbStore { 7 + pub agg_db: Db, 8 + pub label_db: Db, 9 + 10 + pub follows: Tree, 11 + pub followers: Tree, 12 + pub likes: Tree, 13 + pub replies: Tree, 14 + pub reposts: Tree, 15 + pub embeds: Tree, 16 + pub profile_posts: Tree, 17 + pub profile_lists: Tree, 18 + pub profile_feeds: Tree, 19 + pub profile_starterpacks: Tree, 20 + } 21 + 22 + impl DbStore { 23 + pub fn new(db_root: PathBuf) -> eyre::Result<Self> { 24 + let agg_db = sled::open(db_root.join("aggdb"))?; 25 + let label_db = sled::open(db_root.join("labeldb"))?; 26 + 27 + Ok(DbStore { 28 + follows: open_tree(&agg_db, "follows", merge_delta)?, 29 + followers: open_tree(&agg_db, "followers", merge_delta)?, 30 + likes: open_tree(&agg_db, "likes", merge_delta)?, 31 + replies: open_tree(&agg_db, "replies", merge_delta)?, 32 + reposts: open_tree(&agg_db, "reposts", merge_delta)?, 33 + embeds: open_tree(&agg_db, "embeds", merge_delta)?, 34 + profile_posts: open_tree(&agg_db, "profile_posts", merge_delta)?, 35 + profile_lists: open_tree(&agg_db, "profile_lists", merge_delta)?, 36 + profile_feeds: open_tree(&agg_db, "profile_feeds", merge_delta)?, 37 + profile_starterpacks: open_tree(&agg_db, "profile_starterpacks", merge_delta)?, 38 + 39 + agg_db, 40 + label_db, 41 + }) 42 + } 43 + 44 + pub fn get_post_stats(&self, post: &str) -> Option<crate::PostStats> { 45 + let replies = self.replies.get_i32(post); 46 + let likes = self.likes.get_i32(post); 47 + let reposts = self.reposts.get_i32(post); 48 + let quotes = self.embeds.get_i32(post); 49 + 50 + if all_none![replies, likes, reposts, quotes] { 51 + return None; 52 + } 53 + 54 + Some(crate::PostStats { 55 + replies: replies.unwrap_or_default(), 56 + likes: likes.unwrap_or_default(), 57 + reposts: reposts.unwrap_or_default(), 58 + quotes: quotes.unwrap_or_default(), 59 + }) 60 + } 61 + 62 + pub fn get_profile_stats(&self, did: &str) -> Option<crate::ProfileStats> { 63 + let followers = self.followers.get_i32(did); 64 + let following = self.follows.get_i32(did); 65 + let posts = self.profile_posts.get_i32(did); 66 + let lists = self.profile_lists.get_i32(did); 67 + let feeds = self.profile_feeds.get_i32(did); 68 + let starterpacks = self.profile_starterpacks.get_i32(did); 69 + 70 + if all_none![followers, following, posts, lists, feeds, starterpacks] { 71 + return None; 72 + } 73 + 74 + Some(crate::ProfileStats { 75 + followers: followers.unwrap_or_default(), 76 + following: following.unwrap_or_default(), 77 + posts: posts.unwrap_or_default(), 78 + lists: lists.unwrap_or_default(), 79 + feeds: feeds.unwrap_or_default(), 80 + starterpacks: starterpacks.unwrap_or_default(), 81 + }) 82 + } 83 + } 84 + 85 + fn open_tree(db: &Db, name: &str, merge: impl MergeOperator + 'static) -> eyre::Result<Tree> { 86 + let tree = db.open_tree(name)?; 87 + 88 + tree.set_merge_operator(merge); 89 + 90 + Ok(tree) 91 + } 92 + 93 + fn merge_delta(_key: &[u8], old: Option<&[u8]>, new: &[u8]) -> Option<Vec<u8>> { 94 + let old = old.and_then(slice_as_i32); 95 + let new = slice_as_i32(new)?; 96 + 97 + let res = match old { 98 + Some(old) => old + new, 99 + None => new, 100 + }; 101 + 102 + Some(Vec::from_i32(res)) 103 + }

+18

parakeet-index/src/server/mod.rs

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + use std::path::PathBuf; 2 + 3 + pub mod config; 4 + pub mod db; 5 + pub mod service; 6 + mod utils; 7 + 8 + pub struct GlobalState { 9 + pub dbs: db::DbStore, 10 + } 11 + 12 + impl GlobalState { 13 + pub fn new(db_root: PathBuf) -> eyre::Result<Self> { 14 + let dbs = db::DbStore::new(db_root)?; 15 + 16 + Ok(GlobalState { dbs }) 17 + } 18 + }

+201

parakeet-index/src/server/service.rs

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + use crate::index::*; 2 + use crate::server::GlobalState; 3 + use crate::server::utils::TreeExt; 4 + use std::collections::HashMap; 5 + use std::ops::Deref; 6 + use std::sync::Arc; 7 + use tonic::codegen::tokio_stream::StreamExt; 8 + use tonic::{Request, Response, Status, Streaming, async_trait}; 9 + 10 + pub struct Service(Arc<GlobalState>); 11 + 12 + impl Service { 13 + pub fn new(state: Arc<GlobalState>) -> Self { 14 + Service(state) 15 + } 16 + 17 + fn apply_delta( 18 + &self, 19 + uri: &str, 20 + typ: AggregateType, 21 + delta: i32, 22 + ) -> sled::Result<Option<sled::IVec>> { 23 + let val = delta.to_le_bytes(); 24 + 25 + match typ { 26 + AggregateType::Unknown => todo!(), 27 + AggregateType::Follow => self.dbs.follows.merge(uri, val), 28 + AggregateType::Follower => self.dbs.followers.merge(uri, val), 29 + AggregateType::Like => self.dbs.likes.merge(uri, val), 30 + AggregateType::Reply => self.dbs.replies.merge(uri, val), 31 + AggregateType::Repost => self.dbs.reposts.merge(uri, val), 32 + AggregateType::Embed => self.dbs.embeds.merge(uri, val), 33 + AggregateType::ProfilePost => self.dbs.profile_posts.merge(uri, val), 34 + AggregateType::ProfileList => self.dbs.profile_lists.merge(uri, val), 35 + AggregateType::ProfileFeed => self.dbs.profile_feeds.merge(uri, val), 36 + AggregateType::ProfileStarterpack => self.dbs.profile_starterpacks.merge(uri, val), 37 + } 38 + } 39 + } 40 + 41 + impl Deref for Service { 42 + type Target = Arc<GlobalState>; 43 + 44 + fn deref(&self) -> &Self::Target { 45 + &self.0 46 + } 47 + } 48 + 49 + #[async_trait] 50 + impl index_server::Index for Service { 51 + async fn submit_aggregate_delta( 52 + &self, 53 + request: Request<AggregateDeltaReq>, 54 + ) -> Result<Response<AggregateDeltaRes>, Status> { 55 + let inner = request.into_inner(); 56 + 57 + let res = self.apply_delta(&inner.uri, inner.typ(), inner.delta); 58 + 59 + if let Err(e) = res { 60 + tracing::error!("failed to update stats DB: {e}"); 61 + return Err(Status::unknown("failed to update stats DB")); 62 + } 63 + 64 + Ok(Response::new(AggregateDeltaRes {})) 65 + } 66 + 67 + async fn submit_aggregate_delta_batch( 68 + &self, 69 + request: Request<AggregateDeltaBatchReq>, 70 + ) -> Result<Response<AggregateDeltaRes>, Status> { 71 + let inner = request.into_inner(); 72 + 73 + for data in inner.deltas { 74 + let res = self.apply_delta(&data.uri, data.typ(), data.delta); 75 + 76 + if let Err(e) = res { 77 + tracing::error!("failed to update stats DB: {e}"); 78 + return Err(Status::unknown("failed to update stats DB")); 79 + } 80 + } 81 + 82 + Ok(Response::new(AggregateDeltaRes {})) 83 + } 84 + 85 + async fn submit_aggregate_delta_stream( 86 + &self, 87 + request: Request<Streaming<AggregateDeltaReq>>, 88 + ) -> Result<Response<AggregateDeltaRes>, Status> { 89 + let mut inner = request.into_inner(); 90 + 91 + while let Some(req) = inner.next().await { 92 + if let Ok(data) = req { 93 + let res = self.apply_delta(&data.uri, data.typ(), data.delta); 94 + 95 + if let Err(e) = res { 96 + tracing::error!("failed to update stats DB: {e}"); 97 + return Err(Status::unknown("failed to update stats DB")); 98 + } 99 + } else { 100 + tracing::error!("failed to read stream item") 101 + } 102 + } 103 + 104 + Ok(Response::new(AggregateDeltaRes {})) 105 + } 106 + 107 + async fn get_profile_stats( 108 + &self, 109 + request: Request<GetStatsReq>, 110 + ) -> Result<Response<GetProfileStatsRes>, Status> { 111 + let inner = request.into_inner(); 112 + 113 + let stats = self.dbs.get_profile_stats(&inner.uri); 114 + 115 + Ok(Response::new(GetProfileStatsRes { stats })) 116 + } 117 + 118 + async fn get_profile_stats_many( 119 + &self, 120 + request: Request<GetStatsManyReq>, 121 + ) -> Result<Response<GetProfileStatsManyRes>, Status> { 122 + let inner = request.into_inner(); 123 + 124 + // idk if this is the best way of doing this???? 125 + let entries = inner 126 + .uris 127 + .into_iter() 128 + .filter_map(|uri| { 129 + let stats = self.dbs.get_profile_stats(&uri)?; 130 + 131 + Some((uri, stats)) 132 + }) 133 + .collect::<HashMap<_, _>>(); 134 + 135 + Ok(Response::new(GetProfileStatsManyRes { entries })) 136 + } 137 + 138 + async fn get_post_stats( 139 + &self, 140 + request: Request<GetStatsReq>, 141 + ) -> Result<Response<GetPostStatsRes>, Status> { 142 + let inner = request.into_inner(); 143 + 144 + let stats = self.dbs.get_post_stats(&inner.uri); 145 + 146 + Ok(Response::new(GetPostStatsRes { stats })) 147 + } 148 + 149 + async fn get_post_stats_many( 150 + &self, 151 + request: Request<GetStatsManyReq>, 152 + ) -> Result<Response<GetPostStatsManyRes>, Status> { 153 + let inner = request.into_inner(); 154 + 155 + let entries = inner 156 + .uris 157 + .into_iter() 158 + .filter_map(|uri| { 159 + let stats = self.dbs.get_post_stats(&uri)?; 160 + 161 + Some((uri, stats)) 162 + }) 163 + .collect::<HashMap<_, _>>(); 164 + 165 + Ok(Response::new(GetPostStatsManyRes { entries })) 166 + } 167 + 168 + async fn get_like_count( 169 + &self, 170 + request: Request<GetStatsReq>, 171 + ) -> Result<Response<GetLikeCountRes>, Status> { 172 + let inner = request.into_inner(); 173 + 174 + let likes = self 175 + .dbs 176 + .likes 177 + .get_i32(inner.uri) 178 + .map(|likes| LikeCount { likes }); 179 + 180 + Ok(Response::new(GetLikeCountRes { likes })) 181 + } 182 + 183 + async fn get_like_count_many( 184 + &self, 185 + request: Request<GetStatsManyReq>, 186 + ) -> Result<Response<GetLikeCountManyRes>, Status> { 187 + let inner = request.into_inner(); 188 + 189 + let entries = inner 190 + .uris 191 + .into_iter() 192 + .filter_map(|uri| { 193 + let likes = self.dbs.likes.get_i32(&uri)?; 194 + 195 + Some((uri, LikeCount { likes })) 196 + }) 197 + .collect(); 198 + 199 + Ok(Response::new(GetLikeCountManyRes { entries })) 200 + } 201 + }

+59

parakeet-index/src/server/utils.rs

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + use sled::{IVec, Tree}; 2 + 3 + pub trait ToIntExt { 4 + fn as_i32(&self) -> Option<i32>; 5 + fn from_i32(i: i32) -> Self; 6 + } 7 + 8 + impl ToIntExt for IVec { 9 + fn as_i32(&self) -> Option<i32> { 10 + if self.len() == 4 { 11 + let bytes = self[0..4].try_into().ok()?; 12 + Some(i32::from_le_bytes(bytes)) 13 + } else { 14 + None 15 + } 16 + } 17 + 18 + fn from_i32(i: i32) -> Self { 19 + IVec::from(&i.to_le_bytes()) 20 + } 21 + } 22 + 23 + impl ToIntExt for Vec<u8> { 24 + fn as_i32(&self) -> Option<i32> { 25 + if self.len() == 4 { 26 + let bytes = self[0..4].try_into().ok()?; 27 + Some(i32::from_le_bytes(bytes)) 28 + } else { 29 + None 30 + } 31 + } 32 + 33 + fn from_i32(i: i32) -> Self { 34 + Vec::from(&i.to_le_bytes()) 35 + } 36 + } 37 + 38 + pub fn slice_as_i32(data: &[u8]) -> Option<i32> { 39 + let bytes = data[0..4].try_into().ok()?; 40 + 41 + Some(i32::from_le_bytes(bytes)) 42 + } 43 + 44 + pub trait TreeExt { 45 + fn get_i32(&self, key: impl AsRef<[u8]>) -> Option<i32>; 46 + } 47 + 48 + impl TreeExt for Tree { 49 + fn get_i32(&self, key: impl AsRef<[u8]>) -> Option<i32> { 50 + self.get(key).ok().flatten().and_then(|v| v.as_i32()) 51 + } 52 + } 53 + 54 + #[macro_export] 55 + macro_rules! all_none { 56 + ($var0:ident, $($var:ident),*) => { 57 + $var0.is_none() $(&& $var.is_none())* 58 + }; 59 + }

+1

parakeet/Cargo.toml

··· 17 itertools = "0.14.0" 18 lexica = { path = "../lexica" } 19 parakeet-db = { path = "../parakeet-db" } 0 20 serde = { version = "1.0.217", features = ["derive"] } 21 serde_json = "1.0.134" 22 tokio = { version = "1.42.0", features = ["full"] }

··· 17 itertools = "0.14.0" 18 lexica = { path = "../lexica" } 19 parakeet-db = { path = "../parakeet-db" } 20 + parakeet-index = { path = "../parakeet-index" } 21 serde = { version = "1.0.217", features = ["derive"] } 22 serde_json = "1.0.134" 23 tokio = { version = "1.42.0", features = ["full"] }

+1

parakeet/run.sh

··· 0

··· 1 + cargo run

+1

parakeet/src/config.rs

··· 13 14 #[derive(Debug, Deserialize)] 15 pub struct Config { 0 16 pub database_url: String, 17 #[serde(default)] 18 pub server: ConfigServer,

··· 13 14 #[derive(Debug, Deserialize)] 15 pub struct Config { 16 + pub index_uri: String, 17 pub database_url: String, 18 #[serde(default)] 19 pub server: ConfigServer,

+10 -6

parakeet/src/hydration/feedgen.rs

··· 11 feedgen: models::FeedGen, 12 creator: ProfileView, 13 labels: Vec<models::Label>, 0 14 ) -> GeneratorView { 15 let content_mode = feedgen 16 .content_mode ··· 31 avatar: feedgen 32 .avatar_cid 33 .map(|v| format!("https://localhost/feedgen/{v}")), 34 - like_count: 0, 35 accepts_interactions: feedgen.accepts_interactions, 36 labels: map_labels(labels), 37 content_mode, ··· 45 apply_labelers: &[LabelConfigItem], 46 ) -> Option<GeneratorView> { 47 let labels = loaders.label.load(&feedgen, apply_labelers).await; 48 - let feedgen = loaders.feedgen.load(feedgen).await?; 49 let profile = hydrate_profile(loaders, feedgen.owner.clone(), apply_labelers).await?; 50 51 - Some(build_feedgen(feedgen, profile, labels)) 52 } 53 54 pub async fn hydrate_feedgens( ··· 61 62 let creators = feedgens 63 .values() 64 - .map(|feedgen| feedgen.owner.clone()) 65 .collect(); 66 let creators = hydrate_profiles(loaders, creators, apply_labelers).await; 67 68 feedgens 69 .into_iter() 70 - .filter_map(|(uri, feedgen)| { 71 let creator = creators.get(&feedgen.owner)?; 72 let labels = labels.get(&uri).cloned().unwrap_or_default(); 73 74 - Some((uri, build_feedgen(feedgen, creator.to_owned(), labels))) 0 0 0 75 }) 76 .collect() 77 }

··· 11 feedgen: models::FeedGen, 12 creator: ProfileView, 13 labels: Vec<models::Label>, 14 + likes: Option<i32>, 15 ) -> GeneratorView { 16 let content_mode = feedgen 17 .content_mode ··· 32 avatar: feedgen 33 .avatar_cid 34 .map(|v| format!("https://localhost/feedgen/{v}")), 35 + like_count: likes.unwrap_or_default() as i64, 36 accepts_interactions: feedgen.accepts_interactions, 37 labels: map_labels(labels), 38 content_mode, ··· 46 apply_labelers: &[LabelConfigItem], 47 ) -> Option<GeneratorView> { 48 let labels = loaders.label.load(&feedgen, apply_labelers).await; 49 + let (feedgen, likes) = loaders.feedgen.load(feedgen).await?; 50 let profile = hydrate_profile(loaders, feedgen.owner.clone(), apply_labelers).await?; 51 52 + Some(build_feedgen(feedgen, profile, labels, likes)) 53 } 54 55 pub async fn hydrate_feedgens( ··· 62 63 let creators = feedgens 64 .values() 65 + .map(|(feedgen, _)| feedgen.owner.clone()) 66 .collect(); 67 let creators = hydrate_profiles(loaders, creators, apply_labelers).await; 68 69 feedgens 70 .into_iter() 71 + .filter_map(|(uri, (feedgen, likes))| { 72 let creator = creators.get(&feedgen.owner)?; 73 let labels = labels.get(&uri).cloned().unwrap_or_default(); 74 75 + Some(( 76 + uri, 77 + build_feedgen(feedgen, creator.to_owned(), labels, likes), 78 + )) 79 }) 80 .collect() 81 }

+17 -12

parakeet/src/hydration/labeler.rs

··· 12 labeler: models::LabelerService, 13 creator: ProfileView, 14 labels: Vec<models::Label>, 0 15 ) -> LabelerView { 16 LabelerView { 17 uri: format!("at://{}/app.bsky.labeler.service/self", labeler.did), 18 cid: labeler.cid, 19 creator, 20 - like_count: 0, 21 labels: map_labels(labels), 22 indexed_at: labeler.indexed_at, 23 } ··· 28 defs: Vec<models::LabelDefinition>, 29 creator: ProfileView, 30 labels: Vec<models::Label>, 0 31 ) -> LabelerViewDetailed { 32 let reason_types = labeler.reasons.map(|v| { 33 v.into_iter() ··· 66 uri: format!("at://{}/app.bsky.labeler.service/self", labeler.did), 67 cid: labeler.cid, 68 creator, 69 - like_count: 0, 70 policies: LabelerPolicy { 71 label_values, 72 label_value_definitions, ··· 85 apply_labelers: &[LabelConfigItem], 86 ) -> Option<LabelerView> { 87 let labels = loaders.label.load(&labeler, apply_labelers).await; 88 - let (labeler, _) = loaders.labeler.load(labeler).await?; 89 let creator = hydrate_profile(loaders, labeler.did.clone(), apply_labelers).await?; 90 91 - Some(build_view(labeler, creator, labels)) 92 } 93 94 pub async fn hydrate_labelers( ··· 101 102 let creators = labelers 103 .values() 104 - .map(|(labeler, _)| labeler.did.clone()) 105 .collect(); 106 let creators = hydrate_profiles(loaders, creators, apply_labelers).await; 107 108 labelers 109 .into_iter() 110 - .filter_map(|(k, (labeler, _))| { 111 let creator = creators.get(&labeler.did).cloned()?; 112 let labels = labels.get(&k).cloned().unwrap_or_default(); 113 114 - Some((k, build_view(labeler, creator, labels))) 115 }) 116 .collect() 117 } ··· 122 apply_labelers: &[LabelConfigItem], 123 ) -> Option<LabelerViewDetailed> { 124 let labels = loaders.label.load(&labeler, apply_labelers).await; 125 - let (labeler, defs) = loaders.labeler.load(labeler).await?; 126 let creator = hydrate_profile(loaders, labeler.did.clone(), apply_labelers).await?; 127 128 - Some(build_view_detailed(labeler, defs, creator, labels)) 129 } 130 131 pub async fn hydrate_labelers_detailed( ··· 138 139 let creators = labelers 140 .values() 141 - .map(|(labeler, _)| labeler.did.clone()) 142 .collect(); 143 let creators = hydrate_profiles(loaders, creators, apply_labelers).await; 144 145 labelers 146 .into_iter() 147 - .filter_map(|(k, (labeler, defs))| { 148 let creator = creators.get(&labeler.did).cloned()?; 149 let labels = labels.get(&k).cloned().unwrap_or_default(); 150 151 - Some((k, build_view_detailed(labeler, defs, creator, labels))) 0 0 0 152 }) 153 .collect() 154 }

··· 12 labeler: models::LabelerService, 13 creator: ProfileView, 14 labels: Vec<models::Label>, 15 + likes: Option<i32>, 16 ) -> LabelerView { 17 LabelerView { 18 uri: format!("at://{}/app.bsky.labeler.service/self", labeler.did), 19 cid: labeler.cid, 20 creator, 21 + like_count: likes.unwrap_or_default() as i64, 22 labels: map_labels(labels), 23 indexed_at: labeler.indexed_at, 24 } ··· 29 defs: Vec<models::LabelDefinition>, 30 creator: ProfileView, 31 labels: Vec<models::Label>, 32 + likes: Option<i32>, 33 ) -> LabelerViewDetailed { 34 let reason_types = labeler.reasons.map(|v| { 35 v.into_iter() ··· 68 uri: format!("at://{}/app.bsky.labeler.service/self", labeler.did), 69 cid: labeler.cid, 70 creator, 71 + like_count: likes.unwrap_or_default() as i64, 72 policies: LabelerPolicy { 73 label_values, 74 label_value_definitions, ··· 87 apply_labelers: &[LabelConfigItem], 88 ) -> Option<LabelerView> { 89 let labels = loaders.label.load(&labeler, apply_labelers).await; 90 + let (labeler, _, likes) = loaders.labeler.load(labeler).await?; 91 let creator = hydrate_profile(loaders, labeler.did.clone(), apply_labelers).await?; 92 93 + Some(build_view(labeler, creator, labels, likes)) 94 } 95 96 pub async fn hydrate_labelers( ··· 103 104 let creators = labelers 105 .values() 106 + .map(|(labeler, _, _)| labeler.did.clone()) 107 .collect(); 108 let creators = hydrate_profiles(loaders, creators, apply_labelers).await; 109 110 labelers 111 .into_iter() 112 + .filter_map(|(k, (labeler, _, likes))| { 113 let creator = creators.get(&labeler.did).cloned()?; 114 let labels = labels.get(&k).cloned().unwrap_or_default(); 115 116 + Some((k, build_view(labeler, creator, labels, likes))) 117 }) 118 .collect() 119 } ··· 124 apply_labelers: &[LabelConfigItem], 125 ) -> Option<LabelerViewDetailed> { 126 let labels = loaders.label.load(&labeler, apply_labelers).await; 127 + let (labeler, defs, likes) = loaders.labeler.load(labeler).await?; 128 let creator = hydrate_profile(loaders, labeler.did.clone(), apply_labelers).await?; 129 130 + Some(build_view_detailed(labeler, defs, creator, labels, likes)) 131 } 132 133 pub async fn hydrate_labelers_detailed( ··· 140 141 let creators = labelers 142 .values() 143 + .map(|(labeler, _, _)| labeler.did.clone()) 144 .collect(); 145 let creators = hydrate_profiles(loaders, creators, apply_labelers).await; 146 147 labelers 148 .into_iter() 149 + .filter_map(|(k, (labeler, defs, likes))| { 150 let creator = creators.get(&labeler.did).cloned()?; 151 let labels = labels.get(&k).cloned().unwrap_or_default(); 152 153 + Some(( 154 + k, 155 + build_view_detailed(labeler, defs, creator, labels, likes), 156 + )) 157 }) 158 .collect() 159 }

+25 -11

parakeet/src/hydration/posts.rs

··· 7 use lexica::app_bsky::embed::{AspectRatio, Embed}; 8 use lexica::app_bsky::feed::{FeedViewPost, PostView, ReplyRef, ReplyRefPost, ThreadgateView}; 9 use lexica::app_bsky::graph::ListViewBasic; 0 10 use parakeet_db::models; 0 11 use std::collections::HashMap; 12 13 fn build_postview( ··· 16 labels: Vec<models::Label>, 17 embed: Option<Embed>, 18 threadgate: Option<ThreadgateView>, 0 19 ) -> PostView { 0 0 0 0 0 0 0 0 0 20 PostView { 21 uri: post.at_uri, 22 cid: post.cid, 23 author, 24 record: post.record, 25 embed, 26 - stats: Default::default(), 27 labels: map_labels(labels), 28 threadgate, 29 indexed_at: post.created_at, ··· 96 post: String, 97 apply_labelers: &[LabelConfigItem], 98 ) -> Option<PostView> { 99 - let (post, threadgate) = loaders.posts.load(post).await?; 100 let embed = hydrate_embed(loaders, post.at_uri.clone(), apply_labelers).await; 101 let author = hydrate_profile_basic(loaders, post.did.clone(), apply_labelers).await?; 102 let threadgate = hydrate_threadgate(loaders, threadgate, apply_labelers).await; 103 let labels = loaders.label.load(&post.at_uri, apply_labelers).await; 104 105 - Some(build_postview(post, author, labels, embed, threadgate)) 0 0 106 } 107 108 pub async fn hydrate_posts( ··· 114 115 let (authors, post_uris) = posts 116 .values() 117 - .map(|(post, _)| (post.did.clone(), post.at_uri.clone())) 118 .unzip::<_, _, Vec<_>, Vec<_>>(); 119 let authors = hydrate_profiles_basic(loaders, authors, apply_labelers).await; 120 ··· 122 123 let threadgates = posts 124 .values() 125 - .filter_map(|(_, threadgate)| threadgate.clone()) 126 .collect(); 127 let threadgates = hydrate_threadgates(loaders, threadgates, apply_labelers).await; 128 ··· 130 131 posts 132 .into_iter() 133 - .filter_map(|(uri, (post, threadgate))| { 134 let author = authors.get(&post.did)?; 135 let embed = embeds.get(&uri).cloned(); 136 let threadgate = threadgate.and_then(|tg| threadgates.get(&tg.at_uri).cloned()); ··· 138 139 Some(( 140 uri, 141 - build_postview(post, author.to_owned(), labels, embed, threadgate), 142 )) 143 }) 144 .collect() ··· 153 154 let (authors, post_uris) = posts 155 .values() 156 - .map(|(post, _)| (post.did.clone(), post.at_uri.clone())) 157 .unzip::<_, _, Vec<_>, Vec<_>>(); 158 let authors = hydrate_profiles_basic(loaders, authors, apply_labelers).await; 159 ··· 163 164 let reply_refs = posts 165 .values() 166 - .flat_map(|(post, _)| [post.parent_uri.clone(), post.root_uri.clone()]) 167 .flatten() 168 .collect::<Vec<_>>(); 169 ··· 171 172 posts 173 .into_iter() 174 - .filter_map(|(post_uri, (post, threadgate))| { 175 let author = authors.get(&post.did)?; 176 177 let root = post.root_uri.as_ref().and_then(|uri| reply_posts.get(uri)); ··· 203 204 let embed = embeds.get(&post_uri).cloned(); 205 let labels = post_labels.get(&post_uri).cloned().unwrap_or_default(); 206 - let post = build_postview(post, author.to_owned(), labels, embed, None); 207 208 Some(( 209 post_uri,

··· 7 use lexica::app_bsky::embed::{AspectRatio, Embed}; 8 use lexica::app_bsky::feed::{FeedViewPost, PostView, ReplyRef, ReplyRefPost, ThreadgateView}; 9 use lexica::app_bsky::graph::ListViewBasic; 10 + use lexica::app_bsky::RecordStats; 11 use parakeet_db::models; 12 + use parakeet_index::PostStats; 13 use std::collections::HashMap; 14 15 fn build_postview( ··· 18 labels: Vec<models::Label>, 19 embed: Option<Embed>, 20 threadgate: Option<ThreadgateView>, 21 + stats: Option<PostStats>, 22 ) -> PostView { 23 + let stats = stats 24 + .map(|stats| RecordStats { 25 + reply_count: stats.replies as i64, 26 + repost_count: stats.reposts as i64, 27 + like_count: stats.likes as i64, 28 + quote_count: stats.quotes as i64, 29 + }) 30 + .unwrap_or_default(); 31 + 32 PostView { 33 uri: post.at_uri, 34 cid: post.cid, 35 author, 36 record: post.record, 37 embed, 38 + stats, 39 labels: map_labels(labels), 40 threadgate, 41 indexed_at: post.created_at, ··· 108 post: String, 109 apply_labelers: &[LabelConfigItem], 110 ) -> Option<PostView> { 111 + let (post, threadgate, stats) = loaders.posts.load(post).await?; 112 let embed = hydrate_embed(loaders, post.at_uri.clone(), apply_labelers).await; 113 let author = hydrate_profile_basic(loaders, post.did.clone(), apply_labelers).await?; 114 let threadgate = hydrate_threadgate(loaders, threadgate, apply_labelers).await; 115 let labels = loaders.label.load(&post.at_uri, apply_labelers).await; 116 117 + Some(build_postview( 118 + post, author, labels, embed, threadgate, stats, 119 + )) 120 } 121 122 pub async fn hydrate_posts( ··· 128 129 let (authors, post_uris) = posts 130 .values() 131 + .map(|(post, _, _)| (post.did.clone(), post.at_uri.clone())) 132 .unzip::<_, _, Vec<_>, Vec<_>>(); 133 let authors = hydrate_profiles_basic(loaders, authors, apply_labelers).await; 134 ··· 136 137 let threadgates = posts 138 .values() 139 + .filter_map(|(_, threadgate, _)| threadgate.clone()) 140 .collect(); 141 let threadgates = hydrate_threadgates(loaders, threadgates, apply_labelers).await; 142 ··· 144 145 posts 146 .into_iter() 147 + .filter_map(|(uri, (post, threadgate, stats))| { 148 let author = authors.get(&post.did)?; 149 let embed = embeds.get(&uri).cloned(); 150 let threadgate = threadgate.and_then(|tg| threadgates.get(&tg.at_uri).cloned()); ··· 152 153 Some(( 154 uri, 155 + build_postview(post, author.to_owned(), labels, embed, threadgate, stats), 156 )) 157 }) 158 .collect() ··· 167 168 let (authors, post_uris) = posts 169 .values() 170 + .map(|(post, _, _)| (post.did.clone(), post.at_uri.clone())) 171 .unzip::<_, _, Vec<_>, Vec<_>>(); 172 let authors = hydrate_profiles_basic(loaders, authors, apply_labelers).await; 173 ··· 177 178 let reply_refs = posts 179 .values() 180 + .flat_map(|(post, _, _)| [post.parent_uri.clone(), post.root_uri.clone()]) 181 .flatten() 182 .collect::<Vec<_>>(); 183 ··· 185 186 posts 187 .into_iter() 188 + .filter_map(|(post_uri, (post, threadgate, stats))| { 189 let author = authors.get(&post.did)?; 190 191 let root = post.root_uri.as_ref().and_then(|uri| reply_posts.get(uri)); ··· 217 218 let embed = embeds.get(&post_uri).cloned(); 219 let labels = post_labels.get(&post_uri).cloned().unwrap_or_default(); 220 + let post = build_postview(post, author.to_owned(), labels, embed, None, stats); 221 222 Some(( 223 post_uri,

+32 -43

parakeet/src/hydration/profile.rs

··· 2 use crate::loaders::Dataloaders; 3 use lexica::app_bsky::actor::*; 4 use parakeet_db::models; 0 5 use std::collections::HashMap; 6 use std::sync::OnceLock; 7 8 pub static TRUSTED_VERIFIERS: OnceLock<Vec<String>> = OnceLock::new(); 9 10 - fn build_associated(chat: Option<ChatAllowIncoming>, labeler: bool) -> Option<ProfileAssociated> { 11 - if chat.is_some() || labeler { 0 0 0 0 0 0 12 Some(ProfileAssociated { 13 - lists: 0, 14 - feedgens: 0, 15 - starter_packs: 0, 16 labeler, 17 chat: chat.map(|v| ProfileAssociatedChat { allow_incoming: v }), 18 }) ··· 110 is_labeler: bool, 111 labels: Vec<models::Label>, 112 verifications: Option<Vec<models::VerificationEntry>>, 0 113 ) -> ProfileViewBasic { 114 - let associated = build_associated(chat_decl, is_labeler); 115 let verification = build_verification(&profile, &handle, verifications); 116 117 ProfileViewBasic { ··· 135 is_labeler: bool, 136 labels: Vec<models::Label>, 137 verifications: Option<Vec<models::VerificationEntry>>, 0 138 ) -> ProfileView { 139 - let associated = build_associated(chat_decl, is_labeler); 140 let verification = build_verification(&profile, &handle, verifications); 141 142 ProfileView { ··· 158 fn build_detailed( 159 handle: Option<String>, 160 profile: models::Profile, 161 - follow_stats: Option<models::FollowStats>, 162 chat_decl: Option<ChatAllowIncoming>, 163 is_labeler: bool, 164 labels: Vec<models::Label>, 165 verifications: Option<Vec<models::VerificationEntry>>, 0 166 ) -> ProfileViewDetailed { 167 - let associated = build_associated(chat_decl, is_labeler); 168 let verification = build_verification(&profile, &handle, verifications); 169 170 ProfileViewDetailed { ··· 178 banner: profile 179 .banner_cid 180 .map(|v| format!("https://localhost/banner/{v}")), 181 - followers_count: follow_stats 182 - .as_ref() 183 - .map(|v| v.followers as i64) 184 - .unwrap_or_default(), 185 - follows_count: follow_stats 186 - .as_ref() 187 - .map(|v| v.following as i64) 188 - .unwrap_or_default(), 189 associated, 190 labels: map_labels(labels), 191 verification, ··· 201 ) -> Option<ProfileViewBasic> { 202 let labels = loaders.label.load(&did, apply_labelers).await; 203 let verif = loaders.verification.load(did.clone()).await; 204 - let (handle, profile, _, chat_decl, labeler) = loaders.profile.load(did).await?; 205 206 Some(build_basic( 207 - handle, profile, chat_decl, labeler, labels, verif, 208 )) 209 } 210 ··· 219 220 profiles 221 .into_iter() 222 - .map(|(k, (handle, profile, _, chat_decl, labeler))| { 223 let labels = labels.get(&k).cloned().unwrap_or_default(); 224 let verif = verif.get(&k).cloned(); 225 226 ( 227 k, 228 - build_basic(handle, profile, chat_decl, labeler, labels, verif), 229 ) 230 }) 231 .collect() ··· 238 ) -> Option<ProfileView> { 239 let labels = loaders.label.load(&did, apply_labelers).await; 240 let verif = loaders.verification.load(did.clone()).await; 241 - let (handle, profile, _, chat_decl, labeler) = loaders.profile.load(did).await?; 242 243 Some(build_profile( 244 - handle, profile, chat_decl, labeler, labels, verif, 245 )) 246 } 247 ··· 256 257 profiles 258 .into_iter() 259 - .map(|(k, (handle, profile, _, chat_decl, labeler))| { 260 let labels = labels.get(&k).cloned().unwrap_or_default(); 261 let verif = verif.get(&k).cloned(); 262 263 ( 264 k, 265 - build_profile(handle, profile, chat_decl, labeler, labels, verif), 266 ) 267 }) 268 .collect() ··· 275 ) -> Option<ProfileViewDetailed> { 276 let labels = loaders.label.load(&did, apply_labelers).await; 277 let verif = loaders.verification.load(did.clone()).await; 278 - let (handle, profile, follow_stats, chat_decl, labeler) = loaders.profile.load(did).await?; 279 280 Some(build_detailed( 281 - handle, 282 - profile, 283 - follow_stats, 284 - chat_decl, 285 - labeler, 286 - labels, 287 - verif, 288 )) 289 } 290 ··· 299 300 profiles 301 .into_iter() 302 - .map(|(k, (handle, profile, follow_stats, chat_decl, labeler))| { 303 let labels = labels.get(&k).cloned().unwrap_or_default(); 304 let verif = verif.get(&k).cloned(); 305 306 ( 307 k, 308 - build_detailed( 309 - handle, 310 - profile, 311 - follow_stats, 312 - chat_decl, 313 - labeler, 314 - labels, 315 - verif, 316 - ), 317 ) 318 }) 319 .collect()

··· 2 use crate::loaders::Dataloaders; 3 use lexica::app_bsky::actor::*; 4 use parakeet_db::models; 5 + use parakeet_index::ProfileStats; 6 use std::collections::HashMap; 7 use std::sync::OnceLock; 8 9 pub static TRUSTED_VERIFIERS: OnceLock<Vec<String>> = OnceLock::new(); 10 11 + fn build_associated( 12 + chat: Option<ChatAllowIncoming>, 13 + labeler: bool, 14 + stats: Option<ProfileStats>, 15 + ) -> Option<ProfileAssociated> { 16 + if chat.is_some() || labeler || stats.is_some() { 17 + let stats = stats.unwrap_or_default(); 18 + 19 Some(ProfileAssociated { 20 + lists: stats.lists as i64, 21 + feedgens: stats.feeds as i64, 22 + starter_packs: stats.starterpacks as i64, 23 labeler, 24 chat: chat.map(|v| ProfileAssociatedChat { allow_incoming: v }), 25 }) ··· 117 is_labeler: bool, 118 labels: Vec<models::Label>, 119 verifications: Option<Vec<models::VerificationEntry>>, 120 + stats: Option<ProfileStats>, 121 ) -> ProfileViewBasic { 122 + let associated = build_associated(chat_decl, is_labeler, stats); 123 let verification = build_verification(&profile, &handle, verifications); 124 125 ProfileViewBasic { ··· 143 is_labeler: bool, 144 labels: Vec<models::Label>, 145 verifications: Option<Vec<models::VerificationEntry>>, 146 + stats: Option<ProfileStats>, 147 ) -> ProfileView { 148 + let associated = build_associated(chat_decl, is_labeler, stats); 149 let verification = build_verification(&profile, &handle, verifications); 150 151 ProfileView { ··· 167 fn build_detailed( 168 handle: Option<String>, 169 profile: models::Profile, 0 170 chat_decl: Option<ChatAllowIncoming>, 171 is_labeler: bool, 172 labels: Vec<models::Label>, 173 verifications: Option<Vec<models::VerificationEntry>>, 174 + stats: Option<ProfileStats>, 175 ) -> ProfileViewDetailed { 176 + let associated = build_associated(chat_decl, is_labeler, stats); 177 let verification = build_verification(&profile, &handle, verifications); 178 179 ProfileViewDetailed { ··· 187 banner: profile 188 .banner_cid 189 .map(|v| format!("https://localhost/banner/{v}")), 190 + followers_count: stats.map(|v| v.followers as i64).unwrap_or_default(), 191 + follows_count: stats.map(|v| v.following as i64).unwrap_or_default(), 0 0 0 0 0 0 192 associated, 193 labels: map_labels(labels), 194 verification, ··· 204 ) -> Option<ProfileViewBasic> { 205 let labels = loaders.label.load(&did, apply_labelers).await; 206 let verif = loaders.verification.load(did.clone()).await; 207 + let (handle, profile, chat_decl, labeler, stats) = loaders.profile.load(did).await?; 208 209 Some(build_basic( 210 + handle, profile, chat_decl, labeler, labels, verif, stats, 211 )) 212 } 213 ··· 222 223 profiles 224 .into_iter() 225 + .map(|(k, (handle, profile, chat_decl, labeler, stats))| { 226 let labels = labels.get(&k).cloned().unwrap_or_default(); 227 let verif = verif.get(&k).cloned(); 228 229 ( 230 k, 231 + build_basic(handle, profile, chat_decl, labeler, labels, verif, stats), 232 ) 233 }) 234 .collect() ··· 241 ) -> Option<ProfileView> { 242 let labels = loaders.label.load(&did, apply_labelers).await; 243 let verif = loaders.verification.load(did.clone()).await; 244 + let (handle, profile, chat_decl, labeler, stats) = loaders.profile.load(did).await?; 245 246 Some(build_profile( 247 + handle, profile, chat_decl, labeler, labels, verif, stats, 248 )) 249 } 250 ··· 259 260 profiles 261 .into_iter() 262 + .map(|(k, (handle, profile, chat_decl, labeler, stats))| { 263 let labels = labels.get(&k).cloned().unwrap_or_default(); 264 let verif = verif.get(&k).cloned(); 265 266 ( 267 k, 268 + build_profile(handle, profile, chat_decl, labeler, labels, verif, stats), 269 ) 270 }) 271 .collect() ··· 278 ) -> Option<ProfileViewDetailed> { 279 let labels = loaders.label.load(&did, apply_labelers).await; 280 let verif = loaders.verification.load(did.clone()).await; 281 + let (handle, profile, chat_decl, labeler, stats) = loaders.profile.load(did).await?; 282 283 Some(build_detailed( 284 + handle, profile, chat_decl, labeler, labels, verif, stats, 0 0 0 0 0 0 285 )) 286 } 287 ··· 296 297 profiles 298 .into_iter() 299 + .map(|(k, (handle, profile, chat_decl, labeler, stats))| { 300 let labels = labels.get(&k).cloned().unwrap_or_default(); 301 let verif = verif.get(&k).cloned(); 302 303 ( 304 k, 305 + build_detailed(handle, profile, chat_decl, labeler, labels, verif, stats), 0 0 0 0 0 0 0 0 306 ) 307 }) 308 .collect()

+90 -26

parakeet/src/loaders.rs

··· 26 impl Dataloaders { 27 // for the moment, we set up memory cached loaders 28 // we should build a redis/valkey backend at some point in the future. 29 - pub fn new(pool: Pool<AsyncPgConnection>) -> Dataloaders { 30 Dataloaders { 31 embed: Loader::new(EmbedLoader(pool.clone())), 32 - feedgen: Loader::new(FeedGenLoader(pool.clone())), 33 handle: Loader::new(HandleLoader(pool.clone())), 34 label: LabelLoader(pool.clone()), // CARE: never cache this. 35 - labeler: Loader::new(LabelServiceLoader(pool.clone())), 36 list: Loader::new(ListLoader(pool.clone())), 37 - posts: Loader::new(PostLoader(pool.clone())), 38 - profile: Loader::new(ProfileLoader(pool.clone())), 39 starterpacks: Loader::new(StarterPackLoader(pool.clone())), 40 verification: Loader::new(VerificationLoader(pool.clone())), 41 } ··· 66 } 67 } 68 69 - pub struct ProfileLoader(Pool<AsyncPgConnection>); 70 type ProfileLoaderRet = ( 71 Option<String>, 72 models::Profile, 73 - Option<models::FollowStats>, 74 Option<ChatAllowIncoming>, 75 bool, 0 76 ); 77 impl BatchFn<String, ProfileLoaderRet> for ProfileLoader { 78 async fn load(&mut self, keys: &[String]) -> HashMap<String, ProfileLoaderRet> { ··· 91 schema::actors::did, 92 schema::actors::handle, 93 models::Profile::as_select(), 94 - Option::<models::FollowStats>::as_select(), 95 schema::chat_decls::allow_incoming.nullable(), 96 schema::labelers::cid.nullable(), 97 )) ··· 100 String, 101 Option<String>, 102 models::Profile, 103 - Option<models::FollowStats>, 104 Option<String>, 105 Option<String>, 106 )>(&mut conn) 107 .await; 108 0 0 0 0 0 0 0 0 0 0 0 109 match res { 110 Ok(res) => HashMap::from_iter(res.into_iter().map( 111 - |(did, handle, profile, follow_stats, chat_decl, labeler_cid)| { 112 let chat_decl = chat_decl.and_then(|v| ChatAllowIncoming::from_str(&v).ok()); 113 let is_labeler = labeler_cid.is_some(); 0 114 115 - let val = (handle, profile, follow_stats, chat_decl, is_labeler); 116 117 (did, val) 118 }, ··· 157 } 158 } 159 160 - pub struct FeedGenLoader(Pool<AsyncPgConnection>); 161 - type FeedGenLoaderRet = models::FeedGen; //todo: when we have likes, we'll need the count here 162 impl BatchFn<String, FeedGenLoaderRet> for FeedGenLoader { 163 async fn load(&mut self, keys: &[String]) -> HashMap<String, FeedGenLoaderRet> { 164 let mut conn = self.0.get().await.unwrap(); ··· 169 .load(&mut conn) 170 .await; 171 0 0 0 0 0 0 0 0 0 0 0 172 match res { 173 - Ok(res) => HashMap::from_iter( 174 - res.into_iter() 175 - .map(|feedgen| (feedgen.at_uri.clone(), feedgen)), 176 - ), 0 177 Err(e) => { 178 tracing::error!("feedgen load failed: {e}"); 179 HashMap::new() ··· 182 } 183 } 184 185 - pub struct PostLoader(Pool<AsyncPgConnection>); 186 - type PostLoaderRet = (models::Post, Option<models::Threadgate>); 0 0 0 0 187 impl BatchFn<String, PostLoaderRet> for PostLoader { 188 async fn load(&mut self, keys: &[String]) -> HashMap<String, PostLoaderRet> { 189 let mut conn = self.0.get().await.unwrap(); ··· 198 .load(&mut conn) 199 .await; 200 0 0 0 0 0 0 0 0 0 0 0 201 match res { 202 - Ok(res) => HashMap::from_iter( 203 - res.into_iter() 204 - .map(|(post, threadgate)| (post.at_uri.clone(), (post, threadgate))), 205 - ), 0 206 Err(e) => { 207 tracing::error!("post load failed: {e}"); 208 HashMap::new() ··· 323 } 324 } 325 326 - pub struct LabelServiceLoader(Pool<AsyncPgConnection>); 327 - type LabelServiceLoaderRet = (models::LabelerService, Vec<models::LabelDefinition>); 0 0 0 0 328 impl BatchFn<String, LabelServiceLoaderRet> for LabelServiceLoader { 329 async fn load(&mut self, keys: &[String]) -> HashMap<String, LabelServiceLoaderRet> { 330 let mut conn = self.0.get().await.unwrap(); ··· 343 344 let defs = defs.grouped_by(&labelers); 345 0 0 0 0 0 0 0 0 0 0 0 0 0 346 labelers 347 .into_iter() 348 .zip(defs) 349 - .map(|(labeler, defs)| (labeler.did.clone(), (labeler, defs))) 0 0 0 0 0 0 0 0 0 350 .collect() 351 } 352 }

··· 26 impl Dataloaders { 27 // for the moment, we set up memory cached loaders 28 // we should build a redis/valkey backend at some point in the future. 29 + pub fn new(pool: Pool<AsyncPgConnection>, idxc: parakeet_index::Client) -> Dataloaders { 30 Dataloaders { 31 embed: Loader::new(EmbedLoader(pool.clone())), 32 + feedgen: Loader::new(FeedGenLoader(pool.clone(), idxc.clone())), 33 handle: Loader::new(HandleLoader(pool.clone())), 34 label: LabelLoader(pool.clone()), // CARE: never cache this. 35 + labeler: Loader::new(LabelServiceLoader(pool.clone(), idxc.clone())), 36 list: Loader::new(ListLoader(pool.clone())), 37 + posts: Loader::new(PostLoader(pool.clone(), idxc.clone())), 38 + profile: Loader::new(ProfileLoader(pool.clone(), idxc.clone())), 39 starterpacks: Loader::new(StarterPackLoader(pool.clone())), 40 verification: Loader::new(VerificationLoader(pool.clone())), 41 } ··· 66 } 67 } 68 69 + pub struct ProfileLoader(Pool<AsyncPgConnection>, parakeet_index::Client); 70 type ProfileLoaderRet = ( 71 Option<String>, 72 models::Profile, 0 73 Option<ChatAllowIncoming>, 74 bool, 75 + Option<parakeet_index::ProfileStats>, 76 ); 77 impl BatchFn<String, ProfileLoaderRet> for ProfileLoader { 78 async fn load(&mut self, keys: &[String]) -> HashMap<String, ProfileLoaderRet> { ··· 91 schema::actors::did, 92 schema::actors::handle, 93 models::Profile::as_select(), 0 94 schema::chat_decls::allow_incoming.nullable(), 95 schema::labelers::cid.nullable(), 96 )) ··· 99 String, 100 Option<String>, 101 models::Profile, 0 102 Option<String>, 103 Option<String>, 104 )>(&mut conn) 105 .await; 106 107 + let stats_req = parakeet_index::GetStatsManyReq { 108 + uris: keys.to_vec(), 109 + }; 110 + let mut stats = self 111 + .1 112 + .get_profile_stats_many(stats_req) 113 + .await 114 + .unwrap() 115 + .into_inner() 116 + .entries; 117 + 118 match res { 119 Ok(res) => HashMap::from_iter(res.into_iter().map( 120 + |(did, handle, profile, chat_decl, labeler_cid)| { 121 let chat_decl = chat_decl.and_then(|v| ChatAllowIncoming::from_str(&v).ok()); 122 let is_labeler = labeler_cid.is_some(); 123 + let maybe_stats = stats.remove(&did); 124 125 + let val = (handle, profile, chat_decl, is_labeler, maybe_stats); 126 127 (did, val) 128 }, ··· 167 } 168 } 169 170 + pub struct FeedGenLoader(Pool<AsyncPgConnection>, parakeet_index::Client); 171 + type FeedGenLoaderRet = (models::FeedGen, Option<i32>); 172 impl BatchFn<String, FeedGenLoaderRet> for FeedGenLoader { 173 async fn load(&mut self, keys: &[String]) -> HashMap<String, FeedGenLoaderRet> { 174 let mut conn = self.0.get().await.unwrap(); ··· 179 .load(&mut conn) 180 .await; 181 182 + let stats_req = parakeet_index::GetStatsManyReq { 183 + uris: keys.to_vec(), 184 + }; 185 + let mut stats = self 186 + .1 187 + .get_like_count_many(stats_req) 188 + .await 189 + .unwrap() 190 + .into_inner() 191 + .entries; 192 + 193 match res { 194 + Ok(res) => HashMap::from_iter(res.into_iter().map(|feedgen| { 195 + let likes = stats.remove(&feedgen.at_uri).map(|v| v.likes); 196 + 197 + (feedgen.at_uri.clone(), (feedgen, likes)) 198 + })), 199 Err(e) => { 200 tracing::error!("feedgen load failed: {e}"); 201 HashMap::new() ··· 204 } 205 } 206 207 + pub struct PostLoader(Pool<AsyncPgConnection>, parakeet_index::Client); 208 + type PostLoaderRet = ( 209 + models::Post, 210 + Option<models::Threadgate>, 211 + Option<parakeet_index::PostStats>, 212 + ); 213 impl BatchFn<String, PostLoaderRet> for PostLoader { 214 async fn load(&mut self, keys: &[String]) -> HashMap<String, PostLoaderRet> { 215 let mut conn = self.0.get().await.unwrap(); ··· 224 .load(&mut conn) 225 .await; 226 227 + let stats_req = parakeet_index::GetStatsManyReq { 228 + uris: keys.to_vec(), 229 + }; 230 + let mut stats = self 231 + .1 232 + .get_post_stats_many(stats_req) 233 + .await 234 + .unwrap() 235 + .into_inner() 236 + .entries; 237 + 238 match res { 239 + Ok(res) => HashMap::from_iter(res.into_iter().map(|(post, threadgate)| { 240 + let maybe_stats = stats.remove(&post.at_uri); 241 + 242 + (post.at_uri.clone(), (post, threadgate, maybe_stats)) 243 + })), 244 Err(e) => { 245 tracing::error!("post load failed: {e}"); 246 HashMap::new() ··· 361 } 362 } 363 364 + pub struct LabelServiceLoader(Pool<AsyncPgConnection>, parakeet_index::Client); 365 + type LabelServiceLoaderRet = ( 366 + models::LabelerService, 367 + Vec<models::LabelDefinition>, 368 + Option<i32>, 369 + ); 370 impl BatchFn<String, LabelServiceLoaderRet> for LabelServiceLoader { 371 async fn load(&mut self, keys: &[String]) -> HashMap<String, LabelServiceLoaderRet> { 372 let mut conn = self.0.get().await.unwrap(); ··· 385 386 let defs = defs.grouped_by(&labelers); 387 388 + let uris = keys 389 + .iter() 390 + .map(|v| format!("at://{v}/app.bsky.labeler.service/self")) 391 + .collect(); 392 + let stats_req = parakeet_index::GetStatsManyReq { uris }; 393 + let mut stats = self 394 + .1 395 + .get_like_count_many(stats_req) 396 + .await 397 + .unwrap() 398 + .into_inner() 399 + .entries; 400 + 401 labelers 402 .into_iter() 403 .zip(defs) 404 + .map(|(labeler, defs)| { 405 + let likes = stats 406 + .remove(&format!( 407 + "at://{}/app.bsky.labeler.service/self", 408 + &labeler.did 409 + )) 410 + .map(|v| v.likes); 411 + 412 + (labeler.did.clone(), (labeler, defs, likes)) 413 + }) 414 .collect() 415 } 416 }

+12 -2

parakeet/src/main.rs

··· 14 pub struct GlobalState { 15 pub pool: Pool<AsyncPgConnection>, 16 pub dataloaders: Arc<loaders::Dataloaders>, 0 17 } 18 19 #[tokio::main] ··· 25 let db_mgr = AsyncDieselConnectionManager::<AsyncPgConnection>::new(&conf.database_url); 26 let pool = Pool::builder(db_mgr).build()?; 27 28 - let dataloaders = Arc::new(loaders::Dataloaders::new(pool.clone())); 0 0 0 0 0 29 30 #[allow(unused)] 31 hydration::TRUSTED_VERIFIERS.set(conf.trusted_verifiers); ··· 44 ) 45 .layer(TraceLayer::new_for_http()) 46 .layer(cors) 47 - .with_state(GlobalState { pool, dataloaders }); 0 0 0 0 48 49 let addr = std::net::SocketAddr::new(conf.server.bind_address.parse()?, conf.server.port); 50 let listener = tokio::net::TcpListener::bind(addr).await?;

··· 14 pub struct GlobalState { 15 pub pool: Pool<AsyncPgConnection>, 16 pub dataloaders: Arc<loaders::Dataloaders>, 17 + pub index_client: parakeet_index::Client, 18 } 19 20 #[tokio::main] ··· 26 let db_mgr = AsyncDieselConnectionManager::<AsyncPgConnection>::new(&conf.database_url); 27 let pool = Pool::builder(db_mgr).build()?; 28 29 + let index_client = parakeet_index::Client::connect(conf.index_uri).await?; 30 + 31 + let dataloaders = Arc::new(loaders::Dataloaders::new( 32 + pool.clone(), 33 + index_client.clone(), 34 + )); 35 36 #[allow(unused)] 37 hydration::TRUSTED_VERIFIERS.set(conf.trusted_verifiers); ··· 50 ) 51 .layer(TraceLayer::new_for_http()) 52 .layer(cors) 53 + .with_state(GlobalState { 54 + pool, 55 + dataloaders, 56 + index_client, 57 + }); 58 59 let addr = std::net::SocketAddr::new(conf.server.bind_address.parse()?, conf.server.port); 60 let listener = tokio::net::TcpListener::bind(addr).await?;