Add new get_many_to_many XRPC endpoint #7

+1406 -889

39 changed files

Diff round #3

expand all

unified split

.gitignore

.prettierrc

Cargo.lock

Cargo.toml

constellation

Cargo.toml

src

bin

main.rs

lib.rs

server

mod.rs

storage

mem_store.rs

mod.rs

rocks_store.rs

templates

get-backlinks-count.html.j2

get-many-to-many.html.j2

hello.html.j2

try-it-macros.html.j2

lexicons

blue.microcosm

links

getBacklinksCount.json

getManyToMany.json

pocket

Cargo.toml

quasar

Cargo.toml

readme.md

reflector

Cargo.toml

slingshot

Cargo.toml

api-description.md

readme.md

src

consumer.rs

firehose_cache.rs

identity.rs

lib.rs

main.rs

record.rs

server.rs

static

index.html

spacedust

Cargo.toml

src

main.rs

server.rs

ufos

Cargo.toml

src

main.rs

server

mod.rs

who-am-i

Cargo.toml

-1

.gitignore

··· 1 /target 2 local/ 3 - rocks.test

··· 1 /target 2 local/

-4

.prettierrc

··· 1 - { 2 - "tabWidth": 2, 3 - "useTabs": false 4 - }

···

+352 -175

Cargo.lock

··· 24 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" 25 dependencies = [ 26 "cfg-if", 27 "once_cell", 28 "version_check", 29 "zerocopy 0.7.35", ··· 122 checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" 123 124 [[package]] 125 name = "arrayvec" 126 version = "0.7.6" 127 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 155 "proc-macro2", 156 "quote", 157 "serde", 158 - "syn", 159 ] 160 161 [[package]] ··· 197 dependencies = [ 198 "proc-macro2", 199 "quote", 200 - "syn", 201 "synstructure", 202 ] 203 ··· 209 dependencies = [ 210 "proc-macro2", 211 "quote", 212 - "syn", 213 ] 214 215 [[package]] ··· 255 dependencies = [ 256 "proc-macro2", 257 "quote", 258 - "syn", 259 ] 260 261 [[package]] 262 name = "async-trait" ··· 266 dependencies = [ 267 "proc-macro2", 268 "quote", 269 - "syn", 270 ] 271 272 [[package]] ··· 471 "serde_json", 472 "thiserror 1.0.69", 473 "trait-variant", 474 ] 475 476 [[package]] ··· 712 "regex", 713 "rustc-hash 1.1.0", 714 "shlex", 715 - "syn", 716 "which", 717 ] 718 ··· 725 "bitflags", 726 "cexpr", 727 "clang-sys", 728 - "itertools 0.12.1", 729 "proc-macro2", 730 "quote", 731 "regex", 732 "rustc-hash 1.1.0", 733 "shlex", 734 - "syn", 735 ] 736 737 [[package]] ··· 743 "bitflags", 744 "cexpr", 745 "clang-sys", 746 - "itertools 0.12.1", 747 "proc-macro2", 748 "quote", 749 "regex", 750 "rustc-hash 2.1.1", 751 "shlex", 752 - "syn", 753 ] 754 755 [[package]] ··· 803 804 [[package]] 805 name = "bytes" 806 - version = "1.11.1" 807 source = "registry+https://github.com/rust-lang/crates.io-index" 808 - checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" 809 810 [[package]] 811 name = "byteview" ··· 955 956 [[package]] 957 name = "clap" 958 - version = "4.5.56" 959 source = "registry+https://github.com/rust-lang/crates.io-index" 960 - checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" 961 dependencies = [ 962 "clap_builder", 963 "clap_derive", ··· 965 966 [[package]] 967 name = "clap_builder" 968 - version = "4.5.56" 969 source = "registry+https://github.com/rust-lang/crates.io-index" 970 - checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" 971 dependencies = [ 972 "anstream", 973 "anstyle", 974 "clap_lex", 975 - "strsim", 976 ] 977 978 [[package]] 979 name = "clap_derive" 980 - version = "4.5.55" 981 source = "registry+https://github.com/rust-lang/crates.io-index" 982 - checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" 983 dependencies = [ 984 "heck", 985 "proc-macro2", 986 "quote", 987 - "syn", 988 ] 989 990 [[package]] ··· 1136 ] 1137 1138 [[package]] 1139 - name = "core_affinity" 1140 - version = "0.8.3" 1141 - source = "registry+https://github.com/rust-lang/crates.io-index" 1142 - checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342" 1143 - dependencies = [ 1144 - "libc", 1145 - "num_cpus", 1146 - "winapi", 1147 - ] 1148 - 1149 - [[package]] 1150 name = "cpufeatures" 1151 version = "0.2.17" 1152 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1244 1245 [[package]] 1246 name = "darling" 1247 version = "0.20.11" 1248 source = "registry+https://github.com/rust-lang/crates.io-index" 1249 checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" 1250 dependencies = [ 1251 - "darling_core", 1252 - "darling_macro", 1253 ] 1254 1255 [[package]] ··· 1262 "ident_case", 1263 "proc-macro2", 1264 "quote", 1265 - "strsim", 1266 - "syn", 1267 ] 1268 1269 [[package]] ··· 1272 source = "registry+https://github.com/rust-lang/crates.io-index" 1273 checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" 1274 dependencies = [ 1275 - "darling_core", 1276 "quote", 1277 - "syn", 1278 ] 1279 1280 [[package]] ··· 1314 checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" 1315 dependencies = [ 1316 "data-encoding", 1317 - "syn", 1318 ] 1319 1320 [[package]] ··· 1373 source = "registry+https://github.com/rust-lang/crates.io-index" 1374 checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" 1375 dependencies = [ 1376 - "darling", 1377 "proc-macro2", 1378 "quote", 1379 - "syn", 1380 ] 1381 1382 [[package]] ··· 1386 checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" 1387 dependencies = [ 1388 "derive_builder_core", 1389 - "syn", 1390 ] 1391 1392 [[package]] ··· 1406 dependencies = [ 1407 "proc-macro2", 1408 "quote", 1409 - "syn", 1410 "unicode-xid", 1411 ] 1412 1413 [[package]] 1414 name = "digest" 1415 version = "0.10.7" 1416 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1451 dependencies = [ 1452 "proc-macro2", 1453 "quote", 1454 - "syn", 1455 ] 1456 1457 [[package]] ··· 1459 version = "0.1.0" 1460 source = "registry+https://github.com/rust-lang/crates.io-index" 1461 checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" 1462 1463 [[package]] 1464 name = "dropshot" ··· 1504 "thiserror 2.0.16", 1505 "tokio", 1506 "tokio-rustls 0.25.0", 1507 - "toml", 1508 "uuid", 1509 "version_check", 1510 "waitgroup", ··· 1522 "semver", 1523 "serde", 1524 "serde_tokenstream", 1525 - "syn", 1526 ] 1527 1528 [[package]] ··· 1595 "heck", 1596 "proc-macro2", 1597 "quote", 1598 - "syn", 1599 ] 1600 1601 [[package]] ··· 1607 "once_cell", 1608 "proc-macro2", 1609 "quote", 1610 - "syn", 1611 ] 1612 1613 [[package]] ··· 1681 version = "0.1.9" 1682 source = "registry+https://github.com/rust-lang/crates.io-index" 1683 checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" 1684 - 1685 - [[package]] 1686 - name = "fastant" 1687 - version = "0.1.11" 1688 - source = "registry+https://github.com/rust-lang/crates.io-index" 1689 - checksum = "2e825441bfb2d831c47c97d05821552db8832479f44c571b97fededbf0099c07" 1690 - dependencies = [ 1691 - "small_ctor", 1692 - "web-time", 1693 - ] 1694 1695 [[package]] 1696 name = "fastrand" ··· 1767 source = "registry+https://github.com/rust-lang/crates.io-index" 1768 checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" 1769 dependencies = [ 1770 "spin", 1771 ] 1772 ··· 1783 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" 1784 1785 [[package]] 1786 - name = "foldhash" 1787 - version = "0.2.0" 1788 - source = "registry+https://github.com/rust-lang/crates.io-index" 1789 - checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" 1790 - 1791 - [[package]] 1792 name = "foreign-types" 1793 version = "0.3.2" 1794 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1814 1815 [[package]] 1816 name = "foyer" 1817 - version = "0.22.3" 1818 source = "registry+https://github.com/rust-lang/crates.io-index" 1819 - checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" 1820 dependencies = [ 1821 - "anyhow", 1822 "equivalent", 1823 "foyer-common", 1824 "foyer-memory", 1825 "foyer-storage", 1826 - "foyer-tokio", 1827 - "futures-util", 1828 - "mea", 1829 "mixtrics", 1830 "pin-project", 1831 "serde", 1832 "tracing", 1833 ] 1834 1835 [[package]] 1836 name = "foyer-common" 1837 - version = "0.22.3" 1838 source = "registry+https://github.com/rust-lang/crates.io-index" 1839 - checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" 1840 dependencies = [ 1841 - "anyhow", 1842 "bincode 1.3.3", 1843 "bytes", 1844 "cfg-if", 1845 - "foyer-tokio", 1846 "mixtrics", 1847 "parking_lot", 1848 "pin-project", 1849 "serde", 1850 "twox-hash", 1851 ] 1852 ··· 1861 1862 [[package]] 1863 name = "foyer-memory" 1864 - version = "0.22.3" 1865 source = "registry+https://github.com/rust-lang/crates.io-index" 1866 - checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" 1867 dependencies = [ 1868 - "anyhow", 1869 "bitflags", 1870 "cmsketch", 1871 "equivalent", 1872 "foyer-common", 1873 "foyer-intrusive-collections", 1874 - "foyer-tokio", 1875 - "futures-util", 1876 - "hashbrown 0.16.1", 1877 "itertools 0.14.0", 1878 - "mea", 1879 "mixtrics", 1880 "parking_lot", 1881 - "paste", 1882 "pin-project", 1883 "serde", 1884 "tracing", 1885 ] 1886 1887 [[package]] 1888 name = "foyer-storage" 1889 - version = "0.22.3" 1890 source = "registry+https://github.com/rust-lang/crates.io-index" 1891 - checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" 1892 dependencies = [ 1893 "allocator-api2", 1894 "anyhow", 1895 "bytes", 1896 - "core_affinity", 1897 "equivalent", 1898 - "fastant", 1899 "foyer-common", 1900 "foyer-memory", 1901 - "foyer-tokio", 1902 "fs4 0.13.1", 1903 "futures-core", 1904 "futures-util", 1905 - "hashbrown 0.16.1", 1906 - "io-uring", 1907 "itertools 0.14.0", 1908 "libc", 1909 "lz4", 1910 - "mea", 1911 "parking_lot", 1912 "pin-project", 1913 "rand 0.9.1", 1914 "serde", 1915 "tracing", 1916 "twox-hash", 1917 "zstd", 1918 - ] 1919 - 1920 - [[package]] 1921 - name = "foyer-tokio" 1922 - version = "0.22.3" 1923 - source = "registry+https://github.com/rust-lang/crates.io-index" 1924 - checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" 1925 - dependencies = [ 1926 - "tokio", 1927 ] 1928 1929 [[package]] ··· 2008 dependencies = [ 2009 "proc-macro2", 2010 "quote", 2011 - "syn", 2012 ] 2013 2014 [[package]] ··· 2172 version = "0.12.3" 2173 source = "registry+https://github.com/rust-lang/crates.io-index" 2174 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 2175 2176 [[package]] 2177 name = "hashbrown" ··· 2187 dependencies = [ 2188 "allocator-api2", 2189 "equivalent", 2190 - "foldhash 0.1.5", 2191 - ] 2192 - 2193 - [[package]] 2194 - name = "hashbrown" 2195 - version = "0.16.1" 2196 - source = "registry+https://github.com/rust-lang/crates.io-index" 2197 - checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" 2198 - dependencies = [ 2199 - "allocator-api2", 2200 - "equivalent", 2201 - "foldhash 0.2.0", 2202 ] 2203 2204 [[package]] ··· 2632 dependencies = [ 2633 "proc-macro2", 2634 "quote", 2635 - "syn", 2636 ] 2637 2638 [[package]] ··· 2772 2773 [[package]] 2774 name = "itertools" 2775 version = "0.14.0" 2776 source = "registry+https://github.com/rust-lang/crates.io-index" 2777 checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" ··· 2827 dependencies = [ 2828 "proc-macro2", 2829 "quote", 2830 - "syn", 2831 ] 2832 2833 [[package]] ··· 3179 ] 3180 3181 [[package]] 3182 name = "match_cfg" 3183 version = "0.1.0" 3184 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3198 version = "0.8.4" 3199 source = "registry+https://github.com/rust-lang/crates.io-index" 3200 checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" 3201 - 3202 - [[package]] 3203 - name = "mea" 3204 - version = "0.6.3" 3205 - source = "registry+https://github.com/rust-lang/crates.io-index" 3206 - checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" 3207 - dependencies = [ 3208 - "slab", 3209 - ] 3210 3211 [[package]] 3212 name = "mediatype" ··· 3443 ] 3444 3445 [[package]] 3446 name = "native-tls" 3447 version = "0.2.14" 3448 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3502 3503 [[package]] 3504 name = "num-bigint-dig" 3505 - version = "0.8.6" 3506 source = "registry+https://github.com/rust-lang/crates.io-index" 3507 - checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" 3508 dependencies = [ 3509 "lazy_static", 3510 "libm", 3511 "num-integer", ··· 3578 ] 3579 3580 [[package]] 3581 - name = "num_cpus" 3582 - version = "1.17.0" 3583 - source = "registry+https://github.com/rust-lang/crates.io-index" 3584 - checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" 3585 - dependencies = [ 3586 - "hermit-abi", 3587 - "libc", 3588 - ] 3589 - 3590 - [[package]] 3591 name = "num_threads" 3592 version = "0.1.7" 3593 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3658 dependencies = [ 3659 "proc-macro2", 3660 "quote", 3661 - "syn", 3662 ] 3663 3664 [[package]] ··· 3690 ] 3691 3692 [[package]] 3693 name = "p256" 3694 version = "0.13.2" 3695 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3712 ] 3713 3714 [[package]] 3715 name = "parking" 3716 version = "2.2.1" 3717 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3820 "pest_meta", 3821 "proc-macro2", 3822 "quote", 3823 - "syn", 3824 ] 3825 3826 [[package]] ··· 3850 dependencies = [ 3851 "proc-macro2", 3852 "quote", 3853 - "syn", 3854 ] 3855 3856 [[package]] ··· 3968 "proc-macro-crate", 3969 "proc-macro2", 3970 "quote", 3971 - "syn", 3972 ] 3973 3974 [[package]] ··· 4003 source = "registry+https://github.com/rust-lang/crates.io-index" 4004 checksum = "41273b691a3d467a8c44d05506afba9f7b6bd56c9cdf80123de13fe52d7ec587" 4005 dependencies = [ 4006 - "darling", 4007 "http", 4008 "indexmap 2.11.4", 4009 "mime", ··· 4011 "proc-macro2", 4012 "quote", 4013 "regex", 4014 - "syn", 4015 "thiserror 2.0.16", 4016 ] 4017 ··· 4052 checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" 4053 dependencies = [ 4054 "proc-macro2", 4055 - "syn", 4056 ] 4057 4058 [[package]] ··· 4363 dependencies = [ 4364 "proc-macro2", 4365 "quote", 4366 - "syn", 4367 ] 4368 4369 [[package]] ··· 4514 4515 [[package]] 4516 name = "rsa" 4517 - version = "0.9.10" 4518 source = "registry+https://github.com/rust-lang/crates.io-index" 4519 - checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" 4520 dependencies = [ 4521 "const-oid", 4522 "digest", ··· 4745 "proc-macro2", 4746 "quote", 4747 "serde_derive_internals", 4748 - "syn", 4749 ] 4750 4751 [[package]] ··· 4878 dependencies = [ 4879 "proc-macro2", 4880 "quote", 4881 - "syn", 4882 ] 4883 4884 [[package]] ··· 4889 dependencies = [ 4890 "proc-macro2", 4891 "quote", 4892 - "syn", 4893 ] 4894 4895 [[package]] ··· 4943 4944 [[package]] 4945 name = "serde_spanned" 4946 version = "1.0.2" 4947 source = "registry+https://github.com/rust-lang/crates.io-index" 4948 checksum = "5417783452c2be558477e104686f7de5dae53dba813c28435e0e70f82d9b04ee" ··· 4959 "proc-macro2", 4960 "quote", 4961 "serde", 4962 - "syn", 4963 ] 4964 4965 [[package]] ··· 4998 source = "registry+https://github.com/rust-lang/crates.io-index" 4999 checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" 5000 dependencies = [ 5001 - "darling", 5002 "proc-macro2", 5003 "quote", 5004 - "syn", 5005 ] 5006 5007 [[package]] ··· 5093 5094 [[package]] 5095 name = "slab" 5096 - version = "0.4.12" 5097 source = "registry+https://github.com/rust-lang/crates.io-index" 5098 - checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" 5099 5100 [[package]] 5101 name = "slingshot" ··· 5184 ] 5185 5186 [[package]] 5187 - name = "small_ctor" 5188 - version = "0.1.2" 5189 - source = "registry+https://github.com/rust-lang/crates.io-index" 5190 - checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" 5191 - 5192 - [[package]] 5193 name = "smallvec" 5194 version = "1.15.0" 5195 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5277 5278 [[package]] 5279 name = "strsim" 5280 version = "0.11.1" 5281 source = "registry+https://github.com/rust-lang/crates.io-index" 5282 checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" ··· 5289 5290 [[package]] 5291 name = "syn" 5292 version = "2.0.106" 5293 source = "registry+https://github.com/rust-lang/crates.io-index" 5294 checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" ··· 5315 dependencies = [ 5316 "proc-macro2", 5317 "quote", 5318 - "syn", 5319 ] 5320 5321 [[package]] ··· 5401 dependencies = [ 5402 "proc-macro2", 5403 "quote", 5404 - "syn", 5405 ] 5406 5407 [[package]] ··· 5412 dependencies = [ 5413 "proc-macro2", 5414 "quote", 5415 - "syn", 5416 ] 5417 5418 [[package]] ··· 5537 dependencies = [ 5538 "proc-macro2", 5539 "quote", 5540 - "syn", 5541 ] 5542 5543 [[package]] ··· 5623 5624 [[package]] 5625 name = "toml" 5626 version = "0.9.7" 5627 source = "registry+https://github.com/rust-lang/crates.io-index" 5628 checksum = "00e5e5d9bf2475ac9d4f0d9edab68cc573dc2fd644b0dba36b0c30a92dd9eaa0" 5629 dependencies = [ 5630 "indexmap 2.11.4", 5631 "serde_core", 5632 - "serde_spanned", 5633 "toml_datetime 0.7.2", 5634 "toml_parser", 5635 "toml_writer", ··· 5641 version = "0.6.11" 5642 source = "registry+https://github.com/rust-lang/crates.io-index" 5643 checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" 5644 5645 [[package]] 5646 name = "toml_datetime" ··· 5658 checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" 5659 dependencies = [ 5660 "indexmap 2.11.4", 5661 "toml_datetime 0.6.11", 5662 "winnow", 5663 ] 5664 ··· 5670 dependencies = [ 5671 "winnow", 5672 ] 5673 5674 [[package]] 5675 name = "toml_writer" ··· 5743 dependencies = [ 5744 "proc-macro2", 5745 "quote", 5746 - "syn", 5747 ] 5748 5749 [[package]] ··· 5793 dependencies = [ 5794 "proc-macro2", 5795 "quote", 5796 - "syn", 5797 ] 5798 5799 [[package]] ··· 6115 "log", 6116 "proc-macro2", 6117 "quote", 6118 - "syn", 6119 "wasm-bindgen-shared", 6120 ] 6121 ··· 6150 dependencies = [ 6151 "proc-macro2", 6152 "quote", 6153 - "syn", 6154 "wasm-bindgen-backend", 6155 "wasm-bindgen-shared", 6156 ] ··· 6303 dependencies = [ 6304 "proc-macro2", 6305 "quote", 6306 - "syn", 6307 ] 6308 6309 [[package]] ··· 6314 dependencies = [ 6315 "proc-macro2", 6316 "quote", 6317 - "syn", 6318 ] 6319 6320 [[package]] ··· 6620 dependencies = [ 6621 "proc-macro2", 6622 "quote", 6623 - "syn", 6624 "synstructure", 6625 ] 6626 ··· 6650 dependencies = [ 6651 "proc-macro2", 6652 "quote", 6653 - "syn", 6654 ] 6655 6656 [[package]] ··· 6661 dependencies = [ 6662 "proc-macro2", 6663 "quote", 6664 - "syn", 6665 ] 6666 6667 [[package]] ··· 6681 dependencies = [ 6682 "proc-macro2", 6683 "quote", 6684 - "syn", 6685 "synstructure", 6686 ] 6687 ··· 6703 dependencies = [ 6704 "proc-macro2", 6705 "quote", 6706 - "syn", 6707 ] 6708 6709 [[package]] ··· 6725 dependencies = [ 6726 "proc-macro2", 6727 "quote", 6728 - "syn", 6729 ] 6730 6731 [[package]]

··· 24 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" 25 dependencies = [ 26 "cfg-if", 27 + "getrandom 0.2.15", 28 "once_cell", 29 "version_check", 30 "zerocopy 0.7.35", ··· 123 checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" 124 125 [[package]] 126 + name = "arc-swap" 127 + version = "1.7.1" 128 + source = "registry+https://github.com/rust-lang/crates.io-index" 129 + checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" 130 + 131 + [[package]] 132 name = "arrayvec" 133 version = "0.7.6" 134 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 162 "proc-macro2", 163 "quote", 164 "serde", 165 + "syn 2.0.106", 166 ] 167 168 [[package]] ··· 204 dependencies = [ 205 "proc-macro2", 206 "quote", 207 + "syn 2.0.106", 208 "synstructure", 209 ] 210 ··· 216 dependencies = [ 217 "proc-macro2", 218 "quote", 219 + "syn 2.0.106", 220 + ] 221 + 222 + [[package]] 223 + name = "async-channel" 224 + version = "2.5.0" 225 + source = "registry+https://github.com/rust-lang/crates.io-index" 226 + checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" 227 + dependencies = [ 228 + "concurrent-queue", 229 + "event-listener-strategy", 230 + "futures-core", 231 + "pin-project-lite", 232 ] 233 234 [[package]] ··· 274 dependencies = [ 275 "proc-macro2", 276 "quote", 277 + "syn 2.0.106", 278 ] 279 + 280 + [[package]] 281 + name = "async-task" 282 + version = "4.7.1" 283 + source = "registry+https://github.com/rust-lang/crates.io-index" 284 + checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" 285 286 [[package]] 287 name = "async-trait" ··· 291 dependencies = [ 292 "proc-macro2", 293 "quote", 294 + "syn 2.0.106", 295 ] 296 297 [[package]] ··· 496 "serde_json", 497 "thiserror 1.0.69", 498 "trait-variant", 499 + ] 500 + 501 + [[package]] 502 + name = "auto_enums" 503 + version = "0.8.7" 504 + source = "registry+https://github.com/rust-lang/crates.io-index" 505 + checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781" 506 + dependencies = [ 507 + "derive_utils", 508 + "proc-macro2", 509 + "quote", 510 + "syn 2.0.106", 511 ] 512 513 [[package]] ··· 749 "regex", 750 "rustc-hash 1.1.0", 751 "shlex", 752 + "syn 2.0.106", 753 "which", 754 ] 755 ··· 762 "bitflags", 763 "cexpr", 764 "clang-sys", 765 + "itertools 0.13.0", 766 "proc-macro2", 767 "quote", 768 "regex", 769 "rustc-hash 1.1.0", 770 "shlex", 771 + "syn 2.0.106", 772 ] 773 774 [[package]] ··· 780 "bitflags", 781 "cexpr", 782 "clang-sys", 783 + "itertools 0.13.0", 784 "proc-macro2", 785 "quote", 786 "regex", 787 "rustc-hash 2.1.1", 788 "shlex", 789 + "syn 2.0.106", 790 ] 791 792 [[package]] ··· 840 841 [[package]] 842 name = "bytes" 843 + version = "1.10.1" 844 source = "registry+https://github.com/rust-lang/crates.io-index" 845 + checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" 846 847 [[package]] 848 name = "byteview" ··· 992 993 [[package]] 994 name = "clap" 995 + version = "4.5.48" 996 source = "registry+https://github.com/rust-lang/crates.io-index" 997 + checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" 998 dependencies = [ 999 "clap_builder", 1000 "clap_derive", ··· 1002 1003 [[package]] 1004 name = "clap_builder" 1005 + version = "4.5.48" 1006 source = "registry+https://github.com/rust-lang/crates.io-index" 1007 + checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" 1008 dependencies = [ 1009 "anstream", 1010 "anstyle", 1011 "clap_lex", 1012 + "strsim 0.11.1", 1013 ] 1014 1015 [[package]] 1016 name = "clap_derive" 1017 + version = "4.5.47" 1018 source = "registry+https://github.com/rust-lang/crates.io-index" 1019 + checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" 1020 dependencies = [ 1021 "heck", 1022 "proc-macro2", 1023 "quote", 1024 + "syn 2.0.106", 1025 ] 1026 1027 [[package]] ··· 1173 ] 1174 1175 [[package]] 1176 name = "cpufeatures" 1177 version = "0.2.17" 1178 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1270 1271 [[package]] 1272 name = "darling" 1273 + version = "0.14.4" 1274 + source = "registry+https://github.com/rust-lang/crates.io-index" 1275 + checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" 1276 + dependencies = [ 1277 + "darling_core 0.14.4", 1278 + "darling_macro 0.14.4", 1279 + ] 1280 + 1281 + [[package]] 1282 + name = "darling" 1283 version = "0.20.11" 1284 source = "registry+https://github.com/rust-lang/crates.io-index" 1285 checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" 1286 dependencies = [ 1287 + "darling_core 0.20.11", 1288 + "darling_macro 0.20.11", 1289 + ] 1290 + 1291 + [[package]] 1292 + name = "darling_core" 1293 + version = "0.14.4" 1294 + source = "registry+https://github.com/rust-lang/crates.io-index" 1295 + checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" 1296 + dependencies = [ 1297 + "fnv", 1298 + "ident_case", 1299 + "proc-macro2", 1300 + "quote", 1301 + "strsim 0.10.0", 1302 + "syn 1.0.109", 1303 ] 1304 1305 [[package]] ··· 1312 "ident_case", 1313 "proc-macro2", 1314 "quote", 1315 + "strsim 0.11.1", 1316 + "syn 2.0.106", 1317 + ] 1318 + 1319 + [[package]] 1320 + name = "darling_macro" 1321 + version = "0.14.4" 1322 + source = "registry+https://github.com/rust-lang/crates.io-index" 1323 + checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" 1324 + dependencies = [ 1325 + "darling_core 0.14.4", 1326 + "quote", 1327 + "syn 1.0.109", 1328 ] 1329 1330 [[package]] ··· 1333 source = "registry+https://github.com/rust-lang/crates.io-index" 1334 checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" 1335 dependencies = [ 1336 + "darling_core 0.20.11", 1337 "quote", 1338 + "syn 2.0.106", 1339 ] 1340 1341 [[package]] ··· 1375 checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" 1376 dependencies = [ 1377 "data-encoding", 1378 + "syn 2.0.106", 1379 ] 1380 1381 [[package]] ··· 1434 source = "registry+https://github.com/rust-lang/crates.io-index" 1435 checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" 1436 dependencies = [ 1437 + "darling 0.20.11", 1438 "proc-macro2", 1439 "quote", 1440 + "syn 2.0.106", 1441 ] 1442 1443 [[package]] ··· 1447 checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" 1448 dependencies = [ 1449 "derive_builder_core", 1450 + "syn 2.0.106", 1451 ] 1452 1453 [[package]] ··· 1467 dependencies = [ 1468 "proc-macro2", 1469 "quote", 1470 + "syn 2.0.106", 1471 "unicode-xid", 1472 ] 1473 1474 [[package]] 1475 + name = "derive_utils" 1476 + version = "0.15.0" 1477 + source = "registry+https://github.com/rust-lang/crates.io-index" 1478 + checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" 1479 + dependencies = [ 1480 + "proc-macro2", 1481 + "quote", 1482 + "syn 2.0.106", 1483 + ] 1484 + 1485 + [[package]] 1486 name = "digest" 1487 version = "0.10.7" 1488 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1523 dependencies = [ 1524 "proc-macro2", 1525 "quote", 1526 + "syn 2.0.106", 1527 ] 1528 1529 [[package]] ··· 1531 version = "0.1.0" 1532 source = "registry+https://github.com/rust-lang/crates.io-index" 1533 checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" 1534 + 1535 + [[package]] 1536 + name = "downcast-rs" 1537 + version = "1.2.1" 1538 + source = "registry+https://github.com/rust-lang/crates.io-index" 1539 + checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" 1540 1541 [[package]] 1542 name = "dropshot" ··· 1582 "thiserror 2.0.16", 1583 "tokio", 1584 "tokio-rustls 0.25.0", 1585 + "toml 0.9.7", 1586 "uuid", 1587 "version_check", 1588 "waitgroup", ··· 1600 "semver", 1601 "serde", 1602 "serde_tokenstream", 1603 + "syn 2.0.106", 1604 ] 1605 1606 [[package]] ··· 1673 "heck", 1674 "proc-macro2", 1675 "quote", 1676 + "syn 2.0.106", 1677 ] 1678 1679 [[package]] ··· 1685 "once_cell", 1686 "proc-macro2", 1687 "quote", 1688 + "syn 2.0.106", 1689 ] 1690 1691 [[package]] ··· 1759 version = "0.1.9" 1760 source = "registry+https://github.com/rust-lang/crates.io-index" 1761 checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" 1762 1763 [[package]] 1764 name = "fastrand" ··· 1835 source = "registry+https://github.com/rust-lang/crates.io-index" 1836 checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" 1837 dependencies = [ 1838 + "futures-core", 1839 + "futures-sink", 1840 + "nanorand", 1841 "spin", 1842 ] 1843 ··· 1854 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" 1855 1856 [[package]] 1857 name = "foreign-types" 1858 version = "0.3.2" 1859 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1879 1880 [[package]] 1881 name = "foyer" 1882 + version = "0.18.0" 1883 source = "registry+https://github.com/rust-lang/crates.io-index" 1884 + checksum = "0b4d8e96374206ff1b4265f2e2e6e1f80bc3048957b2a1e7fdeef929d68f318f" 1885 dependencies = [ 1886 "equivalent", 1887 "foyer-common", 1888 "foyer-memory", 1889 "foyer-storage", 1890 + "madsim-tokio", 1891 "mixtrics", 1892 "pin-project", 1893 "serde", 1894 + "thiserror 2.0.16", 1895 + "tokio", 1896 "tracing", 1897 ] 1898 1899 [[package]] 1900 name = "foyer-common" 1901 + version = "0.18.0" 1902 source = "registry+https://github.com/rust-lang/crates.io-index" 1903 + checksum = "911b8e3f23d5fe55b0b240f75af1d2fa5cb7261d3f9b38ef1c57bbc9f0449317" 1904 dependencies = [ 1905 "bincode 1.3.3", 1906 "bytes", 1907 "cfg-if", 1908 + "itertools 0.14.0", 1909 + "madsim-tokio", 1910 "mixtrics", 1911 "parking_lot", 1912 "pin-project", 1913 "serde", 1914 + "thiserror 2.0.16", 1915 + "tokio", 1916 "twox-hash", 1917 ] 1918 ··· 1927 1928 [[package]] 1929 name = "foyer-memory" 1930 + version = "0.18.0" 1931 source = "registry+https://github.com/rust-lang/crates.io-index" 1932 + checksum = "506883d5a8500dea1b1662f7180f3534bdcbfa718d3253db7179552ef83612fa" 1933 dependencies = [ 1934 + "arc-swap", 1935 "bitflags", 1936 "cmsketch", 1937 "equivalent", 1938 "foyer-common", 1939 "foyer-intrusive-collections", 1940 + "hashbrown 0.15.2", 1941 "itertools 0.14.0", 1942 + "madsim-tokio", 1943 "mixtrics", 1944 "parking_lot", 1945 "pin-project", 1946 "serde", 1947 + "thiserror 2.0.16", 1948 + "tokio", 1949 "tracing", 1950 ] 1951 1952 [[package]] 1953 name = "foyer-storage" 1954 + version = "0.18.0" 1955 source = "registry+https://github.com/rust-lang/crates.io-index" 1956 + checksum = "1ba8403a54a2f2032fb647e49c442e5feeb33f3989f7024f1b178341a016f06d" 1957 dependencies = [ 1958 "allocator-api2", 1959 "anyhow", 1960 + "auto_enums", 1961 "bytes", 1962 "equivalent", 1963 + "flume", 1964 "foyer-common", 1965 "foyer-memory", 1966 "fs4 0.13.1", 1967 "futures-core", 1968 "futures-util", 1969 "itertools 0.14.0", 1970 "libc", 1971 "lz4", 1972 + "madsim-tokio", 1973 + "ordered_hash_map", 1974 "parking_lot", 1975 + "paste", 1976 "pin-project", 1977 "rand 0.9.1", 1978 "serde", 1979 + "thiserror 2.0.16", 1980 + "tokio", 1981 "tracing", 1982 "twox-hash", 1983 "zstd", 1984 ] 1985 1986 [[package]] ··· 2065 dependencies = [ 2066 "proc-macro2", 2067 "quote", 2068 + "syn 2.0.106", 2069 ] 2070 2071 [[package]] ··· 2229 version = "0.12.3" 2230 source = "registry+https://github.com/rust-lang/crates.io-index" 2231 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 2232 + 2233 + [[package]] 2234 + name = "hashbrown" 2235 + version = "0.13.2" 2236 + source = "registry+https://github.com/rust-lang/crates.io-index" 2237 + checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" 2238 + dependencies = [ 2239 + "ahash", 2240 + ] 2241 2242 [[package]] 2243 name = "hashbrown" ··· 2253 dependencies = [ 2254 "allocator-api2", 2255 "equivalent", 2256 + "foldhash", 2257 ] 2258 2259 [[package]] ··· 2687 dependencies = [ 2688 "proc-macro2", 2689 "quote", 2690 + "syn 2.0.106", 2691 ] 2692 2693 [[package]] ··· 2827 2828 [[package]] 2829 name = "itertools" 2830 + version = "0.13.0" 2831 + source = "registry+https://github.com/rust-lang/crates.io-index" 2832 + checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" 2833 + dependencies = [ 2834 + "either", 2835 + ] 2836 + 2837 + [[package]] 2838 + name = "itertools" 2839 version = "0.14.0" 2840 source = "registry+https://github.com/rust-lang/crates.io-index" 2841 checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" ··· 2891 dependencies = [ 2892 "proc-macro2", 2893 "quote", 2894 + "syn 2.0.106", 2895 ] 2896 2897 [[package]] ··· 3243 ] 3244 3245 [[package]] 3246 + name = "madsim" 3247 + version = "0.2.32" 3248 + source = "registry+https://github.com/rust-lang/crates.io-index" 3249 + checksum = "db6694555643da293dfb89e33c2880a13b62711d64b6588bc7df6ce4110b27f1" 3250 + dependencies = [ 3251 + "ahash", 3252 + "async-channel", 3253 + "async-stream", 3254 + "async-task", 3255 + "bincode 1.3.3", 3256 + "bytes", 3257 + "downcast-rs", 3258 + "futures-util", 3259 + "lazy_static", 3260 + "libc", 3261 + "madsim-macros", 3262 + "naive-timer", 3263 + "panic-message", 3264 + "rand 0.8.5", 3265 + "rand_xoshiro 0.6.0", 3266 + "rustversion", 3267 + "serde", 3268 + "spin", 3269 + "tokio", 3270 + "tokio-util", 3271 + "toml 0.8.23", 3272 + "tracing", 3273 + "tracing-subscriber", 3274 + ] 3275 + 3276 + [[package]] 3277 + name = "madsim-macros" 3278 + version = "0.2.12" 3279 + source = "registry+https://github.com/rust-lang/crates.io-index" 3280 + checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" 3281 + dependencies = [ 3282 + "darling 0.14.4", 3283 + "proc-macro2", 3284 + "quote", 3285 + "syn 1.0.109", 3286 + ] 3287 + 3288 + [[package]] 3289 + name = "madsim-tokio" 3290 + version = "0.2.30" 3291 + source = "registry+https://github.com/rust-lang/crates.io-index" 3292 + checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5" 3293 + dependencies = [ 3294 + "madsim", 3295 + "spin", 3296 + "tokio", 3297 + ] 3298 + 3299 + [[package]] 3300 name = "match_cfg" 3301 version = "0.1.0" 3302 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3316 version = "0.8.4" 3317 source = "registry+https://github.com/rust-lang/crates.io-index" 3318 checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" 3319 3320 [[package]] 3321 name = "mediatype" ··· 3552 ] 3553 3554 [[package]] 3555 + name = "naive-timer" 3556 + version = "0.2.0" 3557 + source = "registry+https://github.com/rust-lang/crates.io-index" 3558 + checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" 3559 + 3560 + [[package]] 3561 + name = "nanorand" 3562 + version = "0.7.0" 3563 + source = "registry+https://github.com/rust-lang/crates.io-index" 3564 + checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" 3565 + dependencies = [ 3566 + "getrandom 0.2.15", 3567 + ] 3568 + 3569 + [[package]] 3570 name = "native-tls" 3571 version = "0.2.14" 3572 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3626 3627 [[package]] 3628 name = "num-bigint-dig" 3629 + version = "0.8.4" 3630 source = "registry+https://github.com/rust-lang/crates.io-index" 3631 + checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" 3632 dependencies = [ 3633 + "byteorder", 3634 "lazy_static", 3635 "libm", 3636 "num-integer", ··· 3703 ] 3704 3705 [[package]] 3706 name = "num_threads" 3707 version = "0.1.7" 3708 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3773 dependencies = [ 3774 "proc-macro2", 3775 "quote", 3776 + "syn 2.0.106", 3777 ] 3778 3779 [[package]] ··· 3805 ] 3806 3807 [[package]] 3808 + name = "ordered_hash_map" 3809 + version = "0.4.0" 3810 + source = "registry+https://github.com/rust-lang/crates.io-index" 3811 + checksum = "ab0e5f22bf6dd04abd854a8874247813a8fa2c8c1260eba6fbb150270ce7c176" 3812 + dependencies = [ 3813 + "hashbrown 0.13.2", 3814 + ] 3815 + 3816 + [[package]] 3817 name = "p256" 3818 version = "0.13.2" 3819 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3836 ] 3837 3838 [[package]] 3839 + name = "panic-message" 3840 + version = "0.3.0" 3841 + source = "registry+https://github.com/rust-lang/crates.io-index" 3842 + checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" 3843 + 3844 + [[package]] 3845 name = "parking" 3846 version = "2.2.1" 3847 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3950 "pest_meta", 3951 "proc-macro2", 3952 "quote", 3953 + "syn 2.0.106", 3954 ] 3955 3956 [[package]] ··· 3980 dependencies = [ 3981 "proc-macro2", 3982 "quote", 3983 + "syn 2.0.106", 3984 ] 3985 3986 [[package]] ··· 4098 "proc-macro-crate", 4099 "proc-macro2", 4100 "quote", 4101 + "syn 2.0.106", 4102 ] 4103 4104 [[package]] ··· 4133 source = "registry+https://github.com/rust-lang/crates.io-index" 4134 checksum = "41273b691a3d467a8c44d05506afba9f7b6bd56c9cdf80123de13fe52d7ec587" 4135 dependencies = [ 4136 + "darling 0.20.11", 4137 "http", 4138 "indexmap 2.11.4", 4139 "mime", ··· 4141 "proc-macro2", 4142 "quote", 4143 "regex", 4144 + "syn 2.0.106", 4145 "thiserror 2.0.16", 4146 ] 4147 ··· 4182 checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" 4183 dependencies = [ 4184 "proc-macro2", 4185 + "syn 2.0.106", 4186 ] 4187 4188 [[package]] ··· 4493 dependencies = [ 4494 "proc-macro2", 4495 "quote", 4496 + "syn 2.0.106", 4497 ] 4498 4499 [[package]] ··· 4644 4645 [[package]] 4646 name = "rsa" 4647 + version = "0.9.8" 4648 source = "registry+https://github.com/rust-lang/crates.io-index" 4649 + checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" 4650 dependencies = [ 4651 "const-oid", 4652 "digest", ··· 4875 "proc-macro2", 4876 "quote", 4877 "serde_derive_internals", 4878 + "syn 2.0.106", 4879 ] 4880 4881 [[package]] ··· 5008 dependencies = [ 5009 "proc-macro2", 5010 "quote", 5011 + "syn 2.0.106", 5012 ] 5013 5014 [[package]] ··· 5019 dependencies = [ 5020 "proc-macro2", 5021 "quote", 5022 + "syn 2.0.106", 5023 ] 5024 5025 [[package]] ··· 5073 5074 [[package]] 5075 name = "serde_spanned" 5076 + version = "0.6.9" 5077 + source = "registry+https://github.com/rust-lang/crates.io-index" 5078 + checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" 5079 + dependencies = [ 5080 + "serde", 5081 + ] 5082 + 5083 + [[package]] 5084 + name = "serde_spanned" 5085 version = "1.0.2" 5086 source = "registry+https://github.com/rust-lang/crates.io-index" 5087 checksum = "5417783452c2be558477e104686f7de5dae53dba813c28435e0e70f82d9b04ee" ··· 5098 "proc-macro2", 5099 "quote", 5100 "serde", 5101 + "syn 2.0.106", 5102 ] 5103 5104 [[package]] ··· 5137 source = "registry+https://github.com/rust-lang/crates.io-index" 5138 checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" 5139 dependencies = [ 5140 + "darling 0.20.11", 5141 "proc-macro2", 5142 "quote", 5143 + "syn 2.0.106", 5144 ] 5145 5146 [[package]] ··· 5232 5233 [[package]] 5234 name = "slab" 5235 + version = "0.4.9" 5236 source = "registry+https://github.com/rust-lang/crates.io-index" 5237 + checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" 5238 + dependencies = [ 5239 + "autocfg", 5240 + ] 5241 5242 [[package]] 5243 name = "slingshot" ··· 5326 ] 5327 5328 [[package]] 5329 name = "smallvec" 5330 version = "1.15.0" 5331 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5413 5414 [[package]] 5415 name = "strsim" 5416 + version = "0.10.0" 5417 + source = "registry+https://github.com/rust-lang/crates.io-index" 5418 + checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 5419 + 5420 + [[package]] 5421 + name = "strsim" 5422 version = "0.11.1" 5423 source = "registry+https://github.com/rust-lang/crates.io-index" 5424 checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" ··· 5431 5432 [[package]] 5433 name = "syn" 5434 + version = "1.0.109" 5435 + source = "registry+https://github.com/rust-lang/crates.io-index" 5436 + checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 5437 + dependencies = [ 5438 + "proc-macro2", 5439 + "quote", 5440 + "unicode-ident", 5441 + ] 5442 + 5443 + [[package]] 5444 + name = "syn" 5445 version = "2.0.106" 5446 source = "registry+https://github.com/rust-lang/crates.io-index" 5447 checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" ··· 5468 dependencies = [ 5469 "proc-macro2", 5470 "quote", 5471 + "syn 2.0.106", 5472 ] 5473 5474 [[package]] ··· 5554 dependencies = [ 5555 "proc-macro2", 5556 "quote", 5557 + "syn 2.0.106", 5558 ] 5559 5560 [[package]] ··· 5565 dependencies = [ 5566 "proc-macro2", 5567 "quote", 5568 + "syn 2.0.106", 5569 ] 5570 5571 [[package]] ··· 5690 dependencies = [ 5691 "proc-macro2", 5692 "quote", 5693 + "syn 2.0.106", 5694 ] 5695 5696 [[package]] ··· 5776 5777 [[package]] 5778 name = "toml" 5779 + version = "0.8.23" 5780 + source = "registry+https://github.com/rust-lang/crates.io-index" 5781 + checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" 5782 + dependencies = [ 5783 + "serde", 5784 + "serde_spanned 0.6.9", 5785 + "toml_datetime 0.6.11", 5786 + "toml_edit", 5787 + ] 5788 + 5789 + [[package]] 5790 + name = "toml" 5791 version = "0.9.7" 5792 source = "registry+https://github.com/rust-lang/crates.io-index" 5793 checksum = "00e5e5d9bf2475ac9d4f0d9edab68cc573dc2fd644b0dba36b0c30a92dd9eaa0" 5794 dependencies = [ 5795 "indexmap 2.11.4", 5796 "serde_core", 5797 + "serde_spanned 1.0.2", 5798 "toml_datetime 0.7.2", 5799 "toml_parser", 5800 "toml_writer", ··· 5806 version = "0.6.11" 5807 source = "registry+https://github.com/rust-lang/crates.io-index" 5808 checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" 5809 + dependencies = [ 5810 + "serde", 5811 + ] 5812 5813 [[package]] 5814 name = "toml_datetime" ··· 5826 checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" 5827 dependencies = [ 5828 "indexmap 2.11.4", 5829 + "serde", 5830 + "serde_spanned 0.6.9", 5831 "toml_datetime 0.6.11", 5832 + "toml_write", 5833 "winnow", 5834 ] 5835 ··· 5841 dependencies = [ 5842 "winnow", 5843 ] 5844 + 5845 + [[package]] 5846 + name = "toml_write" 5847 + version = "0.1.2" 5848 + source = "registry+https://github.com/rust-lang/crates.io-index" 5849 + checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" 5850 5851 [[package]] 5852 name = "toml_writer" ··· 5920 dependencies = [ 5921 "proc-macro2", 5922 "quote", 5923 + "syn 2.0.106", 5924 ] 5925 5926 [[package]] ··· 5970 dependencies = [ 5971 "proc-macro2", 5972 "quote", 5973 + "syn 2.0.106", 5974 ] 5975 5976 [[package]] ··· 6292 "log", 6293 "proc-macro2", 6294 "quote", 6295 + "syn 2.0.106", 6296 "wasm-bindgen-shared", 6297 ] 6298 ··· 6327 dependencies = [ 6328 "proc-macro2", 6329 "quote", 6330 + "syn 2.0.106", 6331 "wasm-bindgen-backend", 6332 "wasm-bindgen-shared", 6333 ] ··· 6480 dependencies = [ 6481 "proc-macro2", 6482 "quote", 6483 + "syn 2.0.106", 6484 ] 6485 6486 [[package]] ··· 6491 dependencies = [ 6492 "proc-macro2", 6493 "quote", 6494 + "syn 2.0.106", 6495 ] 6496 6497 [[package]] ··· 6797 dependencies = [ 6798 "proc-macro2", 6799 "quote", 6800 + "syn 2.0.106", 6801 "synstructure", 6802 ] 6803 ··· 6827 dependencies = [ 6828 "proc-macro2", 6829 "quote", 6830 + "syn 2.0.106", 6831 ] 6832 6833 [[package]] ··· 6838 dependencies = [ 6839 "proc-macro2", 6840 "quote", 6841 + "syn 2.0.106", 6842 ] 6843 6844 [[package]] ··· 6858 dependencies = [ 6859 "proc-macro2", 6860 "quote", 6861 + "syn 2.0.106", 6862 "synstructure", 6863 ] 6864 ··· 6880 dependencies = [ 6881 "proc-macro2", 6882 "quote", 6883 + "syn 2.0.106", 6884 ] 6885 6886 [[package]] ··· 6902 dependencies = [ 6903 "proc-macro2", 6904 "quote", 6905 + "syn 2.0.106", 6906 ] 6907 6908 [[package]]

-3

Cargo.toml

··· 13 "pocket", 14 "reflector", 15 ] 16 - 17 - [workspace.dependencies] 18 - clap = { version = "4.5.56", features = ["derive", "env"] }

··· 13 "pocket", 14 "reflector", 15 ]

+1 -1

constellation/Cargo.toml

··· 11 axum-extra = { version = "0.10.0", features = ["query", "typed-header"] } 12 axum-metrics = "0.2" 13 bincode = "1.3.3" 14 - clap = { workspace = true } 15 ctrlc = "3.4.5" 16 flume = { version = "0.11.1", default-features = false } 17 fs4 = { version = "0.12.0", features = ["sync"] }

··· 11 axum-extra = { version = "0.10.0", features = ["query", "typed-header"] } 12 axum-metrics = "0.2" 13 bincode = "1.3.3" 14 + clap = { version = "4.5.26", features = ["derive"] } 15 ctrlc = "3.4.5" 16 flume = { version = "0.11.1", default-features = false } 17 fs4 = { version = "0.12.0", features = ["sync"] }

+49 -54

constellation/src/bin/main.rs

··· 26 #[arg(long)] 27 #[clap(default_value = "0.0.0.0:6789")] 28 bind: SocketAddr, 29 - /// enable metrics collection and serving 30 - #[arg(long, action)] 31 collect_metrics: bool, 32 /// metrics server's listen address 33 - #[arg(long, requires("collect_metrics"))] 34 #[clap(default_value = "0.0.0.0:8765")] 35 bind_metrics: SocketAddr, 36 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: ··· 45 #[arg(short, long)] 46 #[clap(value_enum, default_value_t = StorageBackend::Memory)] 47 backend: StorageBackend, 48 - /// Serve a did:web document for this domain 49 - #[arg(long)] 50 - did_web_domain: Option<String>, 51 /// Initiate a database backup into this dir, if supported by the storage 52 #[arg(long)] 53 backup: Option<PathBuf>, ··· 106 MemStorage::new(), 107 fixture, 108 None, 109 - args.did_web_domain, 110 stream, 111 bind, 112 metrics_bind, ··· 142 rocks, 143 fixture, 144 args.data, 145 - args.did_web_domain, 146 stream, 147 bind, 148 metrics_bind, ··· 164 mut storage: impl LinkStorage, 165 fixture: Option<PathBuf>, 166 data_dir: Option<PathBuf>, 167 - did_web_domain: Option<String>, 168 stream: String, 169 bind: SocketAddr, 170 metrics_bind: SocketAddr, ··· 217 if collect_metrics { 218 install_metrics_server(metrics_bind)?; 219 } 220 - serve(readable, bind, did_web_domain, staying_alive).await 221 }) 222 .unwrap(); 223 stay_alive.drop_guard(); ··· 227 // only spawn monitoring thread if the metrics server is running 228 if collect_metrics { 229 s.spawn(move || { // monitor thread 230 - let stay_alive = stay_alive.clone(); 231 - let check_alive = stay_alive.clone(); 232 233 - let process_collector = metrics_process::Collector::default(); 234 - process_collector.describe(); 235 - metrics::describe_gauge!( 236 - "storage_available", 237 - metrics::Unit::Bytes, 238 - "available to be allocated" 239 - ); 240 - metrics::describe_gauge!( 241 - "storage_free", 242 - metrics::Unit::Bytes, 243 - "unused bytes in filesystem" 244 - ); 245 - if let Some(ref p) = data_dir { 246 - if let Err(e) = fs4::available_space(p) { 247 - eprintln!("fs4 failed to get available space. may not be supported here? space metrics may be absent. e: {e:?}"); 248 - } else { 249 - println!("disk space monitoring should work, watching at {p:?}"); 250 } 251 } 252 253 - 'monitor: loop { 254 - match readable.get_stats() { 255 - Ok(StorageStats { dids, targetables, linking_records, .. }) => { 256 - metrics::gauge!("storage.stats.dids").set(dids as f64); 257 - metrics::gauge!("storage.stats.targetables").set(targetables as f64); 258 - metrics::gauge!("storage.stats.linking_records").set(linking_records as f64); 259 - } 260 - Err(e) => eprintln!("failed to get stats: {e:?}"), 261 } 262 - 263 - process_collector.collect(); 264 - if let Some(ref p) = data_dir { 265 - if let Ok(avail) = fs4::available_space(p) { 266 - metrics::gauge!("storage.available").set(avail as f64); 267 - } 268 - if let Ok(free) = fs4::free_space(p) { 269 - metrics::gauge!("storage.free").set(free as f64); 270 - } 271 } 272 - let wait = time::Instant::now(); 273 - while wait.elapsed() < MONITOR_INTERVAL { 274 - thread::sleep(time::Duration::from_millis(100)); 275 - if check_alive.is_cancelled() { 276 - break 'monitor 277 - } 278 } 279 } 280 - stay_alive.drop_guard(); 281 - }); 282 } 283 }); 284

··· 26 #[arg(long)] 27 #[clap(default_value = "0.0.0.0:6789")] 28 bind: SocketAddr, 29 + /// optionally disable the metrics server 30 + #[arg(long)] 31 + #[clap(default_value_t = false)] 32 collect_metrics: bool, 33 /// metrics server's listen address 34 + #[arg(long)] 35 #[clap(default_value = "0.0.0.0:8765")] 36 bind_metrics: SocketAddr, 37 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: ··· 46 #[arg(short, long)] 47 #[clap(value_enum, default_value_t = StorageBackend::Memory)] 48 backend: StorageBackend, 49 /// Initiate a database backup into this dir, if supported by the storage 50 #[arg(long)] 51 backup: Option<PathBuf>, ··· 104 MemStorage::new(), 105 fixture, 106 None, 107 stream, 108 bind, 109 metrics_bind, ··· 139 rocks, 140 fixture, 141 args.data, 142 stream, 143 bind, 144 metrics_bind, ··· 160 mut storage: impl LinkStorage, 161 fixture: Option<PathBuf>, 162 data_dir: Option<PathBuf>, 163 stream: String, 164 bind: SocketAddr, 165 metrics_bind: SocketAddr, ··· 212 if collect_metrics { 213 install_metrics_server(metrics_bind)?; 214 } 215 + serve(readable, bind, staying_alive).await 216 }) 217 .unwrap(); 218 stay_alive.drop_guard(); ··· 222 // only spawn monitoring thread if the metrics server is running 223 if collect_metrics { 224 s.spawn(move || { // monitor thread 225 + let stay_alive = stay_alive.clone(); 226 + let check_alive = stay_alive.clone(); 227 228 + let process_collector = metrics_process::Collector::default(); 229 + process_collector.describe(); 230 + metrics::describe_gauge!( 231 + "storage_available", 232 + metrics::Unit::Bytes, 233 + "available to be allocated" 234 + ); 235 + metrics::describe_gauge!( 236 + "storage_free", 237 + metrics::Unit::Bytes, 238 + "unused bytes in filesystem" 239 + ); 240 + if let Some(ref p) = data_dir { 241 + if let Err(e) = fs4::available_space(p) { 242 + eprintln!("fs4 failed to get available space. may not be supported here? space metrics may be absent. e: {e:?}"); 243 + } else { 244 + println!("disk space monitoring should work, watching at {p:?}"); 245 + } 246 + } 247 + 248 + 'monitor: loop { 249 + match readable.get_stats() { 250 + Ok(StorageStats { dids, targetables, linking_records, .. }) => { 251 + metrics::gauge!("storage.stats.dids").set(dids as f64); 252 + metrics::gauge!("storage.stats.targetables").set(targetables as f64); 253 + metrics::gauge!("storage.stats.linking_records").set(linking_records as f64); 254 } 255 + Err(e) => eprintln!("failed to get stats: {e:?}"), 256 } 257 258 + process_collector.collect(); 259 + if let Some(ref p) = data_dir { 260 + if let Ok(avail) = fs4::available_space(p) { 261 + metrics::gauge!("storage.available").set(avail as f64); 262 } 263 + if let Ok(free) = fs4::free_space(p) { 264 + metrics::gauge!("storage.free").set(free as f64); 265 } 266 + } 267 + let wait = time::Instant::now(); 268 + while wait.elapsed() < MONITOR_INTERVAL { 269 + thread::sleep(time::Duration::from_millis(100)); 270 + if check_alive.is_cancelled() { 271 + break 'monitor 272 } 273 } 274 + } 275 + stay_alive.drop_guard(); 276 + }); 277 } 278 }); 279

+1 -1

constellation/src/lib.rs

··· 31 } 32 } 33 34 - #[derive(Debug, PartialEq, Serialize, Deserialize)] 35 pub struct RecordId { 36 pub did: Did, 37 pub collection: String,

··· 31 } 32 } 33 34 + #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] 35 pub struct RecordId { 36 pub did: Did, 37 pub collection: String,

+116 -71

constellation/src/server/mod.rs

··· 3 extract::{Query, Request}, 4 http::{self, header}, 5 middleware::{self, Next}, 6 - response::{IntoResponse, Json, Response}, 7 routing::get, 8 Router, 9 }; ··· 37 http::StatusCode::INTERNAL_SERVER_ERROR 38 } 39 40 - pub async fn serve<S: LinkReader, A: ToSocketAddrs>( 41 - store: S, 42 - addr: A, 43 - did_web_domain: Option<String>, 44 - stay_alive: CancellationToken, 45 - ) -> anyhow::Result<()> { 46 - let mut app = Router::new(); 47 - 48 - if let Some(d) = did_web_domain { 49 - app = app.route( 50 - "/.well-known/did.json", 51 - get({ 52 - let domain = d.clone(); 53 - move || did_web(domain) 54 - }), 55 - ) 56 - } 57 - 58 - let app = app 59 .route("/robots.txt", get(robots)) 60 .route( 61 "/", ··· 79 } 80 }), 81 ) 82 - // deprecated 83 .route( 84 "/links/count", 85 get({ ··· 92 }), 93 ) 94 .route( 95 - "/xrpc/blue.microcosm.links.getBacklinksCount", 96 get({ 97 let store = store.clone(); 98 move |accept, query| async { 99 - spawn_blocking(|| get_backlink_counts(accept, query, store)) 100 .await 101 .map_err(to500)? 102 } 103 }), 104 ) 105 .route( 106 - "/links/count/distinct-dids", 107 get({ 108 let store = store.clone(); 109 move |accept, query| async { 110 - spawn_blocking(|| count_distinct_dids(accept, query, store)) 111 .await 112 .map_err(to500)? 113 } ··· 217 User-agent: * 218 Disallow: /links 219 Disallow: /links/ 220 - Disallow: /xrpc/ 221 " 222 } 223 224 - async fn did_web(domain: String) -> impl IntoResponse { 225 - Json(serde_json::json!({ 226 - "id": format!("did:web:{domain}"), 227 - "service": [{ 228 - "id": "#constellation", 229 - "type": "ConstellationGraphService", 230 - "serviceEndpoint": format!("https://{domain}") 231 - }] 232 - })) 233 - } 234 - 235 #[derive(Template, Serialize, Deserialize)] 236 #[template(path = "hello.html.j2")] 237 struct HelloReponse { ··· 391 Ok(acceptable( 392 accept, 393 GetLinksCountResponse { 394 - total, 395 - query: (*query).clone(), 396 - }, 397 - )) 398 - } 399 - 400 - #[derive(Clone, Deserialize)] 401 - struct GetItemsCountQuery { 402 - subject: String, 403 - source: String, 404 - } 405 - #[derive(Template, Serialize)] 406 - #[template(path = "get-backlinks-count.html.j2")] 407 - struct GetItemsCountResponse { 408 - total: u64, 409 - #[serde(skip_serializing)] 410 - query: GetItemsCountQuery, 411 - } 412 - fn get_backlink_counts( 413 - accept: ExtractAccept, 414 - query: axum_extra::extract::Query<GetItemsCountQuery>, 415 - store: impl LinkReader, 416 - ) -> Result<impl IntoResponse, http::StatusCode> { 417 - let Some((collection, path)) = query.source.split_once(':') else { 418 - return Err(http::StatusCode::BAD_REQUEST); 419 - }; 420 - let path = format!(".{path}"); 421 - let total = store 422 - .get_count(&query.subject, collection, &path) 423 - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 424 - 425 - Ok(acceptable( 426 - accept, 427 - GetItemsCountResponse { 428 total, 429 query: (*query).clone(), 430 }, ··· 661 GetLinkItemsResponse { 662 total: paged.total, 663 linking_records: paged.items, 664 cursor, 665 query: (*query).clone(), 666 },

··· 3 extract::{Query, Request}, 4 http::{self, header}, 5 middleware::{self, Next}, 6 + response::{IntoResponse, Response}, 7 routing::get, 8 Router, 9 }; ··· 37 http::StatusCode::INTERNAL_SERVER_ERROR 38 } 39 40 + pub async fn serve<S, A>(store: S, addr: A, stay_alive: CancellationToken) -> anyhow::Result<()> 41 + where 42 + S: LinkReader, 43 + A: ToSocketAddrs, 44 + { 45 + let app = Router::new() 46 .route("/robots.txt", get(robots)) 47 .route( 48 "/", ··· 66 } 67 }), 68 ) 69 .route( 70 "/links/count", 71 get({ ··· 78 }), 79 ) 80 .route( 81 + "/links/count/distinct-dids", 82 get({ 83 let store = store.clone(); 84 move |accept, query| async { 85 + spawn_blocking(|| count_distinct_dids(accept, query, store)) 86 .await 87 .map_err(to500)? 88 } 89 }), 90 ) 91 .route( 92 + "/xrpc/blue.microcosm.links.getManyToMany", 93 get({ 94 let store = store.clone(); 95 move |accept, query| async { 96 + spawn_blocking(|| get_many_to_many(accept, query, store)) 97 .await 98 .map_err(to500)? 99 } ··· 203 User-agent: * 204 Disallow: /links 205 Disallow: /links/ 206 " 207 } 208 209 #[derive(Template, Serialize, Deserialize)] 210 #[template(path = "hello.html.j2")] 211 struct HelloReponse { ··· 365 Ok(acceptable( 366 accept, 367 GetLinksCountResponse { 368 total, 369 query: (*query).clone(), 370 }, ··· 601 GetLinkItemsResponse { 602 total: paged.total, 603 linking_records: paged.items, 604 + cursor, 605 + query: (*query).clone(), 606 + }, 607 + )) 608 + } 609 + 610 + #[derive(Clone, Deserialize)] 611 + #[serde(rename_all = "camelCase")] 612 + struct GetManyToManyItemsQuery { 613 + subject: String, 614 + source: String, 615 + /// path to the secondary link in the linking record 616 + path_to_other: String, 617 + /// filter to linking records (join of the m2m) by these DIDs 618 + #[serde(default)] 619 + did: Vec<String>, 620 + /// filter to specific secondary records 621 + #[serde(default)] 622 + other_subject: Vec<String>, 623 + cursor: Option<OpaqueApiCursor>, 624 + #[serde(default = "get_default_cursor_limit")] 625 + limit: u64, 626 + } 627 + #[derive(Debug, Serialize, Clone)] 628 + struct ManyToManyItem { 629 + link: RecordId, 630 + subject: String, 631 + } 632 + #[derive(Template, Serialize)] 633 + #[template(path = "get-many-to-many.html.j2")] 634 + struct GetManyToManyItemsResponse { 635 + items: Vec<ManyToManyItem>, 636 + cursor: Option<OpaqueApiCursor>, 637 + #[serde(skip_serializing)] 638 + query: GetManyToManyItemsQuery, 639 + } 640 + fn get_many_to_many( 641 + accept: ExtractAccept, 642 + query: axum_extra::extract::Query<GetManyToManyItemsQuery>, // supports multiple param occurrences 643 + store: impl LinkReader, 644 + ) -> Result<impl IntoResponse, http::StatusCode> { 645 + let after = query 646 + .cursor 647 + .clone() 648 + .map(|oc| ApiKeyedCursor::try_from(oc).map_err(|_| http::StatusCode::BAD_REQUEST)) 649 + .transpose()? 650 + .map(|c| c.next); 651 + 652 + let limit = query.limit; 653 + if limit > DEFAULT_CURSOR_LIMIT_MAX { 654 + return Err(http::StatusCode::BAD_REQUEST); 655 + } 656 + 657 + let filter_dids: HashSet<Did> = HashSet::from_iter( 658 + query 659 + .did 660 + .iter() 661 + .map(|d| d.trim()) 662 + .filter(|d| !d.is_empty()) 663 + .map(|d| Did(d.to_string())), 664 + ); 665 + 666 + let filter_other_subjects: HashSet<String> = HashSet::from_iter( 667 + query 668 + .other_subject 669 + .iter() 670 + .map(|s| s.trim().to_string()) 671 + .filter(|s| !s.is_empty()), 672 + ); 673 + 674 + let Some((collection, path)) = query.source.split_once(':') else { 675 + return Err(http::StatusCode::BAD_REQUEST); 676 + }; 677 + let path = format!(".{path}"); 678 + 679 + let path_to_other = format!(".{}", query.path_to_other); 680 + 681 + let paged = store 682 + .get_many_to_many( 683 + &query.subject, 684 + collection, 685 + &path, 686 + &path_to_other, 687 + limit, 688 + after, 689 + &filter_dids, 690 + &filter_other_subjects, 691 + ) 692 + .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; 693 + 694 + let cursor = paged.next.map(|next| ApiKeyedCursor { next }.into()); 695 + 696 + let items: Vec<ManyToManyItem> = paged 697 + .items 698 + .into_iter() 699 + .map(|(record_id, subject)| ManyToManyItem { 700 + link: record_id, 701 + subject, 702 + }) 703 + .collect(); 704 + 705 + Ok(acceptable( 706 + accept, 707 + GetManyToManyItemsResponse { 708 + items, 709 cursor, 710 query: (*query).clone(), 711 },

+98

constellation/src/storage/mem_store.rs

··· 234 .len() as u64) 235 } 236 237 fn get_links( 238 &self, 239 target: &str,

··· 234 .len() as u64) 235 } 236 237 + fn get_many_to_many( 238 + &self, 239 + target: &str, 240 + collection: &str, 241 + path: &str, 242 + path_to_other: &str, 243 + limit: u64, 244 + after: Option<String>, 245 + filter_dids: &HashSet<Did>, 246 + filter_to_targets: &HashSet<String>, 247 + ) -> Result<PagedOrderedCollection<(RecordId, String), String>> { 248 + let empty_res = Ok(PagedOrderedCollection { 249 + items: Vec::new(), 250 + next: None, 251 + }); 252 + 253 + // struct MemStorageData { 254 + // dids: HashMap<Did, bool>, 255 + // targets: HashMap<Target, HashMap<Source, Linkers>>, 256 + // links: HashMap<Did, HashMap<RepoId, Vec<(RecordPath, Target)>>>, 257 + // } 258 + let data = self.0.lock().unwrap(); 259 + 260 + let Some(sources) = data.targets.get(&Target::new(target)) else { 261 + return empty_res; 262 + }; 263 + let Some(linkers) = sources.get(&Source::new(collection, path)) else { 264 + return empty_res; 265 + }; 266 + let path_to_other = RecordPath::new(path_to_other); 267 + 268 + // Convert filter_to_targets to Target objects for comparison 269 + let filter_to_target_objs: HashSet<Target> = 270 + HashSet::from_iter(filter_to_targets.iter().map(|s| Target::new(s))); 271 + 272 + let mut grouped_links: HashMap<Target, Vec<RecordId>> = HashMap::new(); 273 + for (did, rkey) in linkers.iter().flatten().cloned() { 274 + // Filter by DID if filter is provided 275 + if !filter_dids.is_empty() && !filter_dids.contains(&did) { 276 + continue; 277 + } 278 + if let Some(fwd_target) = data 279 + .links 280 + .get(&did) 281 + .unwrap_or(&HashMap::new()) 282 + .get(&RepoId { 283 + collection: collection.to_string(), 284 + rkey: rkey.clone(), 285 + }) 286 + .unwrap_or(&Vec::new()) 287 + .iter() 288 + .find_map(|(path, target)| { 289 + if *path == path_to_other 290 + && (filter_to_target_objs.is_empty() 291 + || filter_to_target_objs.contains(target)) 292 + { 293 + Some(target) 294 + } else { 295 + None 296 + } 297 + }) 298 + { 299 + let record_ids = grouped_links.entry(fwd_target.clone()).or_default(); 300 + record_ids.push(RecordId { 301 + did, 302 + collection: collection.to_string(), 303 + rkey: rkey.0, 304 + }); 305 + } 306 + } 307 + 308 + let mut items = grouped_links 309 + .into_iter() 310 + .flat_map(|(target, records)| { 311 + records 312 + .iter() 313 + .map(move |r| (r.clone(), target.0.clone())) 314 + .collect::<Vec<_>>() 315 + }) 316 + .collect::<Vec<_>>(); 317 + 318 + items.sort_by(|a: &(RecordId, String), b| a.1.cmp(&b.1)); 319 + 320 + items = items 321 + .into_iter() 322 + .skip_while(|item| after.as_ref().map(|a| &item.1 <= a).unwrap_or(false)) 323 + .take(limit as usize) 324 + .collect(); 325 + 326 + let next = if items.len() as u64 >= limit { 327 + items.last().map(|item| item.1.clone()) 328 + } else { 329 + None 330 + }; 331 + 332 + Ok(PagedOrderedCollection { items, next }) 333 + } 334 + 335 fn get_links( 336 &self, 337 target: &str,

+234

constellation/src/storage/mod.rs

··· 135 fn get_all_record_counts(&self, _target: &str) 136 -> Result<HashMap<String, HashMap<String, u64>>>; 137 138 fn get_all_counts( 139 &self, 140 _target: &str, ··· 1670 next: None, 1671 } 1672 ); 1673 }); 1674 }

··· 135 fn get_all_record_counts(&self, _target: &str) 136 -> Result<HashMap<String, HashMap<String, u64>>>; 137 138 + fn get_many_to_many( 139 + &self, 140 + target: &str, 141 + collection: &str, 142 + path: &str, 143 + path_to_other: &str, 144 + limit: u64, 145 + after: Option<String>, 146 + filter_dids: &HashSet<Did>, 147 + filter_to_targets: &HashSet<String>, 148 + ) -> Result<PagedOrderedCollection<(RecordId, String), String>>; 149 + 150 fn get_all_counts( 151 &self, 152 _target: &str, ··· 1682 next: None, 1683 } 1684 ); 1685 + }); 1686 + 1687 + test_each_storage!(get_m2m_empty, |storage| { 1688 + assert_eq!( 1689 + storage.get_many_to_many( 1690 + "a.com", 1691 + "a.b.c", 1692 + ".d.e", 1693 + ".f.g", 1694 + 10, 1695 + None, 1696 + &HashSet::new(), 1697 + &HashSet::new(), 1698 + )?, 1699 + PagedOrderedCollection { 1700 + items: vec![], 1701 + next: None, 1702 + } 1703 + ); 1704 + }); 1705 + 1706 + test_each_storage!(get_m2m_single, |storage| { 1707 + storage.push( 1708 + &ActionableEvent::CreateLinks { 1709 + record_id: RecordId { 1710 + did: "did:plc:asdf".into(), 1711 + collection: "app.t.c".into(), 1712 + rkey: "asdf".into(), 1713 + }, 1714 + links: vec![ 1715 + CollectedLink { 1716 + target: Link::Uri("a.com".into()), 1717 + path: ".abc.uri".into(), 1718 + }, 1719 + CollectedLink { 1720 + target: Link::Uri("b.com".into()), 1721 + path: ".def.uri".into(), 1722 + }, 1723 + CollectedLink { 1724 + target: Link::Uri("b.com".into()), 1725 + path: ".ghi.uri".into(), 1726 + }, 1727 + ], 1728 + }, 1729 + 0, 1730 + )?; 1731 + assert_eq!( 1732 + storage.get_many_to_many( 1733 + "a.com", 1734 + "app.t.c", 1735 + ".abc.uri", 1736 + ".def.uri", 1737 + 10, 1738 + None, 1739 + &HashSet::new(), 1740 + &HashSet::new(), 1741 + )?, 1742 + PagedOrderedCollection { 1743 + items: vec![( 1744 + RecordId { 1745 + did: "did:plc:asdf".into(), 1746 + collection: "app.t.c".into(), 1747 + rkey: "asdf".into(), 1748 + }, 1749 + "b.com".to_string(), 1750 + )], 1751 + next: None, 1752 + } 1753 + ); 1754 + }); 1755 + 1756 + test_each_storage!(get_m2m_no_filters, |storage| { 1757 + storage.push( 1758 + &ActionableEvent::CreateLinks { 1759 + record_id: RecordId { 1760 + did: "did:plc:asdf".into(), 1761 + collection: "app.t.c".into(), 1762 + rkey: "asdf".into(), 1763 + }, 1764 + links: vec![ 1765 + CollectedLink { 1766 + target: Link::Uri("a.com".into()), 1767 + path: ".abc.uri".into(), 1768 + }, 1769 + CollectedLink { 1770 + target: Link::Uri("b.com".into()), 1771 + path: ".def.uri".into(), 1772 + }, 1773 + ], 1774 + }, 1775 + 0, 1776 + )?; 1777 + storage.push( 1778 + &ActionableEvent::CreateLinks { 1779 + record_id: RecordId { 1780 + did: "did:plc:asdf".into(), 1781 + collection: "app.t.c".into(), 1782 + rkey: "asdf2".into(), 1783 + }, 1784 + links: vec![ 1785 + CollectedLink { 1786 + target: Link::Uri("a.com".into()), 1787 + path: ".abc.uri".into(), 1788 + }, 1789 + CollectedLink { 1790 + target: Link::Uri("b.com".into()), 1791 + path: ".def.uri".into(), 1792 + }, 1793 + ], 1794 + }, 1795 + 1, 1796 + )?; 1797 + storage.push( 1798 + &ActionableEvent::CreateLinks { 1799 + record_id: RecordId { 1800 + did: "did:plc:fdsa".into(), 1801 + collection: "app.t.c".into(), 1802 + rkey: "fdsa".into(), 1803 + }, 1804 + links: vec![ 1805 + CollectedLink { 1806 + target: Link::Uri("a.com".into()), 1807 + path: ".abc.uri".into(), 1808 + }, 1809 + CollectedLink { 1810 + target: Link::Uri("c.com".into()), 1811 + path: ".def.uri".into(), 1812 + }, 1813 + ], 1814 + }, 1815 + 2, 1816 + )?; 1817 + storage.push( 1818 + &ActionableEvent::CreateLinks { 1819 + record_id: RecordId { 1820 + did: "did:plc:fdsa".into(), 1821 + collection: "app.t.c".into(), 1822 + rkey: "fdsa2".into(), 1823 + }, 1824 + links: vec![ 1825 + CollectedLink { 1826 + target: Link::Uri("a.com".into()), 1827 + path: ".abc.uri".into(), 1828 + }, 1829 + CollectedLink { 1830 + target: Link::Uri("c.com".into()), 1831 + path: ".def.uri".into(), 1832 + }, 1833 + ], 1834 + }, 1835 + 3, 1836 + )?; 1837 + 1838 + // Test without filters - should get all records as flat items 1839 + let result = storage.get_many_to_many( 1840 + "a.com", 1841 + "app.t.c", 1842 + ".abc.uri", 1843 + ".def.uri", 1844 + 10, 1845 + None, 1846 + &HashSet::new(), 1847 + &HashSet::new(), 1848 + )?; 1849 + assert_eq!(result.items.len(), 4); 1850 + assert_eq!(result.next, None); 1851 + // Check b.com items 1852 + let b_items: Vec<_> = result 1853 + .items 1854 + .iter() 1855 + .filter(|(_, subject)| subject == "b.com") 1856 + .collect(); 1857 + assert_eq!(b_items.len(), 2); 1858 + assert!(b_items 1859 + .iter() 1860 + .any(|(r, _)| r.did.0 == "did:plc:asdf" && r.rkey == "asdf")); 1861 + assert!(b_items 1862 + .iter() 1863 + .any(|(r, _)| r.did.0 == "did:plc:asdf" && r.rkey == "asdf2")); 1864 + // Check c.com items 1865 + let c_items: Vec<_> = result 1866 + .items 1867 + .iter() 1868 + .filter(|(_, subject)| subject == "c.com") 1869 + .collect(); 1870 + assert_eq!(c_items.len(), 2); 1871 + assert!(c_items 1872 + .iter() 1873 + .any(|(r, _)| r.did.0 == "did:plc:fdsa" && r.rkey == "fdsa")); 1874 + assert!(c_items 1875 + .iter() 1876 + .any(|(r, _)| r.did.0 == "did:plc:fdsa" && r.rkey == "fdsa2")); 1877 + 1878 + // Test with DID filter - should only get records from did:plc:fdsa 1879 + let result = storage.get_many_to_many( 1880 + "a.com", 1881 + "app.t.c", 1882 + ".abc.uri", 1883 + ".def.uri", 1884 + 10, 1885 + None, 1886 + &HashSet::from_iter([Did("did:plc:fdsa".to_string())]), 1887 + &HashSet::new(), 1888 + )?; 1889 + assert_eq!(result.items.len(), 2); 1890 + assert!(result.items.iter().all(|(_, subject)| subject == "c.com")); 1891 + assert!(result.items.iter().all(|(r, _)| r.did.0 == "did:plc:fdsa")); 1892 + 1893 + // Test with target filter - should only get records linking to b.com 1894 + let result = storage.get_many_to_many( 1895 + "a.com", 1896 + "app.t.c", 1897 + ".abc.uri", 1898 + ".def.uri", 1899 + 10, 1900 + None, 1901 + &HashSet::new(), 1902 + &HashSet::from_iter(["b.com".to_string()]), 1903 + )?; 1904 + assert_eq!(result.items.len(), 2); 1905 + assert!(result.items.iter().all(|(_, subject)| subject == "b.com")); 1906 + assert!(result.items.iter().all(|(r, _)| r.did.0 == "did:plc:asdf")); 1907 }); 1908 }

+145

constellation/src/storage/rocks_store.rs

··· 1122 } 1123 } 1124 1125 fn get_links( 1126 &self, 1127 target: &str,

··· 1122 } 1123 } 1124 1125 + fn get_many_to_many( 1126 + &self, 1127 + target: &str, 1128 + collection: &str, 1129 + path: &str, 1130 + path_to_other: &str, 1131 + limit: u64, 1132 + after: Option<String>, 1133 + filter_dids: &HashSet<Did>, 1134 + filter_to_targets: &HashSet<String>, 1135 + ) -> Result<PagedOrderedCollection<(RecordId, String), String>> { 1136 + let collection = Collection(collection.to_string()); 1137 + let path = RPath(path.to_string()); 1138 + 1139 + let target_key = TargetKey(Target(target.to_string()), collection.clone(), path); 1140 + 1141 + let after = after.map(|s| s.parse::<u64>().map(TargetId)).transpose()?; 1142 + 1143 + let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { 1144 + eprintln!("Target not found for {target_key:?}"); 1145 + return Ok(PagedOrderedCollection::empty()); 1146 + }; 1147 + 1148 + let filter_did_ids: HashMap<DidId, bool> = filter_dids 1149 + .iter() 1150 + .filter_map(|did| self.did_id_table.get_id_val(&self.db, did).transpose()) 1151 + .collect::<Result<Vec<DidIdValue>>>()? 1152 + .into_iter() 1153 + .map(|DidIdValue(id, active)| (id, active)) 1154 + .collect(); 1155 + 1156 + let mut filter_to_target_ids: HashSet<TargetId> = HashSet::new(); 1157 + for t in filter_to_targets { 1158 + for (_, target_id) in self.iter_targets_for_target(&Target(t.to_string())) { 1159 + filter_to_target_ids.insert(target_id); 1160 + } 1161 + } 1162 + 1163 + let linkers = self.get_target_linkers(&target_id)?; 1164 + 1165 + // we want to provide many to many which effectively means that we want to show a specific 1166 + // list of reords that is linked to by a specific number of linkers 1167 + let mut grouped_links: BTreeMap<TargetId, Vec<RecordId>> = BTreeMap::new(); 1168 + for (did_id, rkey) in linkers.0 { 1169 + if did_id.is_empty() { 1170 + continue; 1171 + } 1172 + 1173 + if !filter_did_ids.is_empty() && filter_did_ids.get(&did_id) != Some(&true) { 1174 + continue; 1175 + } 1176 + 1177 + // Make sure the current did is active 1178 + let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0)? else { 1179 + eprintln!("failed to look up did from did_id {did_id:?}"); 1180 + continue; 1181 + }; 1182 + let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did)? else { 1183 + eprintln!("failed to look up did_value from did_id {did_id:?}: {did:?}: data consistency bug?"); 1184 + continue; 1185 + }; 1186 + if !active { 1187 + continue; 1188 + } 1189 + 1190 + let record_link_key = RecordLinkKey(did_id, collection.clone(), rkey.clone()); 1191 + let Some(targets) = self.get_record_link_targets(&record_link_key)? else { 1192 + continue; 1193 + }; 1194 + 1195 + let Some(fwd_target) = targets 1196 + .0 1197 + .into_iter() 1198 + .filter_map(|RecordLinkTarget(rpath, target_id)| { 1199 + if rpath.0 == path_to_other 1200 + && (filter_to_target_ids.is_empty() 1201 + || filter_to_target_ids.contains(&target_id)) 1202 + { 1203 + Some(target_id) 1204 + } else { 1205 + None 1206 + } 1207 + }) 1208 + .take(1) 1209 + .next() 1210 + else { 1211 + eprintln!("no forward match found."); 1212 + continue; 1213 + }; 1214 + 1215 + // pagination logic mirrors what is currently done in get_many_to_many_counts 1216 + if after.as_ref().map(|a| fwd_target <= *a).unwrap_or(false) { 1217 + continue; 1218 + } 1219 + let page_is_full = grouped_links.len() as u64 >= limit; 1220 + if page_is_full { 1221 + let current_max = grouped_links.keys().next_back().unwrap(); 1222 + if fwd_target > *current_max { 1223 + continue; 1224 + } 1225 + } 1226 + 1227 + // pagination, continued 1228 + let mut should_evict = false; 1229 + let entry = grouped_links.entry(fwd_target.clone()).or_insert_with(|| { 1230 + should_evict = page_is_full; 1231 + Vec::default() 1232 + }); 1233 + entry.push(RecordId { 1234 + did, 1235 + collection: collection.0.clone(), 1236 + rkey: rkey.0, 1237 + }); 1238 + 1239 + if should_evict { 1240 + grouped_links.pop_last(); 1241 + } 1242 + } 1243 + 1244 + let mut items: Vec<(RecordId, String)> = Vec::with_capacity(grouped_links.len()); 1245 + for (fwd_target_id, records) in &grouped_links { 1246 + let Some(target_key) = self 1247 + .target_id_table 1248 + .get_val_from_id(&self.db, fwd_target_id.0)? 1249 + else { 1250 + eprintln!("failed to look up target from target_id {fwd_target_id:?}"); 1251 + continue; 1252 + }; 1253 + 1254 + let target_string = target_key.0 .0; 1255 + 1256 + records 1257 + .iter() 1258 + .for_each(|r| items.push((r.clone(), target_string.clone()))); 1259 + } 1260 + 1261 + let next = if grouped_links.len() as u64 >= limit { 1262 + grouped_links.keys().next_back().map(|k| format!("{}", k.0)) 1263 + } else { 1264 + None 1265 + }; 1266 + 1267 + Ok(PagedOrderedCollection { items, next }) 1268 + } 1269 + 1270 fn get_links( 1271 &self, 1272 target: &str,

-38

constellation/templates/get-backlinks-count.html.j2

··· 1 - {% extends "base.html.j2" %} 2 - {% import "try-it-macros.html.j2" as try_it %} 3 - 4 - {% block title %}Link Count{% endblock %} 5 - {% block description %}Count of {{ query.source }} records linking to {{ query.subject }}{% endblock %} 6 - 7 - {% block content %} 8 - 9 - {% call try_it::get_backlinks_count( 10 - query.subject, 11 - query.source, 12 - ) %} 13 - 14 - <h2> 15 - Total links to <code>{{ query.subject }}</code> 16 - {% if let Some(browseable_uri) = query.subject|to_browseable %} 17 - <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 18 - {% endif %} 19 - </h2> 20 - 21 - <p><strong><code>{{ total|human_number }}</code></strong> total links from <code>{{ query.source }}</code> to <code>{{ query.subject }}</code></p> 22 - 23 - <ul> 24 - <li> 25 - See direct backlinks at <code>/xrpc/blue.microcosm.links.getBacklinks</code>: 26 - <a href="/xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject|urlencode }}&source={{ query.source|urlencode }}"> 27 - /xrpc/blue.microcosm.links.getBacklinks?subject={{ query.subject }}&source={{ query.source }} 28 - </a> 29 - </li> 30 - <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.subject|urlencode }}">/links/all?target={{ query.subject }}</a></li> 31 - </ul> 32 - 33 - <details> 34 - <summary>Raw JSON response</summary> 35 - <pre class="code">{{ self|tojson }}</pre> 36 - </details> 37 - 38 - {% endblock %}

···

+58

constellation/templates/get-many-to-many.html.j2

···

··· 1 + {% extends "base.html.j2" %} 2 + {% import "try-it-macros.html.j2" as try_it %} 3 + 4 + {% block title %}Many-to-Many Links{% endblock %} 5 + {% block description %}All {{ query.source }} records with many-to-many links to {{ query.subject }} joining through {{ query.path_to_other }}{% endblock %} 6 + 7 + {% block content %} 8 + 9 + {% call try_it::get_many_to_many(query.subject, query.source, query.path_to_other, query.did, query.other_subject, query.limit) %} 10 + 11 + <h2> 12 + Many-to-many links to <code>{{ query.subject }}</code> 13 + {% if let Some(browseable_uri) = query.subject|to_browseable %} 14 + <small style="font-weight: normal; font-size: 1rem"><a href="{{ browseable_uri }}">browse record</a></small> 15 + {% endif %} 16 + </h2> 17 + 18 + <p><strong>Many-to-many links</strong> from <code>{{ query.source }}</code> joining through <code>{{ query.path_to_other }}</code></p> 19 + 20 + <ul> 21 + <li>See all links to this target at <code>/links/all</code>: <a href="/links/all?target={{ query.subject|urlencode }}">/links/all?target={{ query.subject }}</a></li> 22 + </ul> 23 + 24 + <h3>Many-to-many links, most recent first:</h3> 25 + 26 + {% for item in items %} 27 + <pre style="display: block; margin: 1em 2em" class="code"><strong>Subject</strong>: <a href="/links/all?target={{ item.subject|urlencode }}">{{ item.subject }}</a> 28 + <strong>DID</strong>: {{ item.link.did().0 }} 29 + <strong>Collection</strong>: {{ item.link.collection }} 30 + <strong>RKey</strong>: {{ item.link.rkey }} 31 + -> <a href="https://pdsls.dev/at://{{ item.link.did().0 }}/{{ item.link.collection }}/{{ item.link.rkey }}">browse record</a></pre> 32 + {% endfor %} 33 + 34 + {% if let Some(c) = cursor %} 35 + <form method="get" action="/xrpc/blue.microcosm.links.getManyToMany"> 36 + <input type="hidden" name="subject" value="{{ query.subject }}" /> 37 + <input type="hidden" name="source" value="{{ query.source }}" /> 38 + <input type="hidden" name="pathToOther" value="{{ query.path_to_other }}" /> 39 + {% for did in query.did %} 40 + <input type="hidden" name="did" value="{{ did }}" /> 41 + {% endfor %} 42 + {% for other in query.other_subject %} 43 + <input type="hidden" name="otherSubject" value="{{ other }}" /> 44 + {% endfor %} 45 + <input type="hidden" name="limit" value="{{ query.limit }}" /> 46 + <input type="hidden" name="cursor" value={{ c|json|safe }} /> 47 + <button type="submit">next page…</button> 48 + </form> 49 + {% else %} 50 + <button disabled><em>end of results</em></button> 51 + {% endif %} 52 + 53 + <details> 54 + <summary>Raw JSON response</summary> 55 + <pre class="code">{{ self|tojson }}</pre> 56 + </details> 57 + 58 + {% endblock %}

+20 -14

constellation/templates/hello.html.j2

··· 83 ) %} 84 85 86 <h3 class="route"><code>GET /links</code></h3> 87 88 <p>A list of records linking to a target.</p> ··· 121 {% call try_it::dids("at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r", "app.bsky.feed.like", ".subject.uri") %} 122 123 124 - <h3 class="route deprecated"><code>[deprecated] GET /links/count</code></h3> 125 126 <p>The total number of links pointing at a given target.</p> 127 ··· 137 <p style="margin-bottom: 0"><strong>Try it:</strong></p> 138 {% call try_it::links_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block", ".subject") %} 139 140 - <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getBacklinksCount</code></h3> 141 - 142 - <p>The total number of links pointing at a given target.</p> 143 - 144 - <h4>Query parameters:</h4> 145 - 146 - <ul> 147 - <li><code>subject</code>: required, must url-encode. The target being linked to. Example: <code>did:plc:vc7f4oafdgxsihk4cry2xpze</code> or <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></li> 148 - <li><code>source</code>: required. Collection and path specification for the primary link. Example: <code>app.bsky.feed.like:subject.uri</code></li> 149 - </ul> 150 - 151 - <p style="margin-bottom: 0"><strong>Try it:</strong></p> 152 - {% call try_it::get_backlinks_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block:subject") %} 153 154 <h3 class="route"><code>GET /links/count/distinct-dids</code></h3> 155

··· 83 ) %} 84 85 86 + <h3 class="route"><code>GET /xrpc/blue.microcosm.links.getManyToMany</code></h3> 87 + 88 + <p>A list of many-to-many join records linking to a target and a secondary target.</p> 89 + 90 + <h4>Query parameters:</h4> 91 + 92 + <ul> 93 + <li><p><code>subject</code>: required, must url-encode. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></p></li> 94 + <li><p><code>source</code>: required. Example: <code>app.bsky.feed.like:subject.uri</code></p></li> 95 + <li><p><code>pathToOther</code>: required. Path to the secondary link in the many-to-many record. Example: <code>otherThing.uri</code></p></li> 96 + <li><p><code>did</code>: optional, filter links to those from specific users. Include multiple times to filter by multiple users. Example: <code>did=did:plc:vc7f4oafdgxsihk4cry2xpze&did=did:plc:vc7f4oafdgxsihk4cry2xpze</code></p></li> 97 + <li><p><code>otherSubject</code>: optional, filter secondary links to specific subjects. Include multiple times to filter by multiple subjects. Example: <code>at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r</code></p></li> 98 + <li><p><code>limit</code>: optional. Default: <code>16</code>. Maximum: <code>100</code></p></li> 99 + </ul> 100 + 101 + <p style="margin-bottom: 0"><strong>Try it:</strong></p> 102 + {% call try_it::get_many_to_many("at://did:plc:a4pqq234yw7fqbddawjo7y35/app.bsky.feed.post/3m237ilwc372e", "app.bsky.feed.like:subject.uri", "reply.parent.uri", [""], [""], 16) %} 103 + 104 + 105 <h3 class="route"><code>GET /links</code></h3> 106 107 <p>A list of records linking to a target.</p> ··· 140 {% call try_it::dids("at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r", "app.bsky.feed.like", ".subject.uri") %} 141 142 143 + <h3 class="route"><code>GET /links/count</code></h3> 144 145 <p>The total number of links pointing at a given target.</p> 146 ··· 156 <p style="margin-bottom: 0"><strong>Try it:</strong></p> 157 {% call try_it::links_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block", ".subject") %} 158 159 160 <h3 class="route"><code>GET /links/count/distinct-dids</code></h3> 161

+30 -7

constellation/templates/try-it-macros.html.j2

··· 68 </script> 69 {% endmacro %} 70 71 {% macro links(target, collection, path, dids, limit) %} 72 <form method="get" action="/links"> 73 <pre class="code"><strong>GET</strong> /links ··· 104 </form> 105 {% endmacro %} 106 107 - {% macro get_backlinks_count(subject, source) %} 108 - <form method="get" action="/xrpc/blue.microcosm.links.getBacklinksCount"> 109 - <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getBacklinksCount 110 - ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="subject" /> 111 - &source= <input type="text" name="source" value="{{ source }}" placeholder="source" /> <button type="submit">get links count</button></pre> 112 - </form> 113 - {% endmacro %} 114 115 {% macro links_count(target, collection, path) %} 116 <form method="get" action="/links/count">

··· 68 </script> 69 {% endmacro %} 70 71 + {% macro get_many_to_many(subject, source, pathToOther, dids, otherSubjects, limit) %} 72 + <form method="get" action="/xrpc/blue.microcosm.links.getManyToMany"> 73 + <pre class="code"><strong>GET</strong> /xrpc/blue.microcosm.links.getManyToMany 74 + ?subject= <input type="text" name="subject" value="{{ subject }}" placeholder="at-uri, did, uri..." /> 75 + &source= <input type="text" name="source" value="{{ source }}" placeholder="app.bsky.feed.like:subject" /> 76 + &pathToOther= <input type="text" name="pathToOther" value="{{ pathToOther }}" placeholder="otherThing" /> 77 + {%- for did in dids %}{% if !did.is_empty() %} 78 + &did= <input type="text" name="did" value="{{ did }}" placeholder="did:plc:..." />{% endif %}{% endfor %} 79 + <span id="m2m-did-placeholder"></span> <button id="m2m-add-did">+ did filter</button> 80 + {%- for otherSubject in otherSubjects %}{% if !otherSubject.is_empty() %} 81 + &otherSubject= <input type="text" name="otherSubject" value="{{ otherSubject }}" placeholder="at-uri, did, uri..." />{% endif %}{% endfor %} 82 + <span id="m2m-other-placeholder"></span> <button id="m2m-add-other">+ other subject filter</button> 83 + &limit= <input type="number" name="limit" value="{{ limit }}" max="100" placeholder="100" /> <button type="submit">get many-to-many links</button></pre> 84 + </form> 85 + <script> 86 + const m2mAddDidButton = document.getElementById('m2m-add-did'); 87 + const m2mDidPlaceholder = document.getElementById('m2m-did-placeholder'); 88 + m2mAddDidButton.addEventListener('click', e => { 89 + e.preventDefault(); 90 + const i = document.createElement('input'); 91 + i.placeholder = 'did:plc:...'; 92 + i.name = "did" 93 + const p = m2mAddDidButton.parentNode; 94 + p.insertBefore(document.createTextNode('&did= '), m2mDidPlaceholder); 95 + p.insertBefore(i, m2mDidPlaceholder); 96 + p.insertBefore(document.createTextNode('\n '), m2mDidPlaceholder); 97 + }); 98 + </script> 99 + {% endmacro %} 100 + 101 {% macro links(target, collection, path, dids, limit) %} 102 <form method="get" action="/links"> 103 <pre class="code"><strong>GET</strong> /links ··· 134 </form> 135 {% endmacro %} 136 137 138 {% macro links_count(target, collection, path) %} 139 <form method="get" action="/links/count">

-38

lexicons/blue.microcosm/links/getBacklinksCount.json

··· 1 - { 2 - "lexicon": 1, 3 - "id": "blue.microcosm.links.getBacklinksCount", 4 - "defs": { 5 - "main": { 6 - "type": "query", 7 - "description": "count records that link to another record", 8 - "parameters": { 9 - "type": "params", 10 - "required": ["subject", "source"], 11 - "properties": { 12 - "subject": { 13 - "type": "string", 14 - "format": "uri", 15 - "description": "the primary target being linked to (at-uri, did, or uri)" 16 - }, 17 - "source": { 18 - "type": "string", 19 - "description": "collection and path specification for the primary link" 20 - } 21 - } 22 - }, 23 - "output": { 24 - "encoding": "application/json", 25 - "schema": { 26 - "type": "object", 27 - "required": ["total"], 28 - "properties": { 29 - "total": { 30 - "type": "integer", 31 - "description": "total number of matching links" 32 - } 33 - } 34 - } 35 - } 36 - } 37 - } 38 - }

···

+104

lexicons/blue.microcosm/links/getManyToMany.json

···

··· 1 + { 2 + "lexicon": 1, 3 + "id": "blue.microcosm.links.getManyToMany", 4 + "defs": { 5 + "main": { 6 + "type": "query", 7 + "description": "Get records that link to a primary subject along with the secondary subjects they also reference", 8 + "parameters": { 9 + "type": "params", 10 + "required": ["subject", "source", "pathToOther"], 11 + "properties": { 12 + "subject": { 13 + "type": "string", 14 + "format": "uri", 15 + "description": "the primary target being linked to (at-uri, did, or uri)" 16 + }, 17 + "source": { 18 + "type": "string", 19 + "description": "collection and path specification for the primary link (e.g., 'app.bsky.feed.like:subject.uri')" 20 + }, 21 + "pathToOther": { 22 + "type": "string", 23 + "description": "path to the secondary link in the many-to-many record (e.g., 'otherThing.uri')" 24 + }, 25 + "did": { 26 + "type": "array", 27 + "description": "filter links to those from specific users", 28 + "items": { 29 + "type": "string", 30 + "format": "did" 31 + } 32 + }, 33 + "otherSubject": { 34 + "type": "array", 35 + "description": "filter secondary links to specific subjects", 36 + "items": { 37 + "type": "string" 38 + } 39 + }, 40 + "limit": { 41 + "type": "integer", 42 + "minimum": 1, 43 + "maximum": 100, 44 + "default": 16, 45 + "description": "number of results to return" 46 + } 47 + } 48 + }, 49 + "output": { 50 + "encoding": "application/json", 51 + "schema": { 52 + "type": "object", 53 + "required": ["items"], 54 + "properties": { 55 + "items": { 56 + "type": "array", 57 + "items": { 58 + "type": "ref", 59 + "ref": "#item" 60 + } 61 + }, 62 + "cursor": { 63 + "type": "string" 64 + } 65 + } 66 + } 67 + } 68 + }, 69 + "item": { 70 + "type": "object", 71 + "required": ["link", "subject"], 72 + "properties": { 73 + "link": { 74 + "type": "ref", 75 + "ref": "#linkRecord" 76 + }, 77 + "subject": { 78 + "type": "string" 79 + } 80 + } 81 + }, 82 + "linkRecord": { 83 + "type": "object", 84 + "required": ["did", "collection", "rkey"], 85 + "description": "A record identifier consisting of a DID, collection, and record key", 86 + "properties": { 87 + "did": { 88 + "type": "string", 89 + "format": "did", 90 + "description": "the DID of the linking record's repository" 91 + }, 92 + "collection": { 93 + "type": "string", 94 + "format": "nsid", 95 + "description": "the collection of the linking record" 96 + }, 97 + "rkey": { 98 + "type": "string", 99 + "format": "record-key" 100 + } 101 + } 102 + } 103 + } 104 + }

+1 -1

pocket/Cargo.toml

··· 5 6 [dependencies] 7 atrium-crypto = "0.1.2" 8 - clap = { workspace = true } 9 jwt-compact = { git = "https://github.com/fatfingers23/jwt-compact.git", features = ["es256k"] } 10 log = "0.4.27" 11 poem = { version = "3.1.12", features = ["acme", "static-files"] }

··· 5 6 [dependencies] 7 atrium-crypto = "0.1.2" 8 + clap = { version = "4.5.41", features = ["derive"] } 9 jwt-compact = { git = "https://github.com/fatfingers23/jwt-compact.git", features = ["es256k"] } 10 log = "0.4.27" 11 poem = { version = "3.1.12", features = ["acme", "static-files"] }

+1 -1

quasar/Cargo.toml

··· 4 edition = "2024" 5 6 [dependencies] 7 - clap = { workspace = true } 8 fjall = "2.11.2"

··· 4 edition = "2024" 5 6 [dependencies] 7 + clap = { version = "4.5.46", features = ["derive"] } 8 fjall = "2.11.2"

+1 -1

readme.md

··· 10 Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://discord.gg/tcDfe4PGVB), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere. 11 12 > [!tip] 13 - > This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.org/microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! 14 15 16 🌌 [Constellation](./constellation/)

··· 10 Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://discord.gg/tcDfe4PGVB), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere. 11 12 > [!tip] 13 + > This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.sh/@microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! 14 15 16 🌌 [Constellation](./constellation/)

+1 -1

reflector/Cargo.toml

··· 4 edition = "2024" 5 6 [dependencies] 7 - clap = { workspace = true } 8 log = "0.4.28" 9 poem = "3.1.12" 10 serde = { version = "1.0.219", features = ["derive"] }

··· 4 edition = "2024" 5 6 [dependencies] 7 + clap = { version = "4.5.47", features = ["derive"] } 8 log = "0.4.28" 9 poem = "3.1.12" 10 serde = { version = "1.0.219", features = ["derive"] }

+2 -2

slingshot/Cargo.toml

··· 8 atrium-common = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 9 atrium-identity = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 10 atrium-oauth = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 11 - clap = { workspace = true } 12 ctrlc = "3.4.7" 13 - foyer = { version = "0.22.3", features = ["serde"] } 14 hickory-resolver = "0.25.2" 15 jetstream = { path = "../jetstream", features = ["metrics"] } 16 links = { path = "../links" }

··· 8 atrium-common = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 9 atrium-identity = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 10 atrium-oauth = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } 11 + clap = { version = "4.5.41", features = ["derive"] } 12 ctrlc = "3.4.7" 13 + foyer = { version = "0.18.0", features = ["serde"] } 14 hickory-resolver = "0.25.2" 15 jetstream = { path = "../jetstream", features = ["metrics"] } 16 links = { path = "../links" }

+2 -2

slingshot/api-description.md

··· 1 _A [gravitational slingshot](https://en.wikipedia.org/wiki/Gravity_assist) makes use of the gravity and relative movements of celestial bodies to accelerate a spacecraft and change its trajectory._ 2 3 4 - # Slingshot: edge record and identity cache 5 6 Applications in [ATProtocol](https://atproto.com/) store data in users' own [PDS](https://atproto.com/guides/self-hosting) (Personal Data Server), which are distributed across thousands of independently-run servers all over the world. Trying to access this data poses challenges for client applications: 7 ··· 90 - [🎇 Spacedust](https://spacedust.microcosm.blue/), a firehose of all social interactions 91 92 > [!success] 93 - > All microcosm projects are [open source](https://tangled.org/bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)!

··· 1 _A [gravitational slingshot](https://en.wikipedia.org/wiki/Gravity_assist) makes use of the gravity and relative movements of celestial bodies to accelerate a spacecraft and change its trajectory._ 2 3 4 + # Slingshot: edge record cache 5 6 Applications in [ATProtocol](https://atproto.com/) store data in users' own [PDS](https://atproto.com/guides/self-hosting) (Personal Data Server), which are distributed across thousands of independently-run servers all over the world. Trying to access this data poses challenges for client applications: 7 ··· 90 - [🎇 Spacedust](https://spacedust.microcosm.blue/), a firehose of all social interactions 91 92 > [!success] 93 + > All microcosm projects are [open source](https://tangled.sh/@bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)!

-34

slingshot/readme.md

··· 5 ```bash 6 RUST_LOG=info,slingshot=trace ulimit -n 4096 && RUST_LOG=info cargo run -- --jetstream us-east-1 --cache-dir ./foyer 7 ``` 8 - 9 - the identity cache uses a lot of files so you probably need to bump ulimit 10 - 11 - on macos: 12 - 13 - ```bash 14 - ulimit -n 4096 15 - ``` 16 - 17 - ## prod deploy 18 - 19 - you **must** setcap the binary to run it on apollo!!!! 20 - 21 - ```bash 22 - sudo setcap CAP_NET_BIND_SERVICE=+eip ../target/release/slingshot 23 - ``` 24 - 25 - then run with 26 - 27 - ```bash 28 - RUST_BACKTRACE=1 RUST_LOG=info,slingshot=trace /home/ubuntu/links/target/release/slingshot \ 29 - --jetstream wss://jetstream1.us-east.fire.hose.cam/subscribe \ 30 - --healthcheck https://hc-ping.com/[REDACTED] \ 31 - --cache-dir ./foyer \ 32 - --record-cache-memory-mb 2048 \ 33 - --record-cache-disk-gb 32 \ 34 - --identity-cache-memory-mb 1024 \ 35 - --identity-cache-disk-gb 8 \ 36 - --collect-metrics \ 37 - --acme-ipv6 \ 38 - --acme-domain slingshot.microcosm.blue \ 39 - --acme-contact phil@bad-example.com \ 40 - --acme-cache-path /home/ubuntu/certs 41 - ```

··· 5 ```bash 6 RUST_LOG=info,slingshot=trace ulimit -n 4096 && RUST_LOG=info cargo run -- --jetstream us-east-1 --cache-dir ./foyer 7 ```

+25 -41

slingshot/src/consumer.rs

··· 1 use crate::error::ConsumerError; 2 - use crate::{CachedRecord, Identity, IdentityKey}; 3 use foyer::HybridCache; 4 use jetstream::{ 5 DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, ··· 11 jetstream_endpoint: String, 12 cursor: Option<Cursor>, 13 no_zstd: bool, 14 - identity: Identity, 15 shutdown: CancellationToken, 16 cache: HybridCache<String, CachedRecord>, 17 ) -> Result<(), ConsumerError> { ··· 47 break; 48 }; 49 50 - match event.kind { 51 - EventKind::Commit => { 52 - let Some(ref mut commit) = event.commit else { 53 - log::warn!("consumer: commit event missing commit data, ignoring"); 54 - continue; 55 - }; 56 57 - // TODO: something a bit more robust 58 - let at_uri = format!( 59 - "at://{}/{}/{}", 60 - &*event.did, &*commit.collection, &*commit.rkey 61 - ); 62 63 - if commit.operation == CommitOp::Delete { 64 - cache.insert(at_uri, CachedRecord::Deleted); 65 - } else { 66 - let Some(record) = commit.record.take() else { 67 - log::warn!("consumer: commit insert or update missing record, ignoring"); 68 - continue; 69 - }; 70 - let Some(cid) = commit.cid.take() else { 71 - log::warn!("consumer: commit insert or update missing CID, ignoring"); 72 - continue; 73 - }; 74 75 - cache.insert(at_uri, CachedRecord::Found((cid, record).into())); 76 - } 77 - } 78 - EventKind::Identity => { 79 - let Some(ident) = event.identity else { 80 - log::warn!("consumer: identity event missing identity data, ignoring"); 81 - continue; 82 - }; 83 - if let Some(handle) = ident.handle { 84 - metrics::counter!("identity_handle_refresh_queued", "reason" => "identity event").increment(1); 85 - identity.queue_refresh(IdentityKey::Handle(handle)).await; 86 - } 87 - metrics::counter!("identity_did_refresh_queued", "reason" => "identity event") 88 - .increment(1); 89 - identity.queue_refresh(IdentityKey::Did(ident.did)).await; 90 - } 91 - EventKind::Account => {} // TODO: handle account events (esp hiding content on deactivate, clearing on delete) 92 } 93 } 94

··· 1 + use crate::CachedRecord; 2 use crate::error::ConsumerError; 3 use foyer::HybridCache; 4 use jetstream::{ 5 DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, ··· 11 jetstream_endpoint: String, 12 cursor: Option<Cursor>, 13 no_zstd: bool, 14 shutdown: CancellationToken, 15 cache: HybridCache<String, CachedRecord>, 16 ) -> Result<(), ConsumerError> { ··· 46 break; 47 }; 48 49 + if event.kind != EventKind::Commit { 50 + continue; 51 + } 52 + let Some(ref mut commit) = event.commit else { 53 + log::warn!("consumer: commit event missing commit data, ignoring"); 54 + continue; 55 + }; 56 57 + // TODO: something a bit more robust 58 + let at_uri = format!( 59 + "at://{}/{}/{}", 60 + &*event.did, &*commit.collection, &*commit.rkey 61 + ); 62 63 + if commit.operation == CommitOp::Delete { 64 + cache.insert(at_uri, CachedRecord::Deleted); 65 + } else { 66 + let Some(record) = commit.record.take() else { 67 + log::warn!("consumer: commit insert or update missing record, ignoring"); 68 + continue; 69 + }; 70 + let Some(cid) = commit.cid.take() else { 71 + log::warn!("consumer: commit insert or update missing CID, ignoring"); 72 + continue; 73 + }; 74 75 + cache.insert(at_uri, CachedRecord::Found((cid, record).into())); 76 } 77 } 78

+9 -21

slingshot/src/firehose_cache.rs

··· 1 use crate::CachedRecord; 2 - use foyer::{ 3 - BlockEngineConfig, DeviceBuilder, FsDeviceBuilder, HybridCache, HybridCacheBuilder, 4 - PsyncIoEngineConfig, 5 - }; 6 use std::path::Path; 7 8 pub async fn firehose_cache( 9 cache_dir: impl AsRef<Path>, 10 - memory_mb: usize, 11 - disk_gb: usize, 12 ) -> Result<HybridCache<String, CachedRecord>, String> { 13 - let device = FsDeviceBuilder::new(cache_dir) 14 - .with_capacity(disk_gb * 2_usize.pow(30)) 15 - .build() 16 - .map_err(|e| format!("foyer device setup error: {e}"))?; 17 - 18 - let engine = BlockEngineConfig::new(device).with_block_size(16 * 2_usize.pow(20)); // note: this does limit the max cached item size 19 - 20 let cache = HybridCacheBuilder::new() 21 .with_name("firehose") 22 - .memory(memory_mb * 2_usize.pow(20)) 23 - .with_weighter(|k: &String, v: &CachedRecord| { 24 - std::mem::size_of_val(k.as_str()) + v.weight() 25 - }) 26 - .storage() 27 - .with_io_engine_config(PsyncIoEngineConfig::default()) 28 - .with_engine_config(engine) 29 .build() 30 .await 31 .map_err(|e| format!("foyer setup error: {e:?}"))?; 32 - 33 Ok(cache) 34 }

··· 1 use crate::CachedRecord; 2 + use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; 3 use std::path::Path; 4 5 pub async fn firehose_cache( 6 cache_dir: impl AsRef<Path>, 7 ) -> Result<HybridCache<String, CachedRecord>, String> { 8 let cache = HybridCacheBuilder::new() 9 .with_name("firehose") 10 + .memory(64 * 2_usize.pow(20)) 11 + .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v)) 12 + .storage(Engine::large()) 13 + .with_device_options( 14 + DirectFsDeviceOptions::new(cache_dir) 15 + .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) 16 + .with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records 17 + ) 18 .build() 19 .await 20 .map_err(|e| format!("foyer setup error: {e:?}"))?; 21 Ok(cache) 22 }

+49 -111

slingshot/src/identity.rs

··· 11 /// 1. handle -> DID resolution: getRecord must accept a handle for `repo` param 12 /// 2. DID -> PDS resolution: so we know where to getRecord 13 /// 3. DID -> handle resolution: for bidirectional handle validation and in case we want to offer this 14 - use std::time::{Duration, Instant}; 15 use tokio::sync::Mutex; 16 use tokio_util::sync::CancellationToken; 17 ··· 26 handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig, DnsTxtResolver}, 27 }; 28 use atrium_oauth::DefaultHttpClient; // it's probably not worth bringing all of atrium_oauth for this but 29 - use foyer::{ 30 - BlockEngineConfig, DeviceBuilder, FsDeviceBuilder, HybridCache, HybridCacheBuilder, 31 - PsyncIoEngineConfig, 32 - }; 33 use serde::{Deserialize, Serialize}; 34 use time::UtcDateTime; 35 ··· 38 const MIN_NOT_FOUND_TTL: Duration = Duration::from_secs(60); 39 40 #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] 41 - pub enum IdentityKey { 42 Handle(Handle), 43 Did(Did), 44 } 45 46 - impl IdentityKey { 47 - fn weight(&self) -> usize { 48 - let s = match self { 49 - IdentityKey::Handle(h) => h.as_str(), 50 - IdentityKey::Did(d) => d.as_str(), 51 - }; 52 - std::mem::size_of::<Self>() + std::mem::size_of_val(s) 53 - } 54 - } 55 - 56 #[derive(Debug, Serialize, Deserialize)] 57 struct IdentityVal(UtcDateTime, IdentityData); 58 ··· 63 Doc(PartialMiniDoc), 64 } 65 66 - impl IdentityVal { 67 - fn weight(&self) -> usize { 68 - let wrapping = std::mem::size_of::<Self>(); 69 - let inner = match &self.1 { 70 - IdentityData::NotFound => 0, 71 - IdentityData::Did(d) => std::mem::size_of_val(d.as_str()), 72 - IdentityData::Doc(d) => { 73 - std::mem::size_of_val(d.unverified_handle.as_str()) 74 - + std::mem::size_of_val(d.pds.as_str()) 75 - + std::mem::size_of_val(d.signing_key.as_str()) 76 - } 77 - }; 78 - wrapping + inner 79 - } 80 - } 81 - 82 /// partial representation of a com.bad-example.identity mini atproto doc 83 /// 84 /// partial because the handle is not verified ··· 115 let Some(maybe_handle) = aka.strip_prefix("at://") else { 116 continue; 117 }; 118 - let Ok(valid_handle) = Handle::new(maybe_handle.to_lowercase()) else { 119 continue; 120 }; 121 unverified_handle = Some(valid_handle); ··· 186 /// multi-producer *single consumer* queue 187 refresh_queue: Arc<Mutex<RefreshQueue>>, 188 /// just a lock to ensure only one refresher (queue consumer) is running (to be improved with a better refresher) 189 - refresher_task: Arc<Mutex<()>>, 190 } 191 192 impl Identity { 193 - pub async fn new( 194 - cache_dir: impl AsRef<Path>, 195 - memory_mb: usize, 196 - disk_gb: usize, 197 - ) -> Result<Self, IdentityError> { 198 let http_client = Arc::new(DefaultHttpClient::default()); 199 let handle_resolver = AtprotoHandleResolver::new(AtprotoHandleResolverConfig { 200 dns_txt_resolver: HickoryDnsTxtResolver::new().unwrap(), ··· 204 plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), 205 http_client: http_client.clone(), 206 }); 207 - 208 - let device = FsDeviceBuilder::new(cache_dir) 209 - .with_capacity(disk_gb * 2_usize.pow(30)) 210 - .build()?; 211 - let engine = BlockEngineConfig::new(device).with_block_size(2_usize.pow(20)); // note: this does limit the max cached item size 212 213 let cache = HybridCacheBuilder::new() 214 .with_name("identity") 215 - .memory(memory_mb * 2_usize.pow(20)) 216 - .with_weighter(|k: &IdentityKey, v: &IdentityVal| k.weight() + v.weight()) 217 - .storage() 218 - .with_io_engine_config(PsyncIoEngineConfig::default()) 219 - .with_engine_config(engine) 220 .build() 221 .await?; 222 ··· 225 did_resolver: Arc::new(did_resolver), 226 cache, 227 refresh_queue: Default::default(), 228 - refresher_task: Default::default(), 229 }) 230 } 231 ··· 264 handle: &Handle, 265 ) -> Result<Option<Did>, IdentityError> { 266 let key = IdentityKey::Handle(handle.clone()); 267 - metrics::counter!("slingshot_get_handle").increment(1); 268 let entry = self 269 .cache 270 - .get_or_fetch(&key, { 271 let handle = handle.clone(); 272 let resolver = self.handle_resolver.clone(); 273 || async move { 274 - let t0 = Instant::now(); 275 - let (res, success) = match resolver.resolve(&handle).await { 276 - Ok(did) => ( 277 - Ok(IdentityVal(UtcDateTime::now(), IdentityData::Did(did))), 278 - "true", 279 - ), 280 - Err(atrium_identity::Error::NotFound) => ( 281 - Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)), 282 - "false", 283 - ), 284 - Err(other) => { 285 - log::debug!("other error resolving handle: {other:?}"); 286 - (Err(IdentityError::ResolutionFailed(other)), "false") 287 } 288 - }; 289 - metrics::histogram!("slingshot_fetch_handle", "success" => success) 290 - .record(t0.elapsed()); 291 - res 292 } 293 }) 294 .await?; ··· 302 } 303 IdentityData::NotFound => { 304 if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { 305 - metrics::counter!("identity_handle_refresh_queued", "reason" => "ttl", "found" => "false").increment(1); 306 self.queue_refresh(key).await; 307 } 308 Ok(None) 309 } 310 IdentityData::Did(did) => { 311 if (now - *last_fetch) >= MIN_TTL { 312 - metrics::counter!("identity_handle_refresh_queued", "reason" => "ttl", "found" => "true").increment(1); 313 self.queue_refresh(key).await; 314 } 315 Ok(Some(did.clone())) ··· 323 did: &Did, 324 ) -> Result<Option<PartialMiniDoc>, IdentityError> { 325 let key = IdentityKey::Did(did.clone()); 326 - metrics::counter!("slingshot_get_did_doc").increment(1); 327 let entry = self 328 .cache 329 - .get_or_fetch(&key, { 330 let did = did.clone(); 331 let resolver = self.did_resolver.clone(); 332 || async move { 333 - let t0 = Instant::now(); 334 - let (res, success) = match resolver.resolve(&did).await { 335 - Ok(did_doc) if did_doc.id != did.to_string() => ( 336 // TODO: fix in atrium: should verify id is did 337 - Err(IdentityError::BadDidDoc( 338 - "did doc's id did not match did".to_string(), 339 - )), 340 - "false", 341 - ), 342 - Ok(did_doc) => match did_doc.try_into() { 343 - Ok(mini_doc) => ( 344 - Ok(IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc))), 345 - "true", 346 - ), 347 - Err(e) => (Err(IdentityError::BadDidDoc(e)), "false"), 348 - }, 349 - Err(atrium_identity::Error::NotFound) => ( 350 - Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)), 351 - "false", 352 - ), 353 - Err(other) => (Err(IdentityError::ResolutionFailed(other)), "false"), 354 - }; 355 - metrics::histogram!("slingshot_fetch_did_doc", "success" => success) 356 - .record(t0.elapsed()); 357 - res 358 } 359 }) 360 .await?; ··· 368 } 369 IdentityData::NotFound => { 370 if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { 371 - metrics::counter!("identity_did_refresh_queued", "reason" => "ttl", "found" => "false").increment(1); 372 self.queue_refresh(key).await; 373 } 374 Ok(None) 375 } 376 IdentityData::Doc(mini_did) => { 377 if (now - *last_fetch) >= MIN_TTL { 378 - metrics::counter!("identity_did_refresh_queued", "reason" => "ttl", "found" => "true").increment(1); 379 self.queue_refresh(key).await; 380 } 381 Ok(Some(mini_did.clone())) ··· 386 /// put a refresh task on the queue 387 /// 388 /// this can be safely called from multiple concurrent tasks 389 - pub async fn queue_refresh(&self, key: IdentityKey) { 390 // todo: max queue size 391 let mut q = self.refresh_queue.lock().await; 392 if !q.items.contains(&key) { ··· 463 /// run the refresh queue consumer 464 pub async fn run_refresher(&self, shutdown: CancellationToken) -> Result<(), IdentityError> { 465 let _guard = self 466 - .refresher_task 467 .try_lock() 468 .expect("there to only be one refresher running"); 469 loop { ··· 485 log::trace!("refreshing handle {handle:?}"); 486 match self.handle_resolver.resolve(handle).await { 487 Ok(did) => { 488 - metrics::counter!("identity_handle_refresh", "success" => "true") 489 - .increment(1); 490 self.cache.insert( 491 task_key.clone(), 492 IdentityVal(UtcDateTime::now(), IdentityData::Did(did)), 493 ); 494 } 495 Err(atrium_identity::Error::NotFound) => { 496 - metrics::counter!("identity_handle_refresh", "success" => "false", "reason" => "not found").increment(1); 497 self.cache.insert( 498 task_key.clone(), 499 IdentityVal(UtcDateTime::now(), IdentityData::NotFound), 500 ); 501 } 502 Err(err) => { 503 - metrics::counter!("identity_handle_refresh", "success" => "false", "reason" => "other").increment(1); 504 log::warn!( 505 "failed to refresh handle: {err:?}. leaving stale (should we eventually do something?)" 506 ); ··· 515 Ok(did_doc) => { 516 // TODO: fix in atrium: should verify id is did 517 if did_doc.id != did.to_string() { 518 - metrics::counter!("identity_did_refresh", "success" => "false", "reason" => "wrong did").increment(1); 519 log::warn!( 520 "refreshed did doc failed: wrong did doc id. dropping refresh." 521 ); ··· 524 let mini_doc = match did_doc.try_into() { 525 Ok(md) => md, 526 Err(e) => { 527 - metrics::counter!("identity_did_refresh", "success" => "false", "reason" => "bad doc").increment(1); 528 log::warn!( 529 "converting mini doc failed: {e:?}. dropping refresh." 530 ); 531 continue; 532 } 533 }; 534 - metrics::counter!("identity_did_refresh", "success" => "true") 535 - .increment(1); 536 self.cache.insert( 537 task_key.clone(), 538 IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc)), 539 ); 540 } 541 Err(atrium_identity::Error::NotFound) => { 542 - metrics::counter!("identity_did_refresh", "success" => "false", "reason" => "not found").increment(1); 543 self.cache.insert( 544 task_key.clone(), 545 IdentityVal(UtcDateTime::now(), IdentityData::NotFound), 546 ); 547 } 548 Err(err) => { 549 - metrics::counter!("identity_did_refresh", "success" => "false", "reason" => "other").increment(1); 550 log::warn!( 551 "failed to refresh did doc: {err:?}. leaving stale (should we eventually do something?)" 552 );

··· 11 /// 1. handle -> DID resolution: getRecord must accept a handle for `repo` param 12 /// 2. DID -> PDS resolution: so we know where to getRecord 13 /// 3. DID -> handle resolution: for bidirectional handle validation and in case we want to offer this 14 + use std::time::Duration; 15 use tokio::sync::Mutex; 16 use tokio_util::sync::CancellationToken; 17 ··· 26 handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig, DnsTxtResolver}, 27 }; 28 use atrium_oauth::DefaultHttpClient; // it's probably not worth bringing all of atrium_oauth for this but 29 + use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; 30 use serde::{Deserialize, Serialize}; 31 use time::UtcDateTime; 32 ··· 35 const MIN_NOT_FOUND_TTL: Duration = Duration::from_secs(60); 36 37 #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] 38 + enum IdentityKey { 39 Handle(Handle), 40 Did(Did), 41 } 42 43 #[derive(Debug, Serialize, Deserialize)] 44 struct IdentityVal(UtcDateTime, IdentityData); 45 ··· 50 Doc(PartialMiniDoc), 51 } 52 53 /// partial representation of a com.bad-example.identity mini atproto doc 54 /// 55 /// partial because the handle is not verified ··· 86 let Some(maybe_handle) = aka.strip_prefix("at://") else { 87 continue; 88 }; 89 + let Ok(valid_handle) = Handle::new(maybe_handle.to_string()) else { 90 continue; 91 }; 92 unverified_handle = Some(valid_handle); ··· 157 /// multi-producer *single consumer* queue 158 refresh_queue: Arc<Mutex<RefreshQueue>>, 159 /// just a lock to ensure only one refresher (queue consumer) is running (to be improved with a better refresher) 160 + refresher: Arc<Mutex<()>>, 161 } 162 163 impl Identity { 164 + pub async fn new(cache_dir: impl AsRef<Path>) -> Result<Self, IdentityError> { 165 let http_client = Arc::new(DefaultHttpClient::default()); 166 let handle_resolver = AtprotoHandleResolver::new(AtprotoHandleResolverConfig { 167 dns_txt_resolver: HickoryDnsTxtResolver::new().unwrap(), ··· 171 plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), 172 http_client: http_client.clone(), 173 }); 174 175 let cache = HybridCacheBuilder::new() 176 .with_name("identity") 177 + .memory(16 * 2_usize.pow(20)) 178 + .with_weighter(|k, v| std::mem::size_of_val(k) + std::mem::size_of_val(v)) 179 + .storage(Engine::small()) 180 + .with_device_options( 181 + DirectFsDeviceOptions::new(cache_dir) 182 + .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) 183 + .with_file_size(2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records 184 + ) 185 .build() 186 .await?; 187 ··· 190 did_resolver: Arc::new(did_resolver), 191 cache, 192 refresh_queue: Default::default(), 193 + refresher: Default::default(), 194 }) 195 } 196 ··· 229 handle: &Handle, 230 ) -> Result<Option<Did>, IdentityError> { 231 let key = IdentityKey::Handle(handle.clone()); 232 let entry = self 233 .cache 234 + .fetch(key.clone(), { 235 let handle = handle.clone(); 236 let resolver = self.handle_resolver.clone(); 237 || async move { 238 + match resolver.resolve(&handle).await { 239 + Ok(did) => Ok(IdentityVal(UtcDateTime::now(), IdentityData::Did(did))), 240 + Err(atrium_identity::Error::NotFound) => { 241 + Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) 242 } 243 + Err(other) => Err(foyer::Error::Other(Box::new({ 244 + log::debug!("other error resolving handle: {other:?}"); 245 + IdentityError::ResolutionFailed(other) 246 + }))), 247 + } 248 } 249 }) 250 .await?; ··· 258 } 259 IdentityData::NotFound => { 260 if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { 261 self.queue_refresh(key).await; 262 } 263 Ok(None) 264 } 265 IdentityData::Did(did) => { 266 if (now - *last_fetch) >= MIN_TTL { 267 self.queue_refresh(key).await; 268 } 269 Ok(Some(did.clone())) ··· 277 did: &Did, 278 ) -> Result<Option<PartialMiniDoc>, IdentityError> { 279 let key = IdentityKey::Did(did.clone()); 280 let entry = self 281 .cache 282 + .fetch(key.clone(), { 283 let did = did.clone(); 284 let resolver = self.did_resolver.clone(); 285 || async move { 286 + match resolver.resolve(&did).await { 287 + Ok(did_doc) => { 288 // TODO: fix in atrium: should verify id is did 289 + if did_doc.id != did.to_string() { 290 + return Err(foyer::Error::other(Box::new( 291 + IdentityError::BadDidDoc( 292 + "did doc's id did not match did".to_string(), 293 + ), 294 + ))); 295 + } 296 + let mini_doc = did_doc.try_into().map_err(|e| { 297 + foyer::Error::Other(Box::new(IdentityError::BadDidDoc(e))) 298 + })?; 299 + Ok(IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc))) 300 + } 301 + Err(atrium_identity::Error::NotFound) => { 302 + Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) 303 + } 304 + Err(other) => Err(foyer::Error::Other(Box::new( 305 + IdentityError::ResolutionFailed(other), 306 + ))), 307 + } 308 } 309 }) 310 .await?; ··· 318 } 319 IdentityData::NotFound => { 320 if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { 321 self.queue_refresh(key).await; 322 } 323 Ok(None) 324 } 325 IdentityData::Doc(mini_did) => { 326 if (now - *last_fetch) >= MIN_TTL { 327 self.queue_refresh(key).await; 328 } 329 Ok(Some(mini_did.clone())) ··· 334 /// put a refresh task on the queue 335 /// 336 /// this can be safely called from multiple concurrent tasks 337 + async fn queue_refresh(&self, key: IdentityKey) { 338 // todo: max queue size 339 let mut q = self.refresh_queue.lock().await; 340 if !q.items.contains(&key) { ··· 411 /// run the refresh queue consumer 412 pub async fn run_refresher(&self, shutdown: CancellationToken) -> Result<(), IdentityError> { 413 let _guard = self 414 + .refresher 415 .try_lock() 416 .expect("there to only be one refresher running"); 417 loop { ··· 433 log::trace!("refreshing handle {handle:?}"); 434 match self.handle_resolver.resolve(handle).await { 435 Ok(did) => { 436 self.cache.insert( 437 task_key.clone(), 438 IdentityVal(UtcDateTime::now(), IdentityData::Did(did)), 439 ); 440 } 441 Err(atrium_identity::Error::NotFound) => { 442 self.cache.insert( 443 task_key.clone(), 444 IdentityVal(UtcDateTime::now(), IdentityData::NotFound), 445 ); 446 } 447 Err(err) => { 448 log::warn!( 449 "failed to refresh handle: {err:?}. leaving stale (should we eventually do something?)" 450 ); ··· 459 Ok(did_doc) => { 460 // TODO: fix in atrium: should verify id is did 461 if did_doc.id != did.to_string() { 462 log::warn!( 463 "refreshed did doc failed: wrong did doc id. dropping refresh." 464 ); ··· 467 let mini_doc = match did_doc.try_into() { 468 Ok(md) => md, 469 Err(e) => { 470 log::warn!( 471 "converting mini doc failed: {e:?}. dropping refresh." 472 ); 473 continue; 474 } 475 }; 476 self.cache.insert( 477 task_key.clone(), 478 IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc)), 479 ); 480 } 481 Err(atrium_identity::Error::NotFound) => { 482 self.cache.insert( 483 task_key.clone(), 484 IdentityVal(UtcDateTime::now(), IdentityData::NotFound), 485 ); 486 } 487 Err(err) => { 488 log::warn!( 489 "failed to refresh did doc: {err:?}. leaving stale (should we eventually do something?)" 490 );

+1 -1

slingshot/src/lib.rs

··· 9 pub use consumer::consume; 10 pub use firehose_cache::firehose_cache; 11 pub use healthcheck::healthcheck; 12 - pub use identity::{Identity, IdentityKey}; 13 pub use record::{CachedRecord, ErrorResponseObject, Repo}; 14 pub use server::serve;

··· 9 pub use consumer::consume; 10 pub use firehose_cache::firehose_cache; 11 pub use healthcheck::healthcheck; 12 + pub use identity::Identity; 13 pub use record::{CachedRecord, ErrorResponseObject, Repo}; 14 pub use server::serve;

+31 -86

slingshot/src/main.rs

··· 4 use slingshot::{ 5 Identity, Repo, consume, error::MainTaskError, firehose_cache, healthcheck, serve, 6 }; 7 - use std::net::SocketAddr; 8 use std::path::PathBuf; 9 10 use clap::Parser; ··· 16 struct Args { 17 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 18 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 19 - #[arg(long, env = "SLINGSHOT_JETSTREAM")] 20 jetstream: String, 21 /// don't request zstd-compressed jetstream events 22 /// 23 /// reduces CPU at the expense of more ingress bandwidth 24 - #[arg(long, action, env = "SLINGSHOT_JETSTREAM_NO_ZSTD")] 25 jetstream_no_zstd: bool, 26 /// where to keep disk caches 27 - #[arg(long, env = "SLINGSHOT_CACHE_DIR")] 28 cache_dir: PathBuf, 29 - /// where to listen for incomming requests 30 - /// 31 - /// cannot be used with acme -- if you need ipv6 see --acme-ipv6 32 - #[arg(long, env = "SLINGSHOT_BIND")] 33 - #[clap(default_value = "0.0.0.0:8080")] 34 - bind: SocketAddr, 35 - /// memory cache size in megabytes for records 36 - #[arg(long, env = "SLINGSHOT_RECORD_CACHE_MEMORY_MB")] 37 - #[clap(default_value_t = 64)] 38 - record_cache_memory_mb: usize, 39 - /// disk cache size in gigabytes for records 40 - #[arg(long, env = "SLINGSHOT_RECORD_CACHE_DISK_DB")] 41 - #[clap(default_value_t = 1)] 42 - record_cache_disk_gb: usize, 43 - /// memory cache size in megabytes for identities 44 - #[arg(long, env = "SLINGSHOT_IDENTITY_CACHE_MEMORY_MB")] 45 - #[clap(default_value_t = 64)] 46 - identity_cache_memory_mb: usize, 47 - /// disk cache size in gigabytes for identities 48 - #[arg(long, env = "SLINGSHOT_IDENTITY_CACHE_DISK_DB")] 49 - #[clap(default_value_t = 1)] 50 - identity_cache_disk_gb: usize, 51 /// the domain pointing to this server 52 /// 53 /// if present: 54 /// - a did:web document will be served at /.well-known/did.json 55 /// - an HTTPS certs will be automatically configured with Acme/letsencrypt 56 /// - TODO: a rate-limiter will be installed 57 - #[arg( 58 - long, 59 - conflicts_with("bind"), 60 - requires("acme_cache_path"), 61 - env = "SLINGSHOT_ACME_DOMAIN" 62 - )] 63 - acme_domain: Option<String>, 64 /// email address for letsencrypt contact 65 /// 66 /// recommended in production, i guess? 67 - #[arg(long, requires("acme_domain"), env = "SLINGSHOT_ACME_CONTACT")] 68 acme_contact: Option<String>, 69 /// a location to cache acme https certs 70 /// 71 - /// required when (and only used when) --acme-domain is specified. 72 /// 73 /// recommended in production, but mind the file permissions. 74 - #[arg(long, requires("acme_domain"), env = "SLINGSHOT_ACME_CACHE_PATH")] 75 - acme_cache_path: Option<PathBuf>, 76 - /// listen for ipv6 when using acme 77 - /// 78 - /// you must also configure the relevant DNS records for this to work 79 - #[arg(long, action, requires("acme_domain"), env = "SLINGSHOT_ACME_IPV6")] 80 - acme_ipv6: bool, 81 /// an web address to send healtcheck pings to every ~51s or so 82 - #[arg(long, env = "SLINGSHOT_HEALTHCHECK")] 83 healthcheck: Option<String>, 84 - /// enable metrics collection and serving 85 - #[arg(long, action, env = "SLINGSHOT_COLLECT_METRICS")] 86 - collect_metrics: bool, 87 - /// metrics server's listen address 88 - #[arg(long, requires("collect_metrics"), env = "SLINGSHOT_BIND_METRICS")] 89 - #[clap(default_value = "[::]:8765")] 90 - bind_metrics: std::net::SocketAddr, 91 } 92 93 #[tokio::main] ··· 101 102 let args = Args::parse(); 103 104 - if args.collect_metrics { 105 - log::trace!("installing metrics server..."); 106 - if let Err(e) = install_metrics_server(args.bind_metrics) { 107 - log::error!("failed to install metrics server: {e:?}"); 108 - } else { 109 - log::info!("metrics listening at http://{}", args.bind_metrics); 110 - } 111 } 112 113 std::fs::create_dir_all(&args.cache_dir).map_err(|e| { ··· 125 log::info!("cache dir ready at at {cache_dir:?}."); 126 127 log::info!("setting up firehose cache..."); 128 - let cache = firehose_cache( 129 - cache_dir.join("./firehose"), 130 - args.record_cache_memory_mb, 131 - args.record_cache_disk_gb, 132 - ) 133 - .await?; 134 log::info!("firehose cache ready."); 135 136 let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new(); 137 138 log::info!("starting identity service..."); 139 - let identity = Identity::new( 140 - cache_dir.join("./identity"), 141 - args.identity_cache_memory_mb, 142 - args.identity_cache_disk_gb, 143 - ) 144 - .await 145 - .map_err(|e| format!("identity setup failed: {e:?}"))?; 146 - 147 log::info!("identity service ready."); 148 let identity_refresher = identity.clone(); 149 let identity_shutdown = shutdown.clone(); ··· 154 155 let repo = Repo::new(identity.clone()); 156 157 - let identity_for_server = identity.clone(); 158 let server_shutdown = shutdown.clone(); 159 let server_cache_handle = cache.clone(); 160 - let bind = args.bind; 161 tasks.spawn(async move { 162 serve( 163 server_cache_handle, 164 - identity_for_server, 165 repo, 166 - args.acme_domain, 167 args.acme_contact, 168 - args.acme_cache_path, 169 - args.acme_ipv6, 170 server_shutdown, 171 - bind, 172 ) 173 .await?; 174 Ok(()) 175 }); 176 177 - let identity_refreshable = identity.clone(); 178 let consumer_shutdown = shutdown.clone(); 179 let consumer_cache = cache.clone(); 180 tasks.spawn(async move { ··· 182 args.jetstream, 183 None, 184 args.jetstream_no_zstd, 185 - identity_refreshable, 186 consumer_shutdown, 187 consumer_cache, 188 ) ··· 230 Ok(()) 231 } 232 233 - fn install_metrics_server( 234 - bind_metrics: std::net::SocketAddr, 235 - ) -> Result<(), metrics_exporter_prometheus::BuildError> { 236 log::info!("installing metrics server..."); 237 PrometheusBuilder::new() 238 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 239 .set_bucket_duration(std::time::Duration::from_secs(300))? 240 .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. 241 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 242 - .with_http_listener(bind_metrics) 243 .install()?; 244 log::info!( 245 - "metrics server installed! listening on http://{}", 246 - bind_metrics 247 ); 248 Ok(()) 249 }

··· 4 use slingshot::{ 5 Identity, Repo, consume, error::MainTaskError, firehose_cache, healthcheck, serve, 6 }; 7 use std::path::PathBuf; 8 9 use clap::Parser; ··· 15 struct Args { 16 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 17 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 18 + #[arg(long)] 19 jetstream: String, 20 /// don't request zstd-compressed jetstream events 21 /// 22 /// reduces CPU at the expense of more ingress bandwidth 23 + #[arg(long, action)] 24 jetstream_no_zstd: bool, 25 /// where to keep disk caches 26 + #[arg(long)] 27 cache_dir: PathBuf, 28 /// the domain pointing to this server 29 /// 30 /// if present: 31 /// - a did:web document will be served at /.well-known/did.json 32 /// - an HTTPS certs will be automatically configured with Acme/letsencrypt 33 /// - TODO: a rate-limiter will be installed 34 + #[arg(long)] 35 + domain: Option<String>, 36 /// email address for letsencrypt contact 37 /// 38 /// recommended in production, i guess? 39 + #[arg(long)] 40 acme_contact: Option<String>, 41 /// a location to cache acme https certs 42 /// 43 + /// only used if --host is specified. omitting requires re-requesting certs 44 + /// on every restart, and letsencrypt has rate limits that are easy to hit. 45 /// 46 /// recommended in production, but mind the file permissions. 47 + #[arg(long)] 48 + certs: Option<PathBuf>, 49 /// an web address to send healtcheck pings to every ~51s or so 50 + #[arg(long)] 51 healthcheck: Option<String>, 52 } 53 54 #[tokio::main] ··· 62 63 let args = Args::parse(); 64 65 + if let Err(e) = install_metrics_server() { 66 + log::error!("failed to install metrics server: {e:?}"); 67 + } else { 68 + log::info!("metrics listening at http://0.0.0.0:8765"); 69 } 70 71 std::fs::create_dir_all(&args.cache_dir).map_err(|e| { ··· 83 log::info!("cache dir ready at at {cache_dir:?}."); 84 85 log::info!("setting up firehose cache..."); 86 + let cache = firehose_cache(cache_dir.join("./firehose")).await?; 87 log::info!("firehose cache ready."); 88 89 let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new(); 90 91 log::info!("starting identity service..."); 92 + let identity = Identity::new(cache_dir.join("./identity")) 93 + .await 94 + .map_err(|e| format!("identity setup failed: {e:?}"))?; 95 log::info!("identity service ready."); 96 let identity_refresher = identity.clone(); 97 let identity_shutdown = shutdown.clone(); ··· 102 103 let repo = Repo::new(identity.clone()); 104 105 let server_shutdown = shutdown.clone(); 106 let server_cache_handle = cache.clone(); 107 tasks.spawn(async move { 108 serve( 109 server_cache_handle, 110 + identity, 111 repo, 112 + args.domain, 113 args.acme_contact, 114 + args.certs, 115 server_shutdown, 116 ) 117 .await?; 118 Ok(()) 119 }); 120 121 let consumer_shutdown = shutdown.clone(); 122 let consumer_cache = cache.clone(); 123 tasks.spawn(async move { ··· 125 args.jetstream, 126 None, 127 args.jetstream_no_zstd, 128 consumer_shutdown, 129 consumer_cache, 130 ) ··· 172 Ok(()) 173 } 174 175 + fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { 176 log::info!("installing metrics server..."); 177 + let host = [0, 0, 0, 0]; 178 + let port = 8765; 179 PrometheusBuilder::new() 180 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 181 .set_bucket_duration(std::time::Duration::from_secs(300))? 182 .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. 183 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 184 + .with_http_listener((host, port)) 185 .install()?; 186 log::info!( 187 + "metrics server installed! listening on http://{}.{}.{}.{}:{port}", 188 + host[0], 189 + host[1], 190 + host[2], 191 + host[3] 192 ); 193 Ok(()) 194 }

+1 -12

slingshot/src/record.rs

··· 42 Deleted, 43 } 44 45 - impl CachedRecord { 46 - pub(crate) fn weight(&self) -> usize { 47 - let wrapping = std::mem::size_of::<Self>(); 48 - let inner = match self { 49 - CachedRecord::Found(RawRecord { record, .. }) => std::mem::size_of_val(record.as_str()), 50 - _ => 0, 51 - }; 52 - wrapping + inner 53 - } 54 - } 55 - 56 //////// upstream record fetching 57 58 #[derive(Deserialize)] ··· 83 pub fn new(identity: Identity) -> Self { 84 let client = Client::builder() 85 .user_agent(format!( 86 - "microcosm slingshot v{} (contact: @bad-example.com)", 87 env!("CARGO_PKG_VERSION") 88 )) 89 .no_proxy()

··· 42 Deleted, 43 } 44 45 //////// upstream record fetching 46 47 #[derive(Deserialize)] ··· 72 pub fn new(identity: Identity) -> Self { 73 let client = Client::builder() 74 .user_agent(format!( 75 + "microcosm slingshot v{} (dev: @bad-example.com)", 76 env!("CARGO_PKG_VERSION") 77 )) 78 .no_proxy()

+28 -103

slingshot/src/server.rs

··· 9 use std::path::PathBuf; 10 use std::str::FromStr; 11 use std::sync::Arc; 12 - use std::time::Instant; 13 use tokio_util::sync::CancellationToken; 14 15 use poem::{ 16 - Endpoint, EndpointExt, IntoResponse, Route, Server, 17 endpoint::{StaticFileEndpoint, make_sync}, 18 http::Method, 19 listener::{ ··· 289 self.get_record_impl(repo, collection, rkey, cid).await 290 } 291 292 - /// blue.microcosm.repo.getRecordByUri 293 - /// 294 - /// alias of `com.bad-example.repo.getUriRecord` with intention to stabilize under this name 295 - #[oai( 296 - path = "/blue.microcosm.repo.getRecordByUri", 297 - method = "get", 298 - tag = "ApiTags::Custom" 299 - )] 300 - async fn get_record_by_uri( 301 - &self, 302 - /// The at-uri of the record 303 - /// 304 - /// The identifier can be a DID or an atproto handle, and the collection 305 - /// and rkey segments must be present. 306 - #[oai(example = "example_uri")] 307 - Query(at_uri): Query<String>, 308 - /// Optional: the CID of the version of the record. 309 - /// 310 - /// If not specified, then return the most recent version. 311 - /// 312 - /// > [!tip] 313 - /// > If specified and a newer version of the record exists, returns 404 not 314 - /// > found. That is: slingshot only retains the most recent version of a 315 - /// > record. 316 - Query(cid): Query<Option<String>>, 317 - ) -> GetRecordResponse { 318 - self.get_uri_record(Query(at_uri), Query(cid)).await 319 - } 320 - 321 /// com.bad-example.repo.getUriRecord 322 /// 323 /// Ergonomic complement to [`com.atproto.repo.getRecord`](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) ··· 405 #[oai(example = "example_handle")] 406 Query(handle): Query<String>, 407 ) -> JustDidResponse { 408 - let Ok(handle) = Handle::new(handle.to_lowercase()) else { 409 return JustDidResponse::BadRequest(xrpc_error("InvalidRequest", "not a valid handle")); 410 }; 411 ··· 443 })) 444 } 445 446 - /// blue.microcosm.identity.resolveMiniDoc 447 - /// 448 - /// alias of `com.bad-example.identity.resolveMiniDoc` with intention to stabilize under this name 449 - #[oai( 450 - path = "/blue.microcosm.identity.resolveMiniDoc", 451 - method = "get", 452 - tag = "ApiTags::Custom" 453 - )] 454 - async fn resolve_mini_doc( 455 - &self, 456 - /// Handle or DID to resolve 457 - #[oai(example = "example_handle")] 458 - Query(identifier): Query<String>, 459 - ) -> ResolveMiniIDResponse { 460 - self.resolve_mini_id(Query(identifier)).await 461 - } 462 - 463 /// com.bad-example.identity.resolveMiniDoc 464 /// 465 /// Like [com.atproto.identity.resolveIdentity](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-identity) ··· 483 let did = match Did::new(identifier.clone()) { 484 Ok(did) => did, 485 Err(_) => { 486 - let Ok(alleged_handle) = Handle::new(identifier.to_lowercase()) else { 487 return invalid("Identifier was not a valid DID or handle"); 488 }; 489 ··· 560 let did = match Did::new(repo.clone()) { 561 Ok(did) => did, 562 Err(_) => { 563 - let Ok(handle) = Handle::new(repo.to_lowercase()) else { 564 return GetRecordResponse::BadRequest(xrpc_error( 565 "InvalidRequest", 566 "Repo was not a valid DID or handle", ··· 610 611 let at_uri = format!("at://{}/{}/{}", &*did, &*collection, &*rkey); 612 613 - metrics::counter!("slingshot_get_record").increment(1); 614 let fr = self 615 .cache 616 - .get_or_fetch(&at_uri, { 617 let cid = cid.clone(); 618 let repo_api = self.repo.clone(); 619 || async move { 620 - let t0 = Instant::now(); 621 - let res = repo_api.get_record(&did, &collection, &rkey, &cid).await; 622 - let success = if res.is_ok() { "true" } else { "false" }; 623 - metrics::histogram!("slingshot_fetch_record", "success" => success) 624 - .record(t0.elapsed()); 625 - res 626 } 627 }) 628 .await; 629 630 let entry = match fr { 631 Ok(e) => e, 632 - Err(e) if e.kind() == foyer::ErrorKind::External => { 633 - let record_error = match e.source().map(|s| s.downcast_ref::<RecordError>()) { 634 - Some(Some(e)) => e, 635 - other => { 636 - if other.is_none() { 637 - log::error!("external error without a source. wat? {e}"); 638 - } else { 639 - log::error!("downcast to RecordError failed...? {e}"); 640 - } 641 return GetRecordResponse::ServerError(xrpc_error( 642 "ServerError", 643 "sorry, something went wrong", 644 )); 645 } 646 }; 647 - let RecordError::UpstreamBadRequest(ErrorResponseObject { 648 - ref error, 649 - ref message, 650 - }) = *record_error 651 else { 652 log::error!("RecordError getting cache entry, {record_error:?}"); 653 return GetRecordResponse::ServerError(xrpc_error( ··· 699 } 700 701 // TODO 702 // #[oai(path = "/com.atproto.identity.resolveDid", method = "get")] 703 // but these are both not specified to do bidirectional validation, which is what we want to offer 704 // com.atproto.identity.resolveIdentity seems right, but requires returning the full did-doc ··· 707 // handle -> verified did + pds url 708 // 709 // we could do horrible things and implement resolveIdentity with only a stripped-down fake did doc 710 - // but this will *definitely* cause problems probably 711 - // 712 - // resolveMiniDoc gets most of this well enough. 713 } 714 715 #[derive(Debug, Clone, Serialize)] ··· 743 make_sync(move |_| doc.clone()) 744 } 745 746 - #[allow(clippy::too_many_arguments)] 747 pub async fn serve( 748 cache: HybridCache<String, CachedRecord>, 749 identity: Identity, 750 repo: Repo, 751 - acme_domain: Option<String>, 752 acme_contact: Option<String>, 753 - acme_cache_path: Option<PathBuf>, 754 - acme_ipv6: bool, 755 shutdown: CancellationToken, 756 - bind: std::net::SocketAddr, 757 ) -> Result<(), ServerError> { 758 let repo = Arc::new(repo); 759 let api_service = OpenApiService::new( ··· 765 "Slingshot", 766 env!("CARGO_PKG_VERSION"), 767 ) 768 - .server(if let Some(ref h) = acme_domain { 769 format!("https://{h}") 770 } else { 771 - format!("http://{bind}") // yeah should probably fix this for reverse-proxy scenarios but it's ok for dev for now 772 }) 773 .url_prefix("/xrpc") 774 .contact( ··· 786 .nest("/openapi", api_service.spec_endpoint()) 787 .nest("/xrpc/", api_service); 788 789 - if let Some(domain) = acme_domain { 790 rustls::crypto::aws_lc_rs::default_provider() 791 .install_default() 792 .expect("alskfjalksdjf"); ··· 799 if let Some(contact) = acme_contact { 800 auto_cert = auto_cert.contact(contact); 801 } 802 - if let Some(cache_path) = acme_cache_path { 803 - auto_cert = auto_cert.cache_path(cache_path); 804 } 805 let auto_cert = auto_cert.build().map_err(ServerError::AcmeBuildError)?; 806 807 run( 808 - TcpListener::bind(if acme_ipv6 { "[::]:443" } else { "0.0.0.0:443" }).acme(auto_cert), 809 app, 810 shutdown, 811 ) 812 .await 813 } else { 814 - run(TcpListener::bind(bind), app, shutdown).await 815 } 816 } 817 ··· 827 .allow_credentials(false), 828 ) 829 .with(CatchPanic::new()) 830 - .around(request_counter) 831 .with(Tracing); 832 - 833 Server::new(listener) 834 .name("slingshot") 835 .run_with_graceful_shutdown(app, shutdown.cancelled(), None) ··· 837 .map_err(ServerError::ServerExited) 838 .inspect(|()| log::info!("server ended. goodbye.")) 839 } 840 - 841 - async fn request_counter<E: Endpoint>(next: E, req: poem::Request) -> poem::Result<poem::Response> { 842 - let t0 = std::time::Instant::now(); 843 - let method = req.method().to_string(); 844 - let path = req.uri().path().to_string(); 845 - let res = next.call(req).await?.into_response(); 846 - metrics::histogram!( 847 - "server_request", 848 - "endpoint" => format!("{method} {path}"), 849 - "status" => res.status().to_string(), 850 - ) 851 - .record(t0.elapsed()); 852 - Ok(res) 853 - }

··· 9 use std::path::PathBuf; 10 use std::str::FromStr; 11 use std::sync::Arc; 12 use tokio_util::sync::CancellationToken; 13 14 use poem::{ 15 + Endpoint, EndpointExt, Route, Server, 16 endpoint::{StaticFileEndpoint, make_sync}, 17 http::Method, 18 listener::{ ··· 288 self.get_record_impl(repo, collection, rkey, cid).await 289 } 290 291 /// com.bad-example.repo.getUriRecord 292 /// 293 /// Ergonomic complement to [`com.atproto.repo.getRecord`](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) ··· 375 #[oai(example = "example_handle")] 376 Query(handle): Query<String>, 377 ) -> JustDidResponse { 378 + let Ok(handle) = Handle::new(handle) else { 379 return JustDidResponse::BadRequest(xrpc_error("InvalidRequest", "not a valid handle")); 380 }; 381 ··· 413 })) 414 } 415 416 /// com.bad-example.identity.resolveMiniDoc 417 /// 418 /// Like [com.atproto.identity.resolveIdentity](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-identity) ··· 436 let did = match Did::new(identifier.clone()) { 437 Ok(did) => did, 438 Err(_) => { 439 + let Ok(alleged_handle) = Handle::new(identifier) else { 440 return invalid("Identifier was not a valid DID or handle"); 441 }; 442 ··· 513 let did = match Did::new(repo.clone()) { 514 Ok(did) => did, 515 Err(_) => { 516 + let Ok(handle) = Handle::new(repo) else { 517 return GetRecordResponse::BadRequest(xrpc_error( 518 "InvalidRequest", 519 "Repo was not a valid DID or handle", ··· 563 564 let at_uri = format!("at://{}/{}/{}", &*did, &*collection, &*rkey); 565 566 let fr = self 567 .cache 568 + .fetch(at_uri.clone(), { 569 let cid = cid.clone(); 570 let repo_api = self.repo.clone(); 571 || async move { 572 + repo_api 573 + .get_record(&did, &collection, &rkey, &cid) 574 + .await 575 + .map_err(|e| foyer::Error::Other(Box::new(e))) 576 } 577 }) 578 .await; 579 580 let entry = match fr { 581 Ok(e) => e, 582 + Err(foyer::Error::Other(e)) => { 583 + let record_error = match e.downcast::<RecordError>() { 584 + Ok(e) => e, 585 + Err(e) => { 586 + log::error!("error (foyer other) getting cache entry, {e:?}"); 587 return GetRecordResponse::ServerError(xrpc_error( 588 "ServerError", 589 "sorry, something went wrong", 590 )); 591 } 592 }; 593 + let RecordError::UpstreamBadRequest(ErrorResponseObject { error, message }) = 594 + *record_error 595 else { 596 log::error!("RecordError getting cache entry, {record_error:?}"); 597 return GetRecordResponse::ServerError(xrpc_error( ··· 643 } 644 645 // TODO 646 + // #[oai(path = "/com.atproto.identity.resolveHandle", method = "get")] 647 // #[oai(path = "/com.atproto.identity.resolveDid", method = "get")] 648 // but these are both not specified to do bidirectional validation, which is what we want to offer 649 // com.atproto.identity.resolveIdentity seems right, but requires returning the full did-doc ··· 652 // handle -> verified did + pds url 653 // 654 // we could do horrible things and implement resolveIdentity with only a stripped-down fake did doc 655 + // but this will *definitely* cause problems because eg. we're not currently storing pubkeys and 656 + // those are a little bit important 657 } 658 659 #[derive(Debug, Clone, Serialize)] ··· 687 make_sync(move |_| doc.clone()) 688 } 689 690 pub async fn serve( 691 cache: HybridCache<String, CachedRecord>, 692 identity: Identity, 693 repo: Repo, 694 + domain: Option<String>, 695 acme_contact: Option<String>, 696 + certs: Option<PathBuf>, 697 shutdown: CancellationToken, 698 ) -> Result<(), ServerError> { 699 let repo = Arc::new(repo); 700 let api_service = OpenApiService::new( ··· 706 "Slingshot", 707 env!("CARGO_PKG_VERSION"), 708 ) 709 + .server(if let Some(ref h) = domain { 710 format!("https://{h}") 711 } else { 712 + "http://localhost:3000".to_string() 713 }) 714 .url_prefix("/xrpc") 715 .contact( ··· 727 .nest("/openapi", api_service.spec_endpoint()) 728 .nest("/xrpc/", api_service); 729 730 + if let Some(domain) = domain { 731 rustls::crypto::aws_lc_rs::default_provider() 732 .install_default() 733 .expect("alskfjalksdjf"); ··· 740 if let Some(contact) = acme_contact { 741 auto_cert = auto_cert.contact(contact); 742 } 743 + if let Some(certs) = certs { 744 + auto_cert = auto_cert.cache_path(certs); 745 } 746 let auto_cert = auto_cert.build().map_err(ServerError::AcmeBuildError)?; 747 748 run( 749 + TcpListener::bind("0.0.0.0:443").acme(auto_cert), 750 app, 751 shutdown, 752 ) 753 .await 754 } else { 755 + run(TcpListener::bind("127.0.0.1:3000"), app, shutdown).await 756 } 757 } 758 ··· 768 .allow_credentials(false), 769 ) 770 .with(CatchPanic::new()) 771 .with(Tracing); 772 Server::new(listener) 773 .name("slingshot") 774 .run_with_graceful_shutdown(app, shutdown.cancelled(), None) ··· 776 .map_err(ServerError::ServerExited) 777 .inspect(|()| log::info!("server ended. goodbye.")) 778 }

+2 -2

slingshot/static/index.html

··· 43 <body> 44 <header class="custom-header scalar-app"> 45 <p> 46 - get atproto records and identities faster 47 </p> 48 <nav> 49 <b>a <a href="https://microcosm.blue">microcosm</a> project</b> 50 <a href="https://bsky.app/profile/microcosm.blue">@microcosm.blue</a> 51 - <a href="https://tangled.org/microcosm.blue/microcosm-rs">tangled</a> 52 </nav> 53 </header> 54

··· 43 <body> 44 <header class="custom-header scalar-app"> 45 <p> 46 + TODO: thing 47 </p> 48 <nav> 49 <b>a <a href="https://microcosm.blue">microcosm</a> project</b> 50 <a href="https://bsky.app/profile/microcosm.blue">@microcosm.blue</a> 51 + <a href="https://github.com/at-microcosm">github</a> 52 </nav> 53 </header> 54

+1 -1

spacedust/Cargo.toml

··· 5 6 [dependencies] 7 async-trait = "0.1.88" 8 - clap = { workspace = true } 9 ctrlc = "3.4.7" 10 dropshot = "0.16.2" 11 env_logger = "0.11.8"

··· 5 6 [dependencies] 7 async-trait = "0.1.88" 8 + clap = { version = "4.5.40", features = ["derive"] } 9 ctrlc = "3.4.7" 10 dropshot = "0.16.2" 11 env_logger = "0.11.8"

+17 -26

spacedust/src/main.rs

··· 16 struct Args { 17 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 18 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 19 - #[arg(long, env = "SPACEDUST_JETSTREAM")] 20 jetstream: String, 21 /// don't request zstd-compressed jetstream events 22 /// 23 /// reduces CPU at the expense of more ingress bandwidth 24 - #[arg(long, action, env = "SPACEDUST_JETSTREAM_NO_ZSTD")] 25 jetstream_no_zstd: bool, 26 - /// spacedust server's listen address 27 - #[arg(long, env = "SPACEDUST_BIND")] 28 - #[clap(default_value = "[::]:8080")] 29 - bind: std::net::SocketAddr, 30 - /// enable metrics collection and serving 31 - #[arg(long, action, env = "SPACEDUST_COLLECT_METRICS")] 32 - collect_metrics: bool, 33 - /// metrics server's listen address 34 - #[arg(long, requires("collect_metrics"), env = "SPACEDUST_BIND_METRICS")] 35 - #[clap(default_value = "[::]:8765")] 36 - bind_metrics: std::net::SocketAddr, 37 } 38 39 #[tokio::main] ··· 71 72 let args = Args::parse(); 73 74 - if args.collect_metrics { 75 - log::trace!("installing metrics server..."); 76 - if let Err(e) = install_metrics_server(args.bind_metrics) { 77 - log::error!("failed to install metrics server: {e:?}"); 78 - }; 79 - } 80 81 let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new(); 82 83 let server_shutdown = shutdown.clone(); 84 - let bind = args.bind; 85 tasks.spawn(async move { 86 - server::serve(b, d, server_shutdown, bind).await?; 87 Ok(()) 88 }); 89 ··· 137 Ok(()) 138 } 139 140 - fn install_metrics_server( 141 - bind: std::net::SocketAddr, 142 - ) -> Result<(), metrics_exporter_prometheus::BuildError> { 143 log::info!("installing metrics server..."); 144 PrometheusBuilder::new() 145 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 146 .set_bucket_duration(std::time::Duration::from_secs(300))? 147 .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. 148 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 149 - .with_http_listener(bind) 150 .install()?; 151 - log::info!("metrics server installed! listening on {bind}"); 152 Ok(()) 153 }

··· 16 struct Args { 17 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 18 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 19 + #[arg(long)] 20 jetstream: String, 21 /// don't request zstd-compressed jetstream events 22 /// 23 /// reduces CPU at the expense of more ingress bandwidth 24 + #[arg(long, action)] 25 jetstream_no_zstd: bool, 26 } 27 28 #[tokio::main] ··· 60 61 let args = Args::parse(); 62 63 + if let Err(e) = install_metrics_server() { 64 + log::error!("failed to install metrics server: {e:?}"); 65 + }; 66 67 let mut tasks: tokio::task::JoinSet<Result<(), MainTaskError>> = tokio::task::JoinSet::new(); 68 69 let server_shutdown = shutdown.clone(); 70 tasks.spawn(async move { 71 + server::serve(b, d, server_shutdown).await?; 72 Ok(()) 73 }); 74 ··· 122 Ok(()) 123 } 124 125 + fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { 126 log::info!("installing metrics server..."); 127 + let host = [0, 0, 0, 0]; 128 + let port = 8765; 129 PrometheusBuilder::new() 130 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 131 .set_bucket_duration(std::time::Duration::from_secs(300))? 132 .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. 133 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 134 + .with_http_listener((host, port)) 135 .install()?; 136 + log::info!( 137 + "metrics server installed! listening on http://{}.{}.{}.{}:{port}", 138 + host[0], 139 + host[1], 140 + host[2], 141 + host[3] 142 + ); 143 Ok(()) 144 }

+1 -2

spacedust/src/server.rs

··· 29 b: broadcast::Sender<Arc<ClientMessage>>, 30 d: broadcast::Sender<Arc<ClientMessage>>, 31 shutdown: CancellationToken, 32 - bind: std::net::SocketAddr, 33 ) -> Result<(), ServerError> { 34 let config_logging = ConfigLogging::StderrTerminal { 35 level: ConfigLoggingLevel::Info, ··· 73 74 let server = ServerBuilder::new(api, ctx, log) 75 .config(ConfigDropshot { 76 - bind_address: bind, 77 ..Default::default() 78 }) 79 .start()?;

··· 29 b: broadcast::Sender<Arc<ClientMessage>>, 30 d: broadcast::Sender<Arc<ClientMessage>>, 31 shutdown: CancellationToken, 32 ) -> Result<(), ServerError> { 33 let config_logging = ConfigLogging::StderrTerminal { 34 level: ConfigLoggingLevel::Info, ··· 72 73 let server = ServerBuilder::new(api, ctx, log) 74 .config(ConfigDropshot { 75 + bind_address: "0.0.0.0:9998".parse().unwrap(), 76 ..Default::default() 77 }) 78 .start()?;

+1 -1

ufos/Cargo.toml

··· 10 bincode = { version = "2.0.1", features = ["serde"] } 11 cardinality-estimator-safe = { version = "4.0.2", features = ["with_serde", "with_digest"] } 12 chrono = { version = "0.4.41", features = ["serde"] } 13 - clap = { workspace = true } 14 dropshot = "0.16.0" 15 env_logger = "0.11.7" 16 fjall = { git = "https://github.com/fjall-rs/fjall.git", rev = "fb229572bb7d1d6966a596994dc1708e47ec57d8", features = ["lz4"] }

··· 10 bincode = { version = "2.0.1", features = ["serde"] } 11 cardinality-estimator-safe = { version = "4.0.2", features = ["with_serde", "with_digest"] } 12 chrono = { version = "0.4.41", features = ["serde"] } 13 + clap = { version = "4.5.31", features = ["derive"] } 14 dropshot = "0.16.0" 15 env_logger = "0.11.7" 16 fjall = { git = "https://github.com/fjall-rs/fjall.git", rev = "fb229572bb7d1d6966a596994dc1708e47ec57d8", features = ["lz4"] }

+21 -27

ufos/src/main.rs

··· 26 struct Args { 27 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 28 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 29 - #[arg(long, env = "UFOS_JETSTREAM")] 30 jetstream: String, 31 /// allow changing jetstream endpoints 32 - #[arg(long, action, env = "UFOS_JETSTREAM_FORCE")] 33 jetstream_force: bool, 34 /// don't request zstd-compressed jetstream events 35 /// 36 /// reduces CPU at the expense of more ingress bandwidth 37 - #[arg(long, action, env = "UFOS_JETSTREAM_NO_ZSTD")] 38 jetstream_no_zstd: bool, 39 - /// ufos server's listen address 40 - #[arg(long, env = "UFOS_BIND")] 41 - #[clap(default_value = "0.0.0.0:9990")] 42 - bind: std::net::SocketAddr, 43 /// Location to store persist data to disk 44 - #[arg(long, env = "UFOS_DATA")] 45 data: PathBuf, 46 /// DEBUG: don't start the jetstream consumer or its write loop 47 - #[arg(long, action, env = "UFOS_PAUSE_WRITER")] 48 pause_writer: bool, 49 /// Adjust runtime settings like background task intervals for efficient backfill 50 - #[arg(long, action, env = "UFOS_BACKFILL_MODE")] 51 backfill: bool, 52 /// DEBUG: force the rw loop to fall behind by pausing it 53 /// todo: restore this 54 #[arg(long, action)] 55 pause_rw: bool, 56 /// reset the rollup cursor, scrape through missed things in the past (backfill) 57 - #[arg(long, action, env = "UFOS_REROLL")] 58 reroll: bool, 59 /// DEBUG: interpret jetstream as a file fixture 60 - #[arg(long, action, env = "UFOS_JETSTREAM_FIXTURE")] 61 jetstream_fixture: bool, 62 - /// enable metrics collection and serving 63 - #[arg(long, action, env = "UFOS_COLLECT_METRICS")] 64 - collect_metrics: bool, 65 - /// metrics server's listen address 66 - #[arg(long, env = "UFOS_BIND_METRICS")] 67 - #[clap(default_value = "0.0.0.0:8765")] 68 - bind_metrics: std::net::SocketAddr, 69 } 70 71 #[tokio::main] ··· 95 let mut consumer_tasks: JoinSet<anyhow::Result<()>> = JoinSet::new(); 96 97 println!("starting server with storage..."); 98 - let serving = server::serve(read_store.clone(), args.bind); 99 whatever_tasks.spawn(async move { 100 serving.await.map_err(|e| { 101 log::warn!("server ended: {e}"); ··· 148 Ok(()) 149 }); 150 151 - if args.collect_metrics { 152 - log::trace!("installing metrics server..."); 153 - install_metrics_server(args.bind_metrics)?; 154 - } 155 156 for (i, t) in consumer_tasks.join_all().await.iter().enumerate() { 157 log::warn!("task {i} done: {t:?}"); ··· 165 Ok(()) 166 } 167 168 - fn install_metrics_server(bind: std::net::SocketAddr) -> anyhow::Result<()> { 169 log::info!("installing metrics server..."); 170 PrometheusBuilder::new() 171 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 172 .set_bucket_duration(Duration::from_secs(60))? 173 .set_bucket_count(std::num::NonZero::new(10).unwrap()) // count * duration = 10 mins. stuff doesn't happen that fast here. 174 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 175 - .with_http_listener(bind) 176 .install()?; 177 - log::info!("metrics server installed! listening on {bind}"); 178 Ok(()) 179 } 180

··· 26 struct Args { 27 /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: 28 /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' 29 + #[arg(long)] 30 jetstream: String, 31 /// allow changing jetstream endpoints 32 + #[arg(long, action)] 33 jetstream_force: bool, 34 /// don't request zstd-compressed jetstream events 35 /// 36 /// reduces CPU at the expense of more ingress bandwidth 37 + #[arg(long, action)] 38 jetstream_no_zstd: bool, 39 /// Location to store persist data to disk 40 + #[arg(long)] 41 data: PathBuf, 42 /// DEBUG: don't start the jetstream consumer or its write loop 43 + #[arg(long, action)] 44 pause_writer: bool, 45 /// Adjust runtime settings like background task intervals for efficient backfill 46 + #[arg(long, action)] 47 backfill: bool, 48 /// DEBUG: force the rw loop to fall behind by pausing it 49 /// todo: restore this 50 #[arg(long, action)] 51 pause_rw: bool, 52 /// reset the rollup cursor, scrape through missed things in the past (backfill) 53 + #[arg(long, action)] 54 reroll: bool, 55 /// DEBUG: interpret jetstream as a file fixture 56 + #[arg(long, action)] 57 jetstream_fixture: bool, 58 } 59 60 #[tokio::main] ··· 84 let mut consumer_tasks: JoinSet<anyhow::Result<()>> = JoinSet::new(); 85 86 println!("starting server with storage..."); 87 + let serving = server::serve(read_store.clone()); 88 whatever_tasks.spawn(async move { 89 serving.await.map_err(|e| { 90 log::warn!("server ended: {e}"); ··· 137 Ok(()) 138 }); 139 140 + install_metrics_server()?; 141 142 for (i, t) in consumer_tasks.join_all().await.iter().enumerate() { 143 log::warn!("task {i} done: {t:?}"); ··· 151 Ok(()) 152 } 153 154 + fn install_metrics_server() -> anyhow::Result<()> { 155 log::info!("installing metrics server..."); 156 + let host = [0, 0, 0, 0]; 157 + let port = 8765; 158 PrometheusBuilder::new() 159 .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? 160 .set_bucket_duration(Duration::from_secs(60))? 161 .set_bucket_count(std::num::NonZero::new(10).unwrap()) // count * duration = 10 mins. stuff doesn't happen that fast here. 162 .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) 163 + .with_http_listener((host, port)) 164 .install()?; 165 + log::info!( 166 + "metrics server installed! listening on http://{}.{}.{}.{}:{port}", 167 + host[0], 168 + host[1], 169 + host[2], 170 + host[3] 171 + ); 172 Ok(()) 173 } 174

+2 -5

ufos/src/server/mod.rs

··· 716 .await 717 } 718 719 - pub async fn serve( 720 - storage: impl StoreReader + 'static, 721 - bind: std::net::SocketAddr, 722 - ) -> Result<(), String> { 723 describe_metrics(); 724 let log = ConfigLogging::StderrTerminal { 725 level: ConfigLoggingLevel::Warn, ··· 761 762 ServerBuilder::new(api, context, log) 763 .config(ConfigDropshot { 764 - bind_address: bind, 765 ..Default::default() 766 }) 767 .start()

··· 716 .await 717 } 718 719 + pub async fn serve(storage: impl StoreReader + 'static) -> Result<(), String> { 720 describe_metrics(); 721 let log = ConfigLogging::StderrTerminal { 722 level: ConfigLoggingLevel::Warn, ··· 758 759 ServerBuilder::new(api, context, log) 760 .config(ConfigDropshot { 761 + bind_address: "0.0.0.0:9999".parse().unwrap(), 762 ..Default::default() 763 }) 764 .start()

+1 -1

who-am-i/Cargo.toml

··· 11 axum = "0.8.4" 12 axum-extra = { version = "0.10.1", features = ["cookie-signed", "typed-header"] } 13 axum-template = { version = "3.0.0", features = ["handlebars"] } 14 - clap = { workspace = true } 15 ctrlc = "3.4.7" 16 dashmap = "6.1.0" 17 elliptic-curve = "0.13.8"

··· 11 axum = "0.8.4" 12 axum-extra = { version = "0.10.1", features = ["cookie-signed", "typed-header"] } 13 axum-template = { version = "3.0.0", features = ["handlebars"] } 14 + clap = { version = "4.5.40", features = ["derive", "env"] } 15 ctrlc = "3.4.7" 16 dashmap = "6.1.0" 17 elliptic-curve = "0.13.8"

History

8 rounds 13 comments

seoul.systems submitted #7 3w

diff interdiff

11 commits

expand

45e798b8

wip: m2m

f3144f8d

Add tests for new get_many_to_many query handler

21057eca

Fix get_m2m_empty test

4d686c35

Replace tuple with RecordsBySubject struct

4e358f1a

Fix conflicts after rebasing on main

bb107c03

Use record_id/subject tuple as return type for get_many_to_many

90220d0b

Fix get_many_to_many pagination with composite cursor

373d0744

Fix get_many_to_many_counts pagination with fetch N+1

46f5fae4

wip

6d22f695

Fix rocks-store to match mem-store composite cursor

c2d01284

Address feedback from fig

expand 0 comments

pull request successfully merged

seoul.systems submitted #6 4w

diff interdiff

10 commits

expand

45e798b8

wip: m2m

f3144f8d

Add tests for new get_many_to_many query handler

21057eca

Fix get_m2m_empty test

4d686c35

Replace tuple with RecordsBySubject struct

4e358f1a

Fix conflicts after rebasing on main

bb107c03

Use record_id/subject tuple as return type for get_many_to_many

90220d0b

Fix get_many_to_many pagination with composite cursor

373d0744

Fix get_many_to_many_counts pagination with fetch N+1

46f5fae4

wip

6d22f695

Fix rocks-store to match mem-store composite cursor

expand 0 comments

seoul.systems submitted #5 4w

diff interdiff

8 commits

expand

45e798b8

wip: m2m

f3144f8d

Add tests for new get_many_to_many query handler

21057eca

Fix get_m2m_empty test

4d686c35

Replace tuple with RecordsBySubject struct

4e358f1a

Fix conflicts after rebasing on main

bb107c03

Use record_id/subject tuple as return type for get_many_to_many

90220d0b

Fix get_many_to_many pagination with composite cursor

373d0744

Fix get_many_to_many_counts pagination with fetch N+1

expand 1 comment

seoul.systems 4w

Okay. I wrapped my head around the composite cursor you proposed and am working on refactoring both storage implementations towards that. I think I might re-submit another round tomorrow :)

seoul.systems submitted #4 4w

diff interdiff

6 commits

expand

45e798b8

wip: m2m

f3144f8d

Add tests for new get_many_to_many query handler

21057eca

Fix get_m2m_empty test

4d686c35

Replace tuple with RecordsBySubject struct

4e358f1a

Fix conflicts after rebasing on main

bb107c03

Use record_id/subject tuple as return type for get_many_to_many

expand 3 comments

seoul.systems 4w

Found a bug in how we handle some of the pagination logic in cases where the number of items and the user selected limit are identical to very close too each other (already working on a fix)

bad-example.com 4w

thanks for the rebase! i tried to write things in the tiny text box but ended up needing to make a diagram: https://bsky.app/profile/did:plc:hdhoaan3xa3jiuq4fg4mefid/post/3mejuq44twc2t

key thing is that where the focus of getManyToManyCounts was the other subject (aggregation was against that, so grouping happened with it),

i think the focus of disagreggated many-to-many is on the linking records themselves

to me that takes me toward a few things

i don't think we should need to group the links by target (does the current code build up the full aggregation on every requested page? we should be able to avoid doing that)
i think the order of the response should actually be based on the linking record itself (since we have a row in the output), not the other subject, unlike with the aggregated/count version. this means you get eg. list items in order they were added instead of the order of the listed things being created. (i haven't fully wrapped my head around the grouping/ordering code here yet)
since any linking record can have a path_to_other with multiple links, i think a composite cursor could work here:

a 2-tuple of (backlink_vec_idx, forward_vec_idx).

for normal cases where the many-to-many record points to exactly one other subject, it would just be advancing backlink_vec_idx like normal backlinks

for cases where the many-to-many record actually has multiple foward links at the given path_to_other, the second part of the tuple would track progress through that list

i think that allows us to hold the necessary state between calls without needing to reconstruct too much in memory each time?

(also it's hard to write in this tiny tiny textbox and have a sense of whether what i'm saying makes sense)

seoul.systems 4w

Interesting approach! I have to think through this for a bit to be honest. Maybe I tried to follow the existing counts implementation too closely

Having said that, I added a new composite cursor to fix a couple of bugs that would arrive when hitting a couple of possible edge-cases in the pagination logic. This affects both the new get-many-to-many endpoint as well as the existing get-many-to-many-counts endpoint. As the changes are split over two distinct commits things should be straightforward to review.

Your assumption is still correct in the sense that we do indeed have to build up the aggregation again for every request. I have to double-check the get-backlinks endpoint to get a better sense of where you're going at.

Finally, I agree that the interface here doesn't necessarily make the whole thing easier to understand, unfortunately

seoul.systems submitted #3 4w

interdiff

6 commits

expand

dbbef261

wip: m2m

52039af4

Add tests for new get_many_to_many query handler

da6fb97c

Fix get_m2m_empty test

9bd3a217

Replace tuple with RecordsBySubject struct

6f1351b9

Fix conflicts after rebasing on main

235fb506

Use record_id/subject tuple as return type for get_many_to_many

expand 2 comments

bad-example.com 4w

i think something got funky with a rebase or the way tangled is showing it -- some of my changes on main seem to be getting shown (reverted) in the diff.

i don't mind sorting it locally but will mostly get to it tomorrow, in case you want to see what's up before i do.

seoul.systems 4w

That's one on me, sorry! Rebased again on main and now everything seems fine

seoul.systems submitted #2 7w

diff interdiff

5 commits

expand

dbbef261

wip: m2m

52039af4

Add tests for new get_many_to_many query handler

da6fb97c

Fix get_m2m_empty test

9bd3a217

Replace tuple with RecordsBySubject struct

6f1351b9

Fix conflicts after rebasing on main

expand 5 comments

seoul.systems 7w

Rebased on main. As we discussed in the PR for the order query parameter, I didn't include this here as it's not a particular sensible fit.

bad-example.com 6w

i need to get into the code properly but my initial thought is that this endpoint should return a flat list of results, like

{
  "items": [
    {
      "link": { did, collection, rkey }, // the m2m link record
      "subject": "a.com"
    },
    {
      "link": { did, collection, rkey },
      "subject": "a.com"
    },
    {
      "link": { did, collection, rkey },
      "subject": "b.com"
    },
  ]
}

this will require a bit of tricks in the cursor to track pages across half-finished groups of links

(also this isn't an immediate change request, just getting it down for discussion!)

(and separately, i've also been wondering about moving more toward returning at-uris instead of broken-out did/collection/rkey objects. which isn't specifically about this PR, but if that happens then switching before releasing it is nice)

seoul.systems 6w

Hmm, I wonder how this would then work with the path_to_other parameter. Currently we have this nested grouping in order to show and disambiguate different relationships between different links.

For instance take the following query and it's results:

http://localhost:6789/xrpc/blue.microcosm.links.getManyToMany?subject=at://did:plc:2w45zyhuklwihpdc7oj3mi63/app.bsky.feed.post/3mdbbkuq6t32y&source=app.bsky.feed.post:reply.root.uri&pathToOther=reply.parent.uri&limit=16

This query asks: "Show me all posts in this thread, grouped by who they're responding to."

A flat list would just give us all the posts in the thread. The nested structure answers a richer question: who's talking to whom? Some posts are direct responses to the original article. Others are replies to other commenters, forming side conversations that branch off from the main thread.

The pathToOther grouping preserves that distinction. Without it, we'd lose the information about who's talking to whom.

{
  "linking_records": [
    {
      "subject": "at://did:plc:2w45zyhuklwihpdc7oj3mi63/app.bsky.feed.post/3mdbbkuq6t32y",
      "records": [
        {
          "did": "did:plc:lznqwrsbnyf6fdxohikqj6h3",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd27pja7s2y"
        },
        {
          "did": "did:plc:uffx77au6hoauuuumkbuvqdr",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd2tt5efc2a"
        },
        {
          "did": "did:plc:y7qyxzo7dns5m54dlq3youu3",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd2wtjxgc2d"
        },
        {
          "did": "did:plc:yaakslxyqydb76ybgkhrr4jk",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd35hyads22"
        },
        {
          "did": "did:plc:fia7w2kbnrdjwp6zvxywt7qv",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd37j3ldk2m"
        },
        {
          "did": "did:plc:xtecipifublblkomwau5x2ok",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd3dbtbz22n"
        },
        {
          "did": "did:plc:hl5lhiy2qr4nf5e4eefldvme",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd42hpw7c2e"
        },
        {
          "did": "did:plc:fgquypfh32pewivn3bcmzseb",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd46jteoc2m"
        }
      ]
    },
    {
      "subject": "at://did:plc:3rhjxwwui6wwfokh4at3q2dl/app.bsky.feed.post/3mdczc7c4gk2i",
      "records": [
        {
          "did": "did:plc:3rhjxwwui6wwfokh4at3q2dl",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdczt7cwhk2i"
        }
      ]
    },
    {
      "subject": "at://did:plc:6buibzhkqr4vkqu75ezr2uv2/app.bsky.feed.post/3mdby25hbbk2v",
      "records": [
        {
          "did": "did:plc:fgeie2bmzlmx37iglj3xbzuj",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd26ulf4k2j"
        }
      ]
    },
    {
      "subject": "at://did:plc:lwgvv5oqh5stzb6dxa5d7z3n/app.bsky.feed.post/3mdcxqbkkfk2i",
      "records": [
        {
          "did": "did:plc:hl5lhiy2qr4nf5e4eefldvme",
          "collection": "app.bsky.feed.post",
          "rkey": "3mdd45u56sk2e"
        }
      ]
    }
  ],
  "cursor": null
}

Correct me if I'm somehow wrong here!

Regarding returning at-uris: I think this might be a nice idea as users might be able to split these up when they feel the need to any way and it feels conceptually more complete. But, it might be easier to do this in a different PR over all existing XRPC endpoints. This would allow us to add this new endpoint already while working on the updated return values in the meantime. I'd like to avoid doing too much distinct stuff in one PR. :)

bad-example.com 6w

at-uris: totally fair, holding off for a follow-up.

flat list: i might have messed it up in my example but i think what i meant is actually equivalent to the grouped version: flattened, with the subject ("group by") included with every item in the flatted list.

clients can collect the flat list and group on subject to get back to your structured example, if they want.

my motivations are probably part sql-brain, part flat-list-enjoyer, and part cursor-related. i'm trying to disregard the first two, and i'm curious about your thoughts about how to handle the cursor:

with a flat list it's easy (from the client perspective at least) -- just keep chasing the cursor for as much of the data as you need. (cursors can happen in the middle of a subject)

with nested results grouped by subject it's less obvious to me. correct me if i'm wrong (need another block of time to actually get into the code) but i think the grouped item sub-list is unbounded size in the proposed code here? so cursors are only limiting the number of groups.

if we go with the grouped nested response, i think maybe we'd want something like:

a cursor at the end for fetching more groups, and
a cursor for each group-list that lets you fetch more items from just that group-list.

(i think this kind of nested paging is pretty neat!)

seoul.systems 6w

Interesting. Now that you mention it I feel I kinda get where you're going at!

I think the whole cursor thing, albeit possible for sure, is kinda creating more unnecessary complexity so I'll probably go with your suggestion.

It seems easier to create custom groupings on their own for most users (having more freedom is always great) and I think from an ergonomic perspective the two cursors might create more friction.

seoul.systems submitted #1 2mo

diff interdiff

4 commits

expand

a4177589

wip: m2m

81b7df55

Add tests for new get_many_to_many query handler

c6742917

Fix get_m2m_empty test

f0b16b50

Replace tuple with RecordsBySubject struct

expand 1 comment

seoul.systems 2mo

Added the missing lexicon entry for the new endpoint and changed the return type as well. Commented this wrongly at the other PR that I was working on. Sorry about that lol.

seoul.systems submitted #0 2mo

diff

3 commits

expand

a4177589

wip: m2m

81b7df55

Add tests for new get_many_to_many query handler

c6742917

Fix get_m2m_empty test

expand 1 comment

seoul.systems 2mo

I think the existing get_many_to_many_counts handler and the new get_many_to_many handler are similar enough that we might extract the bulk of their logic in a shared piece of logic. Maybe a method that takes the existing identical function parameters and a new additional callback parameter (that handles what we do with found matches, i.e. calculate counts or join URIs) might be one way to go for it.

I am not too sure yet though if this is indeed the right thing to do as the new shared implementation might be a bit complicated. But given the strong similarities between the two I think it's worth at least considering.