-110

.github/workflows/security.yml

··· 1 - # yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json 2 - 3 - name: Security 4 - 5 - on: 6 - push: 7 - branches: [main, develop] 8 - pull_request: 9 - branches: [main, develop] 10 - schedule: 11 - # Run security checks daily at 2 AM sunday 12 - - cron: "0 2 * * 0" 13 - 14 - jobs: 15 - codeql-analysis: 16 - name: CodeQL Analysis 17 - runs-on: ubuntu-latest 18 - permissions: 19 - actions: read 20 - contents: read 21 - security-events: write 22 - 23 - steps: 24 - - name: Checkout repository 25 - uses: actions/checkout@v4 26 - 27 - - name: Initialize CodeQL 28 - uses: github/codeql-action/init@v3 29 - with: 30 - languages: "javascript,typescript,rust" 31 - queries: security-extended,security-and-quality 32 - 33 - - name: Setup environment for all languages 34 - uses: ./.github/actions/setup 35 - with: 36 - setup-node: "true" 37 - setup-rust: "true" 38 - 39 - - name: Perform a full build for CodeQL 40 - run: | 41 - echo "Building Node.js projects..." 42 - pnpm build 43 - echo "Building Rust projects..." 44 - (cd services && cargo build --all-features) 45 - (cd apps/aqua && cargo build --all-features) 46 - 47 - - name: Perform CodeQL Analysis 48 - uses: github/codeql-action/analyze@v3 49 - 50 - docker-security-scan: 51 - name: Docker Security Scan 52 - runs-on: ubuntu-latest 53 - if: github.event_name == 'push' || github.event_name == 'schedule' 54 - strategy: 55 - matrix: 56 - service: [aqua, cadet] 57 - steps: 58 - - name: Checkout repository 59 - uses: actions/checkout@v4 60 - 61 - - name: Setup environment 62 - uses: ./.github/actions/setup 63 - with: 64 - setup-node: "true" 65 - lexicons-only-rust: "true" 66 - 67 - - name: Set up Docker Buildx 68 - uses: docker/setup-buildx-action@v3 69 - 70 - - name: Build Docker image 71 - uses: docker/build-push-action@v5 72 - with: 73 - context: . 74 - file: ${{ matrix.service == 'aqua' && './apps/aqua/Dockerfile' || './services/cadet/Dockerfile' }} 75 - load: true 76 - tags: ${{ matrix.service }}:latest 77 - cache-from: type=gha,scope=${{ matrix.service }} 78 - cache-to: type=gha,mode=max,scope=${{ matrix.service }} 79 - 80 - - name: Run Trivy vulnerability scanner 81 - uses: aquasecurity/trivy-action@master 82 - with: 83 - image-ref: "${{ matrix.service }}:latest" 84 - format: "sarif" 85 - output: "trivy-results-${{ matrix.service }}.sarif" 86 - severity: "CRITICAL,HIGH" 87 - exit-code: "1" 88 - 89 - - name: Upload Trivy scan results to GitHub Security tab 90 - uses: github/codeql-action/upload-sarif@v3 91 - if: always() 92 - with: 93 - sarif_file: "trivy-results-${{ matrix.service }}.sarif" 94 - 95 - secrets-scan: 96 - name: Secrets Scan 97 - runs-on: ubuntu-latest 98 - steps: 99 - - name: Checkout repository 100 - uses: actions/checkout@v4 101 - with: 102 - fetch-depth: 0 103 - 104 - - name: Run TruffleHog OSS 105 - uses: trufflesecurity/trufflehog@main 106 - with: 107 - path: ./ 108 - base: main 109 - head: HEAD 110 - extra_args: --debug --only-verified

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

+189 -42

Cargo.lock

··· 124 "atmst", 125 "atrium-api", 126 "axum", 127 - "base64", 128 "chrono", 129 "clap", 130 "dotenvy", ··· 247 "atrium-common", 248 "atrium-xrpc", 249 "chrono", 250 - "http", 251 "ipld-core", 252 "langtag", 253 "regex", ··· 280 source = "registry+https://github.com/rust-lang/crates.io-index" 281 checksum = "0216ad50ce34e9ff982e171c3659e65dedaa2ed5ac2994524debdc9a9647ffa8" 282 dependencies = [ 283 - "http", 284 "serde", 285 "serde_html_form", 286 "serde_json", ··· 328 "bytes", 329 "form_urlencoded", 330 "futures-util", 331 - "http", 332 "http-body", 333 "http-body-util", 334 "hyper", ··· 361 dependencies = [ 362 "bytes", 363 "futures-core", 364 - "http", 365 "http-body", 366 "http-body-util", 367 "mime", ··· 422 423 [[package]] 424 name = "base64" 0 0 0 0 0 0 425 version = "0.22.1" 426 source = "registry+https://github.com/rust-lang/crates.io-index" 427 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" ··· 568 "async-trait", 569 "atmst", 570 "atrium-api", 571 - "base64", 572 "chrono", 573 "cid 0.11.1", 574 "dotenvy", ··· 590 "sqlx", 591 "time", 592 "tokio", 593 - "tokio-tungstenite", 594 "tracing", 595 "tracing-subscriber", 596 "types", ··· 826 827 [[package]] 828 name = "core-foundation" 0 0 0 0 0 0 0 0 0 0 829 version = "0.10.1" 830 source = "registry+https://github.com/rust-lang/crates.io-index" 831 checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" ··· 1504 "fnv", 1505 "futures-core", 1506 "futures-sink", 1507 - "http", 1508 "indexmap", 1509 "slab", 1510 "tokio", ··· 1579 1580 [[package]] 1581 name = "http" 0 0 0 0 0 0 0 0 0 0 0 1582 version = "1.3.1" 1583 source = "registry+https://github.com/rust-lang/crates.io-index" 1584 checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" ··· 1595 checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" 1596 dependencies = [ 1597 "bytes", 1598 - "http", 1599 ] 1600 1601 [[package]] ··· 1606 dependencies = [ 1607 "bytes", 1608 "futures-core", 1609 - "http", 1610 "http-body", 1611 "pin-project-lite", 1612 ] ··· 1633 "futures-channel", 1634 "futures-util", 1635 "h2", 1636 - "http", 1637 "http-body", 1638 "httparse", 1639 "httpdate", ··· 1650 source = "registry+https://github.com/rust-lang/crates.io-index" 1651 checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" 1652 dependencies = [ 1653 - "http", 1654 "hyper", 1655 "hyper-util", 1656 - "rustls", 1657 - "rustls-native-certs", 1658 "rustls-pki-types", 1659 "tokio", 1660 - "tokio-rustls", 1661 "tower-service", 1662 "webpki-roots 1.0.2", 1663 ] ··· 1668 source = "registry+https://github.com/rust-lang/crates.io-index" 1669 checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" 1670 dependencies = [ 1671 - "base64", 1672 "bytes", 1673 "futures-channel", 1674 "futures-core", 1675 "futures-util", 1676 - "http", 1677 "http-body", 1678 "hyper", 1679 "ipnet", ··· 2225 source = "registry+https://github.com/rust-lang/crates.io-index" 2226 checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" 2227 dependencies = [ 2228 - "base64", 2229 "http-body-util", 2230 "hyper", 2231 "hyper-rustls", ··· 2319 "bytes", 2320 "encoding_rs", 2321 "futures-util", 2322 - "http", 2323 "httparse", 2324 "memchr", 2325 "mime", ··· 2793 "quinn-proto", 2794 "quinn-udp", 2795 "rustc-hash 2.1.1", 2796 - "rustls", 2797 "socket2 0.5.10", 2798 "thiserror 2.0.12", 2799 "tokio", ··· 2813 "rand 0.9.2", 2814 "ring", 2815 "rustc-hash 2.1.1", 2816 - "rustls", 2817 "rustls-pki-types", 2818 "slab", 2819 "thiserror 2.0.12", ··· 3024 checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" 3025 dependencies = [ 3026 "async-compression", 3027 - "base64", 3028 "bytes", 3029 "futures-core", 3030 "futures-util", 3031 - "http", 3032 "http-body", 3033 "http-body-util", 3034 "hyper", ··· 3039 "percent-encoding", 3040 "pin-project-lite", 3041 "quinn", 3042 - "rustls", 3043 "rustls-pki-types", 3044 "serde", 3045 "serde_json", 3046 "serde_urlencoded", 3047 "sync_wrapper", 3048 "tokio", 3049 - "tokio-rustls", 3050 "tokio-util", 3051 "tower", 3052 "tower-http", ··· 3095 [[package]] 3096 name = "rocketman" 3097 version = "0.2.3" 0 0 3098 dependencies = [ 3099 "anyhow", 3100 "async-trait", ··· 3102 "derive_builder", 3103 "flume", 3104 "futures-util", 3105 - "metrics 0.23.1", 3106 "rand 0.8.5", 3107 "serde", 3108 "serde_json", 3109 "tokio", 3110 - "tokio-tungstenite", 3111 "tracing", 3112 "tracing-subscriber", 3113 "url", ··· 3189 3190 [[package]] 3191 name = "rustls" 0 0 0 0 0 0 0 0 0 0 0 0 3192 version = "0.23.31" 3193 source = "registry+https://github.com/rust-lang/crates.io-index" 3194 checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" ··· 3197 "once_cell", 3198 "ring", 3199 "rustls-pki-types", 3200 - "rustls-webpki", 3201 "subtle", 3202 "zeroize", 3203 ] 3204 3205 [[package]] 3206 name = "rustls-native-certs" 0 0 0 0 0 0 0 0 0 0 0 0 3207 version = "0.8.1" 3208 source = "registry+https://github.com/rust-lang/crates.io-index" 3209 checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" ··· 3211 "openssl-probe", 3212 "rustls-pki-types", 3213 "schannel", 3214 - "security-framework", 0 0 0 0 0 0 0 0 0 3215 ] 3216 3217 [[package]] ··· 3222 dependencies = [ 3223 "web-time", 3224 "zeroize", 0 0 0 0 0 0 0 0 0 0 3225 ] 3226 3227 [[package]] ··· 3270 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 3271 3272 [[package]] 0 0 0 0 0 0 0 0 0 0 3273 name = "sec1" 3274 version = "0.7.3" 3275 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3285 3286 [[package]] 3287 name = "security-framework" 0 0 0 0 0 0 0 0 0 0 0 0 0 3288 version = "3.2.0" 3289 source = "registry+https://github.com/rust-lang/crates.io-index" 3290 checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" 3291 dependencies = [ 3292 "bitflags 2.9.1", 3293 - "core-foundation", 3294 "core-foundation-sys", 3295 "libc", 3296 "security-framework-sys", ··· 3565 source = "registry+https://github.com/rust-lang/crates.io-index" 3566 checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" 3567 dependencies = [ 3568 - "base64", 3569 "bytes", 3570 "crc", 3571 "crossbeam-queue", ··· 3582 "memchr", 3583 "once_cell", 3584 "percent-encoding", 3585 - "rustls", 3586 "serde", 3587 "serde_json", 3588 "sha2", ··· 3642 checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" 3643 dependencies = [ 3644 "atoi", 3645 - "base64", 3646 "bitflags 2.9.1", 3647 "byteorder", 3648 "bytes", ··· 3686 checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" 3687 dependencies = [ 3688 "atoi", 3689 - "base64", 3690 "bitflags 2.9.1", 3691 "byteorder", 3692 "crc", ··· 4041 4042 [[package]] 4043 name = "tokio-rustls" 0 0 0 0 0 0 0 0 0 0 4044 version = "0.26.2" 4045 source = "registry+https://github.com/rust-lang/crates.io-index" 4046 checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" 4047 dependencies = [ 4048 - "rustls", 4049 "tokio", 4050 ] 4051 ··· 4062 4063 [[package]] 4064 name = "tokio-tungstenite" 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4065 version = "0.24.0" 4066 source = "registry+https://github.com/rust-lang/crates.io-index" 4067 checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" 4068 dependencies = [ 4069 "futures-util", 4070 "log", 4071 - "rustls", 4072 "rustls-pki-types", 4073 "tokio", 4074 - "tokio-rustls", 4075 - "tungstenite", 4076 "webpki-roots 0.26.11", 4077 ] 4078 ··· 4140 "bitflags 2.9.1", 4141 "bytes", 4142 "futures-util", 4143 - "http", 4144 "http-body", 4145 "iri-string", 4146 "pin-project-lite", ··· 4242 4243 [[package]] 4244 name = "tungstenite" 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4245 version = "0.24.0" 4246 source = "registry+https://github.com/rust-lang/crates.io-index" 4247 checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" ··· 4249 "byteorder", 4250 "bytes", 4251 "data-encoding", 4252 - "http", 4253 "httparse", 4254 "log", 4255 "rand 0.8.5", 4256 - "rustls", 4257 "rustls-pki-types", 4258 "sha1", 4259 "thiserror 1.0.69", ··· 4273 "atrium-api", 4274 "atrium-xrpc", 4275 "chrono", 4276 - "http", 4277 "ipld-core", 4278 "langtag", 4279 "regex", ··· 4569 "js-sys", 4570 "wasm-bindgen", 4571 ] 0 0 0 0 0 0 4572 4573 [[package]] 4574 name = "webpki-roots"

··· 124 "atmst", 125 "atrium-api", 126 "axum", 127 + "base64 0.22.1", 128 "chrono", 129 "clap", 130 "dotenvy", ··· 247 "atrium-common", 248 "atrium-xrpc", 249 "chrono", 250 + "http 1.3.1", 251 "ipld-core", 252 "langtag", 253 "regex", ··· 280 source = "registry+https://github.com/rust-lang/crates.io-index" 281 checksum = "0216ad50ce34e9ff982e171c3659e65dedaa2ed5ac2994524debdc9a9647ffa8" 282 dependencies = [ 283 + "http 1.3.1", 284 "serde", 285 "serde_html_form", 286 "serde_json", ··· 328 "bytes", 329 "form_urlencoded", 330 "futures-util", 331 + "http 1.3.1", 332 "http-body", 333 "http-body-util", 334 "hyper", ··· 361 dependencies = [ 362 "bytes", 363 "futures-core", 364 + "http 1.3.1", 365 "http-body", 366 "http-body-util", 367 "mime", ··· 422 423 [[package]] 424 name = "base64" 425 + version = "0.21.7" 426 + source = "registry+https://github.com/rust-lang/crates.io-index" 427 + checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" 428 + 429 + [[package]] 430 + name = "base64" 431 version = "0.22.1" 432 source = "registry+https://github.com/rust-lang/crates.io-index" 433 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" ··· 574 "async-trait", 575 "atmst", 576 "atrium-api", 577 + "base64 0.22.1", 578 "chrono", 579 "cid 0.11.1", 580 "dotenvy", ··· 596 "sqlx", 597 "time", 598 "tokio", 599 + "tokio-tungstenite 0.24.0", 600 "tracing", 601 "tracing-subscriber", 602 "types", ··· 832 833 [[package]] 834 name = "core-foundation" 835 + version = "0.9.4" 836 + source = "registry+https://github.com/rust-lang/crates.io-index" 837 + checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" 838 + dependencies = [ 839 + "core-foundation-sys", 840 + "libc", 841 + ] 842 + 843 + [[package]] 844 + name = "core-foundation" 845 version = "0.10.1" 846 source = "registry+https://github.com/rust-lang/crates.io-index" 847 checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" ··· 1520 "fnv", 1521 "futures-core", 1522 "futures-sink", 1523 + "http 1.3.1", 1524 "indexmap", 1525 "slab", 1526 "tokio", ··· 1595 1596 [[package]] 1597 name = "http" 1598 + version = "0.2.12" 1599 + source = "registry+https://github.com/rust-lang/crates.io-index" 1600 + checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" 1601 + dependencies = [ 1602 + "bytes", 1603 + "fnv", 1604 + "itoa", 1605 + ] 1606 + 1607 + [[package]] 1608 + name = "http" 1609 version = "1.3.1" 1610 source = "registry+https://github.com/rust-lang/crates.io-index" 1611 checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" ··· 1622 checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" 1623 dependencies = [ 1624 "bytes", 1625 + "http 1.3.1", 1626 ] 1627 1628 [[package]] ··· 1633 dependencies = [ 1634 "bytes", 1635 "futures-core", 1636 + "http 1.3.1", 1637 "http-body", 1638 "pin-project-lite", 1639 ] ··· 1660 "futures-channel", 1661 "futures-util", 1662 "h2", 1663 + "http 1.3.1", 1664 "http-body", 1665 "httparse", 1666 "httpdate", ··· 1677 source = "registry+https://github.com/rust-lang/crates.io-index" 1678 checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" 1679 dependencies = [ 1680 + "http 1.3.1", 1681 "hyper", 1682 "hyper-util", 1683 + "rustls 0.23.31", 1684 + "rustls-native-certs 0.8.1", 1685 "rustls-pki-types", 1686 "tokio", 1687 + "tokio-rustls 0.26.2", 1688 "tower-service", 1689 "webpki-roots 1.0.2", 1690 ] ··· 1695 source = "registry+https://github.com/rust-lang/crates.io-index" 1696 checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" 1697 dependencies = [ 1698 + "base64 0.22.1", 1699 "bytes", 1700 "futures-channel", 1701 "futures-core", 1702 "futures-util", 1703 + "http 1.3.1", 1704 "http-body", 1705 "hyper", 1706 "ipnet", ··· 2252 source = "registry+https://github.com/rust-lang/crates.io-index" 2253 checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" 2254 dependencies = [ 2255 + "base64 0.22.1", 2256 "http-body-util", 2257 "hyper", 2258 "hyper-rustls", ··· 2346 "bytes", 2347 "encoding_rs", 2348 "futures-util", 2349 + "http 1.3.1", 2350 "httparse", 2351 "memchr", 2352 "mime", ··· 2820 "quinn-proto", 2821 "quinn-udp", 2822 "rustc-hash 2.1.1", 2823 + "rustls 0.23.31", 2824 "socket2 0.5.10", 2825 "thiserror 2.0.12", 2826 "tokio", ··· 2840 "rand 0.9.2", 2841 "ring", 2842 "rustc-hash 2.1.1", 2843 + "rustls 0.23.31", 2844 "rustls-pki-types", 2845 "slab", 2846 "thiserror 2.0.12", ··· 3051 checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" 3052 dependencies = [ 3053 "async-compression", 3054 + "base64 0.22.1", 3055 "bytes", 3056 "futures-core", 3057 "futures-util", 3058 + "http 1.3.1", 3059 "http-body", 3060 "http-body-util", 3061 "hyper", ··· 3066 "percent-encoding", 3067 "pin-project-lite", 3068 "quinn", 3069 + "rustls 0.23.31", 3070 "rustls-pki-types", 3071 "serde", 3072 "serde_json", 3073 "serde_urlencoded", 3074 "sync_wrapper", 3075 "tokio", 3076 + "tokio-rustls 0.26.2", 3077 "tokio-util", 3078 "tower", 3079 "tower-http", ··· 3122 [[package]] 3123 name = "rocketman" 3124 version = "0.2.3" 3125 + source = "registry+https://github.com/rust-lang/crates.io-index" 3126 + checksum = "9928fe43979c19ff1f46f7920c30b76dfcead7a4d571c9836c4d02da8587f844" 3127 dependencies = [ 3128 "anyhow", 3129 "async-trait", ··· 3131 "derive_builder", 3132 "flume", 3133 "futures-util", 3134 + "metrics 0.24.2", 3135 "rand 0.8.5", 3136 "serde", 3137 "serde_json", 3138 "tokio", 3139 + "tokio-tungstenite 0.20.1", 3140 "tracing", 3141 "tracing-subscriber", 3142 "url", ··· 3218 3219 [[package]] 3220 name = "rustls" 3221 + version = "0.21.12" 3222 + source = "registry+https://github.com/rust-lang/crates.io-index" 3223 + checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" 3224 + dependencies = [ 3225 + "log", 3226 + "ring", 3227 + "rustls-webpki 0.101.7", 3228 + "sct", 3229 + ] 3230 + 3231 + [[package]] 3232 + name = "rustls" 3233 version = "0.23.31" 3234 source = "registry+https://github.com/rust-lang/crates.io-index" 3235 checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" ··· 3238 "once_cell", 3239 "ring", 3240 "rustls-pki-types", 3241 + "rustls-webpki 0.103.4", 3242 "subtle", 3243 "zeroize", 3244 ] 3245 3246 [[package]] 3247 name = "rustls-native-certs" 3248 + version = "0.6.3" 3249 + source = "registry+https://github.com/rust-lang/crates.io-index" 3250 + checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" 3251 + dependencies = [ 3252 + "openssl-probe", 3253 + "rustls-pemfile", 3254 + "schannel", 3255 + "security-framework 2.11.1", 3256 + ] 3257 + 3258 + [[package]] 3259 + name = "rustls-native-certs" 3260 version = "0.8.1" 3261 source = "registry+https://github.com/rust-lang/crates.io-index" 3262 checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" ··· 3264 "openssl-probe", 3265 "rustls-pki-types", 3266 "schannel", 3267 + "security-framework 3.2.0", 3268 + ] 3269 + 3270 + [[package]] 3271 + name = "rustls-pemfile" 3272 + version = "1.0.4" 3273 + source = "registry+https://github.com/rust-lang/crates.io-index" 3274 + checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" 3275 + dependencies = [ 3276 + "base64 0.21.7", 3277 ] 3278 3279 [[package]] ··· 3284 dependencies = [ 3285 "web-time", 3286 "zeroize", 3287 + ] 3288 + 3289 + [[package]] 3290 + name = "rustls-webpki" 3291 + version = "0.101.7" 3292 + source = "registry+https://github.com/rust-lang/crates.io-index" 3293 + checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" 3294 + dependencies = [ 3295 + "ring", 3296 + "untrusted", 3297 ] 3298 3299 [[package]] ··· 3342 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 3343 3344 [[package]] 3345 + name = "sct" 3346 + version = "0.7.1" 3347 + source = "registry+https://github.com/rust-lang/crates.io-index" 3348 + checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" 3349 + dependencies = [ 3350 + "ring", 3351 + "untrusted", 3352 + ] 3353 + 3354 + [[package]] 3355 name = "sec1" 3356 version = "0.7.3" 3357 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3367 3368 [[package]] 3369 name = "security-framework" 3370 + version = "2.11.1" 3371 + source = "registry+https://github.com/rust-lang/crates.io-index" 3372 + checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" 3373 + dependencies = [ 3374 + "bitflags 2.9.1", 3375 + "core-foundation 0.9.4", 3376 + "core-foundation-sys", 3377 + "libc", 3378 + "security-framework-sys", 3379 + ] 3380 + 3381 + [[package]] 3382 + name = "security-framework" 3383 version = "3.2.0" 3384 source = "registry+https://github.com/rust-lang/crates.io-index" 3385 checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" 3386 dependencies = [ 3387 "bitflags 2.9.1", 3388 + "core-foundation 0.10.1", 3389 "core-foundation-sys", 3390 "libc", 3391 "security-framework-sys", ··· 3660 source = "registry+https://github.com/rust-lang/crates.io-index" 3661 checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" 3662 dependencies = [ 3663 + "base64 0.22.1", 3664 "bytes", 3665 "crc", 3666 "crossbeam-queue", ··· 3677 "memchr", 3678 "once_cell", 3679 "percent-encoding", 3680 + "rustls 0.23.31", 3681 "serde", 3682 "serde_json", 3683 "sha2", ··· 3737 checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" 3738 dependencies = [ 3739 "atoi", 3740 + "base64 0.22.1", 3741 "bitflags 2.9.1", 3742 "byteorder", 3743 "bytes", ··· 3781 checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" 3782 dependencies = [ 3783 "atoi", 3784 + "base64 0.22.1", 3785 "bitflags 2.9.1", 3786 "byteorder", 3787 "crc", ··· 4136 4137 [[package]] 4138 name = "tokio-rustls" 4139 + version = "0.24.1" 4140 + source = "registry+https://github.com/rust-lang/crates.io-index" 4141 + checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" 4142 + dependencies = [ 4143 + "rustls 0.21.12", 4144 + "tokio", 4145 + ] 4146 + 4147 + [[package]] 4148 + name = "tokio-rustls" 4149 version = "0.26.2" 4150 source = "registry+https://github.com/rust-lang/crates.io-index" 4151 checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" 4152 dependencies = [ 4153 + "rustls 0.23.31", 4154 "tokio", 4155 ] 4156 ··· 4167 4168 [[package]] 4169 name = "tokio-tungstenite" 4170 + version = "0.20.1" 4171 + source = "registry+https://github.com/rust-lang/crates.io-index" 4172 + checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" 4173 + dependencies = [ 4174 + "futures-util", 4175 + "log", 4176 + "rustls 0.21.12", 4177 + "rustls-native-certs 0.6.3", 4178 + "tokio", 4179 + "tokio-rustls 0.24.1", 4180 + "tungstenite 0.20.1", 4181 + "webpki-roots 0.25.4", 4182 + ] 4183 + 4184 + [[package]] 4185 + name = "tokio-tungstenite" 4186 version = "0.24.0" 4187 source = "registry+https://github.com/rust-lang/crates.io-index" 4188 checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" 4189 dependencies = [ 4190 "futures-util", 4191 "log", 4192 + "rustls 0.23.31", 4193 "rustls-pki-types", 4194 "tokio", 4195 + "tokio-rustls 0.26.2", 4196 + "tungstenite 0.24.0", 4197 "webpki-roots 0.26.11", 4198 ] 4199 ··· 4261 "bitflags 2.9.1", 4262 "bytes", 4263 "futures-util", 4264 + "http 1.3.1", 4265 "http-body", 4266 "iri-string", 4267 "pin-project-lite", ··· 4363 4364 [[package]] 4365 name = "tungstenite" 4366 + version = "0.20.1" 4367 + source = "registry+https://github.com/rust-lang/crates.io-index" 4368 + checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" 4369 + dependencies = [ 4370 + "byteorder", 4371 + "bytes", 4372 + "data-encoding", 4373 + "http 0.2.12", 4374 + "httparse", 4375 + "log", 4376 + "rand 0.8.5", 4377 + "rustls 0.21.12", 4378 + "sha1", 4379 + "thiserror 1.0.69", 4380 + "url", 4381 + "utf-8", 4382 + ] 4383 + 4384 + [[package]] 4385 + name = "tungstenite" 4386 version = "0.24.0" 4387 source = "registry+https://github.com/rust-lang/crates.io-index" 4388 checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" ··· 4390 "byteorder", 4391 "bytes", 4392 "data-encoding", 4393 + "http 1.3.1", 4394 "httparse", 4395 "log", 4396 "rand 0.8.5", 4397 + "rustls 0.23.31", 4398 "rustls-pki-types", 4399 "sha1", 4400 "thiserror 1.0.69", ··· 4414 "atrium-api", 4415 "atrium-xrpc", 4416 "chrono", 4417 + "http 1.3.1", 4418 "ipld-core", 4419 "langtag", 4420 "regex", ··· 4710 "js-sys", 4711 "wasm-bindgen", 4712 ] 4713 + 4714 + [[package]] 4715 + name = "webpki-roots" 4716 + version = "0.25.4" 4717 + source = "registry+https://github.com/rust-lang/crates.io-index" 4718 + checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" 4719 4720 [[package]] 4721 name = "webpki-roots"

+2 -7

Cargo.toml

··· 1 [workspace] 2 - members = [ 3 - "apps/aqua", 4 - "services/cadet", 5 - "services/rocketman", 6 - "tools/teal-cli", 7 - ] 8 resolver = "2" 9 10 [workspace.dependencies] ··· 51 chrono = "0.4" 52 uuid = { version = "1.0", features = ["v4", "serde"] } 53 types = { path = "services/types" } 54 - rocketman = { path = "services/rocketman" } 55 56 # CAR and IPLD dependencies 57 iroh-car = "0.5"

··· 1 [workspace] 2 + members = ["apps/aqua", "services/cadet", "tools/teal-cli"] 0 0 0 0 0 3 resolver = "2" 4 5 [workspace.dependencies] ··· 46 chrono = "0.4" 47 uuid = { version = "1.0", features = ["v4", "serde"] } 48 types = { path = "services/types" } 49 + rocketman = "0.2.3" 50 51 # CAR and IPLD dependencies 52 iroh-car = "0.5"

-24

compose.db-test.yml

··· 1 - version: "3.8" 2 - 3 - services: 4 - postgres: 5 - image: postgres:latest 6 - container_name: postgres_test_db 7 - environment: 8 - POSTGRES_USER: postgres 9 - POSTGRES_PASSWORD: testpass123 10 - POSTGRES_DB: teal_test 11 - ports: 12 - - "5433:5432" 13 - volumes: 14 - - postgres_test_data:/var/lib/postgresql/data 15 - networks: 16 - - test_network 17 - command: postgres -c log_statement=all -c log_destination=stderr 18 - 19 - networks: 20 - test_network: 21 - driver: bridge 22 - 23 - volumes: 24 - postgres_test_data:

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

+2 -2

services/Cargo.toml

··· 1 [workspace] 2 - members = ["cadet", "rocketman", "satellite", "types"] 3 resolver = "2" 4 5 [workspace.dependencies] ··· 32 chrono = { version = "0.4", features = ["serde"] } 33 uuid = { version = "1.0", features = ["v4", "serde"] } 34 types = { path = "types" } 35 - rocketman = { path = "rocketman" } 36 37 # CAR and IPLD dependencies 38 iroh-car = "0.4"

··· 1 [workspace] 2 + members = ["cadet", "satellite", "types"] 3 resolver = "2" 4 5 [workspace.dependencies] ··· 32 chrono = { version = "0.4", features = ["serde"] } 33 uuid = { version = "1.0", features = ["v4", "serde"] } 34 types = { path = "types" } 35 + rocketman = "0.2.5" 36 37 # CAR and IPLD dependencies 38 iroh-car = "0.4"

-34

services/rocketman/Cargo.toml

··· 1 - [package] 2 - name = "rocketman" 3 - version = "0.2.3" 4 - edition = "2021" 5 - 6 - license = "MIT" 7 - authors = ["Natalie B. <nat@natalie.sh>"] 8 - repository = "https://github.com/espeon/cadet" 9 - 10 - readme = "readme.md" 11 - 12 - description = "A modular(ish) jetstream consumer." 13 - 14 - [dependencies] 15 - tokio.workspace = true 16 - tokio-tungstenite.workspace = true 17 - futures-util = "0.3" 18 - url.workspace = true 19 - rand.workspace = true 20 - tracing.workspace = true 21 - tracing-subscriber.workspace = true 22 - metrics.workspace = true 23 - derive_builder = "0.20.2" 24 - bon = "3.3.2" 25 - serde = { workspace = true, features = ["derive"] } 26 - serde_json.workspace = true 27 - flume.workspace = true 28 - anyhow.workspace = true 29 - async-trait.workspace = true 30 - zstd = { version = "0.13.3", optional = true } 31 - 32 - [features] 33 - default = ["zstd"] 34 - zstd = ["dep:zstd"]

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-76

services/rocketman/examples/spew-bsky-posts.rs

··· 1 - use async_trait::async_trait; 2 - use rocketman::{ 3 - connection::JetstreamConnection, 4 - handler, 5 - ingestion::LexiconIngestor, 6 - options::JetstreamOptions, 7 - types::event::{Commit, Event}, 8 - }; 9 - use serde_json::Value; 10 - use std::{collections::HashMap, sync::Arc, sync::Mutex}; 11 - 12 - #[tokio::main] 13 - async fn main() { 14 - // init the builder 15 - let opts = JetstreamOptions::builder() 16 - // your EXACT nsids 17 - .wanted_collections(vec!["app.bsky.feed.post".to_string()]) 18 - .build(); 19 - // create the jetstream connector 20 - let jetstream = JetstreamConnection::new(opts); 21 - 22 - // create your ingestors 23 - let mut ingestors: HashMap<String, Box<dyn LexiconIngestor + Send + Sync>> = HashMap::new(); 24 - ingestors.insert( 25 - // your EXACT nsid 26 - "app.bsky.feed.post".to_string(), 27 - Box::new(MyCoolIngestor), 28 - ); 29 - 30 - // tracks the last message we've processed 31 - let cursor: Arc<Mutex<Option<u64>>> = Arc::new(Mutex::new(None)); 32 - 33 - // get channels 34 - let msg_rx = jetstream.get_msg_rx(); 35 - let reconnect_tx = jetstream.get_reconnect_tx(); 36 - 37 - // spawn a task to process messages from the queue. 38 - // this is a simple implementation, you can use a more complex one based on needs. 39 - let c_cursor = cursor.clone(); 40 - tokio::spawn(async move { 41 - while let Ok(message) = msg_rx.recv_async().await { 42 - if let Err(e) = 43 - handler::handle_message(message, &ingestors, reconnect_tx.clone(), c_cursor.clone()) 44 - .await 45 - { 46 - eprintln!("Error processing message: {}", e); 47 - }; 48 - } 49 - }); 50 - 51 - // connect to jetstream 52 - // retries internally, but may fail if there is an extreme error. 53 - if let Err(e) = jetstream.connect(cursor.clone()).await { 54 - eprintln!("Failed to connect to Jetstream: {}", e); 55 - std::process::exit(1); 56 - } 57 - } 58 - 59 - pub struct MyCoolIngestor; 60 - 61 - /// A cool ingestor implementation. Will just print the message. Does not do verification. 62 - #[async_trait] 63 - impl LexiconIngestor for MyCoolIngestor { 64 - async fn ingest(&self, message: Event<Value>) -> anyhow::Result<()> { 65 - if let Some(Commit { 66 - record: Some(record), 67 - .. 68 - }) = message.commit 69 - { 70 - if let Some(Value::String(text)) = record.get("text") { 71 - println!("{text:?}"); 72 - } 73 - } 74 - Ok(()) 75 - } 76 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-11

services/rocketman/package.json

··· 1 - { 2 - "name": "@repo/rocketman", 3 - "private": true, 4 - "scripts": { 5 - "build": "cargo build --release", 6 - "build:rust": "cargo build --release", 7 - "dev": "cargo watch -x 'run'", 8 - "test": "cargo test", 9 - "test:rust": "cargo test" 10 - } 11 - }

··· 0 0 0 0 0 0 0 0 0 0 0

-74

services/rocketman/readme.md

··· 1 - ## Rocketman 2 - 3 - A modular(ish) jetstream consumer. Backed by Tungstenite. 4 - 5 - 6 - ### Installation 7 - ```toml 8 - [dependencies] 9 - rocketman = "latest" # pyt the latest version here 10 - tokio = { version = "1", features = ["macros", "rt-multi-thread"] } 11 - ``` 12 - ### Usage 13 - ```rs 14 - #[tokio::main] 15 - async fn main() { 16 - // init the builder 17 - let opts = JetstreamOptions::builder() 18 - // your EXACT nsids 19 - .wanted_collections(vec!["com.example.cool.nsid".to_string()]) 20 - .build(); 21 - // create the jetstream connector 22 - let jetstream = JetstreamConnection::new(opts); 23 - 24 - // create your ingestors 25 - let mut ingestors: HashMap<String, Box<dyn LexiconIngestor + Send + Sync>> = HashMap::new(); 26 - ingestors.insert( 27 - // your EXACT nsid 28 - "com.example.cool.nsid".to_string(), 29 - Box::new(MyCoolIngestor), 30 - ); 31 - 32 - 33 - // tracks the last message we've processed 34 - let cursor: Arc<Mutex<Option<u64>>> = Arc::new(Mutex::new(None)); 35 - 36 - // get channels 37 - let msg_rx = jetstream.get_msg_rx(); 38 - let reconnect_tx = jetstream.get_reconnect_tx(); 39 - 40 - // spawn a task to process messages from the queue. 41 - // this is a simple implementation, you can use a more complex one based on needs. 42 - let c_cursor = cursor.clone(); 43 - tokio::spawn(async move { 44 - while let Ok(message) = msg_rx.recv_async().await { 45 - if let Err(e) = 46 - handler::handle_message(message, &ingestors, reconnect_tx.clone(), c_cursor.clone()) 47 - .await 48 - { 49 - error!("Error processing message: {}", e); 50 - }; 51 - } 52 - }); 53 - 54 - // connect to jetstream 55 - // retries internally, but may fail if there is an extreme error. 56 - if let Err(e) = jetstream.connect(cursor.clone()).await { 57 - error!("Failed to connect to Jetstream: {}", e); 58 - std::process::exit(1); 59 - } 60 - } 61 - 62 - pub struct MyCoolIngestor; 63 - 64 - /// A cool ingestor implementation. Will just print the message. Does not do verification. 65 - impl LexiconIngestor for MyCoolIngestor { 66 - async fn ingest(&self, message: Event<Value>) -> Result<()> { 67 - info!("{:?}", message); 68 - // Process message for default lexicon. 69 - Ok(()) 70 - } 71 - } 72 - ``` 73 - ### gratz 74 - Based heavily on [phil's jetstream consumer on atcosm constellation.](https://github.com/atcosm/links/blob/main/constellation/src/consumer/jetstream.rs)

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-335

services/rocketman/src/connection.rs

··· 1 - use flume::{Receiver, Sender}; 2 - use futures_util::StreamExt; 3 - use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; 4 - use std::cmp::{max, min}; 5 - use std::sync::{Arc, Mutex}; 6 - use std::time::Instant; 7 - use tokio::time::{sleep, Duration}; 8 - use tokio_tungstenite::{connect_async, tungstenite::Message}; 9 - use tracing::{error, info}; 10 - use url::Url; 11 - 12 - use crate::options::JetstreamOptions; 13 - use crate::time::system_time::SystemTimeProvider; 14 - use crate::time::TimeProvider; 15 - 16 - pub struct JetstreamConnection { 17 - pub opts: JetstreamOptions, 18 - reconnect_tx: flume::Sender<()>, 19 - reconnect_rx: flume::Receiver<()>, 20 - msg_tx: flume::Sender<Message>, 21 - msg_rx: flume::Receiver<Message>, 22 - } 23 - 24 - impl JetstreamConnection { 25 - pub fn new(opts: JetstreamOptions) -> Self { 26 - let (reconnect_tx, reconnect_rx) = flume::bounded(opts.bound); 27 - let (msg_tx, msg_rx) = flume::bounded(opts.bound); 28 - Self { 29 - opts, 30 - reconnect_tx, 31 - reconnect_rx, 32 - msg_tx, 33 - msg_rx, 34 - } 35 - } 36 - 37 - pub fn get_reconnect_tx(&self) -> Sender<()> { 38 - self.reconnect_tx.clone() 39 - } 40 - 41 - pub fn get_msg_rx(&self) -> Receiver<Message> { 42 - self.msg_rx.clone() 43 - } 44 - 45 - fn build_ws_url(&self, cursor: Arc<Mutex<Option<u64>>>) -> String { 46 - let mut url = Url::parse(&self.opts.ws_url.to_string()).unwrap(); 47 - 48 - // Append query params 49 - if let Some(ref cols) = self.opts.wanted_collections { 50 - for col in cols { 51 - url.query_pairs_mut().append_pair("wantedCollections", col); 52 - } 53 - } 54 - if let Some(ref dids) = self.opts.wanted_dids { 55 - for did in dids { 56 - url.query_pairs_mut().append_pair("wantedDids", did); 57 - } 58 - } 59 - if let Some(cursor) = cursor.lock().unwrap().as_ref() { 60 - url.query_pairs_mut() 61 - .append_pair("cursor", &cursor.to_string()); 62 - } 63 - #[cfg(feature = "zstd")] 64 - if self.opts.compress { 65 - url.query_pairs_mut().append_pair("compress", "true"); 66 - } 67 - 68 - url.to_string() 69 - } 70 - 71 - pub async fn connect( 72 - &self, 73 - cursor: Arc<Mutex<Option<u64>>>, 74 - ) -> Result<(), Box<dyn std::error::Error>> { 75 - describe_counter!( 76 - "jetstream.connection.attempt", 77 - Unit::Count, 78 - "attempts to connect to jetstream service" 79 - ); 80 - describe_counter!( 81 - "jetstream.connection.error", 82 - Unit::Count, 83 - "errors connecting to jetstream service" 84 - ); 85 - describe_histogram!( 86 - "jetstream.connection.duration", 87 - Unit::Seconds, 88 - "Time connected to jetstream service" 89 - ); 90 - describe_counter!( 91 - "jetstream.connection.reconnect", 92 - Unit::Count, 93 - "reconnects to jetstream service" 94 - ); 95 - let mut retry_interval = 1; 96 - 97 - let time_provider = SystemTimeProvider::new(); 98 - 99 - let mut start_time = time_provider.now(); 100 - 101 - loop { 102 - counter!("jetstream.connection.attempt").increment(1); 103 - info!("Connecting to {}", self.opts.ws_url); 104 - let start = Instant::now(); 105 - 106 - let ws_url = self.build_ws_url(cursor.clone()); 107 - 108 - match connect_async(ws_url).await { 109 - Ok((ws_stream, response)) => { 110 - let elapsed = start.elapsed(); 111 - info!("Connected. HTTP status: {}", response.status()); 112 - 113 - let (_, mut read) = ws_stream.split(); 114 - 115 - loop { 116 - // Inner loop to handle messages, reconnect signals, and receive timeout 117 - let receive_timeout = 118 - sleep(Duration::from_secs(self.opts.timeout_time_sec as u64)); 119 - tokio::pin!(receive_timeout); 120 - 121 - loop { 122 - tokio::select! { 123 - message_result = read.next() => { 124 - match message_result { 125 - Some(message) => { 126 - // Reset timeout on message received 127 - receive_timeout.as_mut().reset(tokio::time::Instant::now() + Duration::from_secs(self.opts.timeout_time_sec as u64)); 128 - 129 - histogram!("jetstream.connection.duration").record(elapsed.as_secs_f64()); 130 - match message { 131 - Ok(message) => { 132 - if let Err(err) = self.msg_tx.send_async(message).await { 133 - counter!("jetstream.error").increment(1); 134 - error!("Failed to queue message: {}", err); 135 - } 136 - } 137 - Err(e) => { 138 - counter!("jetstream.error").increment(1); 139 - error!("Error: {}", e); 140 - } 141 - } 142 - } 143 - None => { 144 - info!("Stream closed by server."); 145 - counter!("jetstream.connection.reconnect").increment(1); 146 - break; // Stream ended, break inner loop to reconnect 147 - } 148 - } 149 - } 150 - _ = self.reconnect_rx.recv_async() => { 151 - info!("Reconnect signal received."); 152 - counter!("jetstream.connection.reconnect").increment(1); 153 - break; 154 - } 155 - _ = &mut receive_timeout => { 156 - // last final poll, just in case 157 - match read.next().await { 158 - Some(Ok(message)) => { 159 - if let Err(err) = self.msg_tx.send_async(message).await { 160 - counter!("jetstream.error").increment(1); 161 - error!("Failed to queue message: {}", err); 162 - } 163 - // Reset timeout to continue 164 - receive_timeout.as_mut().reset(tokio::time::Instant::now() + Duration::from_secs(self.opts.timeout_time_sec as u64)); 165 - } 166 - Some(Err(e)) => { 167 - counter!("jetstream.error").increment(1); 168 - error!("Error receiving message during final poll: {}", e); 169 - counter!("jetstream.connection.reconnect").increment(1); 170 - break; 171 - } 172 - None => { 173 - info!("No commits received in {} seconds, reconnecting.", self.opts.timeout_time_sec); 174 - counter!("jetstream.connection.reconnect").increment(1); 175 - break; 176 - } 177 - } 178 - } 179 - } 180 - } 181 - } 182 - } 183 - Err(e) => { 184 - let elapsed_time = time_provider.elapsed(start_time); 185 - // reset if time connected > the time we set 186 - if elapsed_time.as_secs() > self.opts.max_retry_interval_seconds { 187 - retry_interval = 0; 188 - start_time = time_provider.now(); 189 - } 190 - counter!("jetstream.connection.error").increment(1); 191 - error!("Connection error: {}", e); 192 - } 193 - } 194 - 195 - let sleep_time = max(1, min(self.opts.max_retry_interval_seconds, retry_interval)); 196 - info!("Reconnecting in {} seconds...", sleep_time); 197 - sleep(Duration::from_secs(sleep_time)).await; 198 - 199 - if retry_interval > self.opts.max_retry_interval_seconds { 200 - retry_interval = self.opts.max_retry_interval_seconds; 201 - } else { 202 - retry_interval *= 2; 203 - } 204 - } 205 - } 206 - 207 - pub fn force_reconnect(&self) -> Result<(), flume::SendError<()>> { 208 - info!("Force reconnect requested."); 209 - self.reconnect_tx.send(()) // Send a reconnect signal 210 - } 211 - } 212 - 213 - #[cfg(test)] 214 - mod tests { 215 - use super::*; 216 - use std::sync::{Arc, Mutex}; 217 - use tokio::task; 218 - use tokio::time::{timeout, Duration}; 219 - use tokio_tungstenite::tungstenite::Message; 220 - 221 - #[test] 222 - fn test_build_ws_url() { 223 - let opts = JetstreamOptions { 224 - wanted_collections: Some(vec!["col1".to_string(), "col2".to_string()]), 225 - wanted_dids: Some(vec!["did1".to_string()]), 226 - ..Default::default() 227 - }; 228 - let connection = JetstreamConnection::new(opts); 229 - 230 - let test = Arc::new(Mutex::new(Some(8373))); 231 - 232 - let url = connection.build_ws_url(test); 233 - 234 - assert!(url.starts_with("wss://")); 235 - assert!(url.contains("cursor=8373")); 236 - assert!(url.contains("wantedCollections=col1")); 237 - assert!(url.contains("wantedCollections=col2")); 238 - assert!(url.contains("wantedDids=did1")); 239 - } 240 - 241 - #[tokio::test] 242 - async fn test_force_reconnect() { 243 - let opts = JetstreamOptions::default(); 244 - let connection = JetstreamConnection::new(opts); 245 - 246 - // Spawn a task to listen for the reconnect signal 247 - let reconnect_rx = connection.reconnect_rx.clone(); 248 - let recv_task = task::spawn(async move { 249 - reconnect_rx 250 - .recv_async() 251 - .await 252 - .expect("Failed to receive reconnect signal"); 253 - }); 254 - 255 - connection 256 - .force_reconnect() 257 - .expect("Failed to send reconnect signal"); 258 - 259 - // Ensure reconnect signal was received 260 - assert!(recv_task.await.is_ok()); 261 - } 262 - 263 - #[tokio::test] 264 - async fn test_message_queue() { 265 - let opts = JetstreamOptions::default(); 266 - let connection = JetstreamConnection::new(opts); 267 - 268 - let msg_rx = connection.get_msg_rx(); 269 - let msg = Message::Text("test message".into()); 270 - 271 - // Send a message to the queue 272 - connection 273 - .msg_tx 274 - .send_async(msg.clone()) 275 - .await 276 - .expect("Failed to send message"); 277 - 278 - // Receive and verify the message 279 - let received = msg_rx 280 - .recv_async() 281 - .await 282 - .expect("Failed to receive message"); 283 - assert_eq!(received, msg); 284 - } 285 - 286 - #[tokio::test] 287 - async fn test_connection_retries_on_failure() { 288 - let opts = JetstreamOptions::default(); 289 - let connection = Arc::new(JetstreamConnection::new(opts)); 290 - 291 - let cursor = Arc::new(Mutex::new(None)); 292 - 293 - // Timeout to prevent infinite loop 294 - let result = timeout(Duration::from_secs(3), connection.connect(cursor)).await; 295 - 296 - assert!(result.is_err(), "Expected timeout due to retry logic"); 297 - } 298 - 299 - #[tokio::test] 300 - async fn test_reconnect_after_receive_timeout() { 301 - use tokio::net::TcpListener; 302 - use tokio_tungstenite::accept_async; 303 - 304 - let opts = JetstreamOptions { 305 - ws_url: crate::endpoints::JetstreamEndpoints::Custom("ws://127.0.0.1:9001".to_string()), 306 - bound: 5, 307 - max_retry_interval_seconds: 1, 308 - ..Default::default() 309 - }; 310 - let connection = JetstreamConnection::new(opts); 311 - let cursor = Arc::new(Mutex::new(None)); 312 - 313 - // set up dummy "websocket" 314 - let listener = TcpListener::bind("127.0.0.1:9001") 315 - .await 316 - .expect("Failed to bind"); 317 - let server_handle = tokio::spawn(async move { 318 - if let Ok((stream, _)) = listener.accept().await { 319 - let ws_stream = accept_async(stream).await.expect("Failed to accept"); 320 - // send nothing 321 - tokio::time::sleep(Duration::from_secs(6)).await; 322 - drop(ws_stream); 323 - } 324 - }); 325 - 326 - // spawn, then run for >30 seconds to trigger reconnect 327 - let connect_handle = tokio::spawn(async move { 328 - tokio::time::timeout(Duration::from_secs(5), connection.connect(cursor)) 329 - .await 330 - .ok(); 331 - }); 332 - 333 - let _ = tokio::join!(server_handle, connect_handle); 334 - } 335 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-65

services/rocketman/src/endpoints.rs

··· 1 - use std::fmt::{Display, Formatter, Result}; 2 - 3 - #[derive(Debug, Clone, PartialEq, Eq, Hash)] 4 - pub enum JetstreamEndpointLocations { 5 - UsEast, 6 - UsWest, 7 - } 8 - 9 - impl Display for JetstreamEndpointLocations { 10 - fn fmt(&self, f: &mut Formatter<'_>) -> Result { 11 - write!( 12 - f, 13 - "{}", 14 - match self { 15 - Self::UsEast => "us-east", 16 - Self::UsWest => "us-west", 17 - } 18 - ) 19 - } 20 - } 21 - 22 - #[derive(Debug, Clone, PartialEq, Eq, Hash)] 23 - pub enum JetstreamEndpoints { 24 - Public(JetstreamEndpointLocations, i8), 25 - Custom(String), 26 - } 27 - 28 - impl Display for JetstreamEndpoints { 29 - fn fmt(&self, f: &mut Formatter<'_>) -> Result { 30 - match self { 31 - Self::Public(location, id) => write!( 32 - f, 33 - "wss://jetstream{}.{}.bsky.network/subscribe", 34 - id, location 35 - ), 36 - Self::Custom(url) => write!(f, "{}", url), 37 - } 38 - } 39 - } 40 - 41 - impl Default for JetstreamEndpoints { 42 - fn default() -> Self { 43 - Self::Public(JetstreamEndpointLocations::UsEast, 2) 44 - } 45 - } 46 - 47 - #[cfg(test)] 48 - mod tests { 49 - use super::*; 50 - 51 - #[test] 52 - fn test_display_public() { 53 - let endpoint = JetstreamEndpoints::Public(JetstreamEndpointLocations::UsEast, 2); 54 - assert_eq!( 55 - endpoint.to_string(), 56 - "wss://jetstream2.us-east.bsky.network/subscribe" 57 - ); 58 - } 59 - 60 - #[test] 61 - fn test_display_custom() { 62 - let endpoint = JetstreamEndpoints::Custom("wss://custom.bsky.network/subscribe".into()); 63 - assert_eq!(endpoint.to_string(), "wss://custom.bsky.network/subscribe"); 64 - } 65 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-1

services/rocketman/src/err.rs

··· 1 - // TODO: error types instead of using anyhow

··· 0

-452

services/rocketman/src/handler.rs

··· 1 - use anyhow::Result; 2 - use flume::Sender; 3 - use metrics::{counter, describe_counter, Unit}; 4 - use serde_json::Value; 5 - use std::{ 6 - collections::HashMap, 7 - sync::{Arc, Mutex}, 8 - }; 9 - use tokio_tungstenite::tungstenite::{Error, Message}; 10 - use tracing::{debug, error}; 11 - 12 - #[cfg(feature = "zstd")] 13 - use std::io::Cursor as IoCursor; 14 - #[cfg(feature = "zstd")] 15 - use std::sync::LazyLock; 16 - #[cfg(feature = "zstd")] 17 - use zstd::dict::DecoderDictionary; 18 - 19 - use crate::{ 20 - ingestion::LexiconIngestor, 21 - types::event::{Event, Kind}, 22 - }; 23 - 24 - /// The custom `zstd` dictionary used for decoding compressed Jetstream messages. 25 - /// 26 - /// Sourced from the [official Bluesky Jetstream repo.](https://github.com/bluesky-social/jetstream/tree/main/pkg/models) 27 - #[cfg(feature = "zstd")] 28 - static ZSTD_DICTIONARY: LazyLock<DecoderDictionary> = 29 - LazyLock::new(|| DecoderDictionary::copy(include_bytes!("../zstd/dictionary"))); 30 - 31 - pub async fn handle_message( 32 - message: Message, 33 - ingestors: &HashMap<String, Box<dyn LexiconIngestor + Send + Sync>>, 34 - reconnect_tx: Sender<()>, 35 - cursor: Arc<Mutex<Option<u64>>>, 36 - ) -> Result<()> { 37 - describe_counter!( 38 - "jetstream.event", 39 - Unit::Count, 40 - "number of event ingest attempts" 41 - ); 42 - describe_counter!( 43 - "jetstream.event.parse", 44 - Unit::Count, 45 - "events that were successfully processed" 46 - ); 47 - describe_counter!( 48 - "jetstream.event.fail", 49 - Unit::Count, 50 - "events that could not be read" 51 - ); 52 - describe_counter!("jetstream.error", Unit::Count, "errors encountered"); 53 - match message { 54 - Message::Text(text) => { 55 - debug!("Text message received"); 56 - counter!("jetstream.event").increment(1); 57 - let envelope: Event<Value> = serde_json::from_str(&text).map_err(|e| { 58 - anyhow::anyhow!("Failed to parse message: {} with json string {}", e, text) 59 - })?; 60 - debug!("envelope: {:?}", envelope); 61 - handle_envelope(envelope, cursor, ingestors).await?; 62 - Ok(()) 63 - } 64 - #[cfg(feature = "zstd")] 65 - Message::Binary(bytes) => { 66 - debug!("Binary message received"); 67 - counter!("jetstream.event").increment(1); 68 - let decoder = zstd::stream::Decoder::with_prepared_dictionary( 69 - IoCursor::new(bytes), 70 - &ZSTD_DICTIONARY, 71 - )?; 72 - let envelope: Event<Value> = serde_json::from_reader(decoder) 73 - .map_err(|e| anyhow::anyhow!("Failed to parse binary message: {}", e))?; 74 - debug!("envelope: {:?}", envelope); 75 - handle_envelope(envelope, cursor, ingestors).await?; 76 - Ok(()) 77 - } 78 - #[cfg(not(feature = "zstd"))] 79 - Message::Binary(_) => { 80 - debug!("Binary message received"); 81 - Err(anyhow::anyhow!( 82 - "binary message received but zstd feature is not enabled" 83 - )) 84 - } 85 - Message::Close(_) => { 86 - debug!("Server closed connection"); 87 - if let Err(e) = reconnect_tx.send(()) { 88 - counter!("jetstream.event.parse.error", "error" => "failed_to_send_reconnect_signal").increment(1); 89 - error!("Failed to send reconnect signal: {}", e); 90 - } 91 - Err(Error::ConnectionClosed.into()) 92 - } 93 - _ => Ok(()), 94 - } 95 - } 96 - 97 - async fn handle_envelope( 98 - envelope: Event<Value>, 99 - cursor: Arc<Mutex<Option<u64>>>, 100 - ingestors: &HashMap<String, Box<dyn LexiconIngestor + Send + Sync>>, 101 - ) -> Result<()> { 102 - if let Some(ref time_us) = envelope.time_us { 103 - debug!("Time: {}", time_us); 104 - if let Some(cursor) = cursor.lock().unwrap().as_mut() { 105 - debug!("Cursor: {}", cursor); 106 - if time_us > cursor { 107 - debug!("Cursor is behind, resetting"); 108 - *cursor = *time_us; 109 - } 110 - } 111 - } 112 - 113 - match envelope.kind { 114 - Kind::Commit => match extract_commit_nsid(&envelope) { 115 - Ok(nsid) => { 116 - if let Some(fun) = ingestors.get(&nsid) { 117 - match fun.ingest(envelope).await { 118 - Ok(_) => { 119 - counter!("jetstream.event.parse.commit", "nsid" => nsid).increment(1) 120 - } 121 - Err(e) => { 122 - error!("Error ingesting commit with nsid {}: {}", nsid, e); 123 - counter!("jetstream.error").increment(1); 124 - counter!("jetstream.event.fail").increment(1); 125 - } 126 - } 127 - } 128 - } 129 - Err(e) => error!("Error parsing commit: {}", e), 130 - }, 131 - Kind::Identity => { 132 - counter!("jetstream.event.parse.identity").increment(1); 133 - } 134 - Kind::Account => { 135 - counter!("jetstream.event.parse.account").increment(1); 136 - } 137 - Kind::Unknown(kind) => { 138 - counter!("jetstream.event.parse.unknown", "kind" => kind).increment(1); 139 - } 140 - } 141 - Ok(()) 142 - } 143 - 144 - fn extract_commit_nsid(envelope: &Event<Value>) -> anyhow::Result<String> { 145 - // if the type is not a commit 146 - if envelope.commit.is_none() { 147 - return Err(anyhow::anyhow!( 148 - "Message has no commit, so there is no nsid attached." 149 - )); 150 - } else if let Some(ref commit) = envelope.commit { 151 - return Ok(commit.collection.clone()); 152 - } 153 - 154 - Err(anyhow::anyhow!("Failed to extract nsid: unknown error")) 155 - } 156 - 157 - #[cfg(test)] 158 - mod tests { 159 - use super::*; 160 - use crate::types::event::Event; 161 - use anyhow::Result; 162 - use async_trait::async_trait; 163 - use flume::{Receiver, Sender}; 164 - use serde_json::json; 165 - use std::{ 166 - collections::HashMap, 167 - sync::{Arc, Mutex}, 168 - }; 169 - use tokio_tungstenite::tungstenite::Message; 170 - 171 - // Dummy ingestor that records if it was called. 172 - struct DummyIngestor { 173 - pub called: Arc<Mutex<bool>>, 174 - } 175 - 176 - #[async_trait] 177 - impl crate::ingestion::LexiconIngestor for DummyIngestor { 178 - async fn ingest(&self, _event: Event<serde_json::Value>) -> Result<(), anyhow::Error> { 179 - let mut called = self.called.lock().unwrap(); 180 - *called = true; 181 - Ok(()) 182 - } 183 - } 184 - 185 - // Dummy ingestor that always returns an error. 186 - struct ErrorIngestor; 187 - 188 - #[async_trait] 189 - impl crate::ingestion::LexiconIngestor for ErrorIngestor { 190 - async fn ingest(&self, _event: Event<serde_json::Value>) -> Result<(), anyhow::Error> { 191 - Err(anyhow::anyhow!("Ingest error")) 192 - } 193 - } 194 - 195 - // Helper to create a reconnect channel. 196 - fn setup_reconnect_channel() -> (Sender<()>, Receiver<()>) { 197 - flume::unbounded() 198 - } 199 - 200 - #[tokio::test] 201 - async fn test_valid_commit_success() { 202 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 203 - let cursor = Arc::new(Mutex::new(Some(100))); 204 - let called_flag = Arc::new(Mutex::new(false)); 205 - 206 - // Create a valid commit event JSON. 207 - let event_json = json!({ 208 - "did": "did:example:123", 209 - "time_us": 200, 210 - "kind": "commit", 211 - "commit": { 212 - "rev": "1", 213 - "operation": "create", 214 - "collection": "ns1", 215 - "rkey": "rkey1", 216 - "record": { "foo": "bar" }, 217 - "cid": "cid123" 218 - }, 219 - }) 220 - .to_string(); 221 - 222 - let mut ingestors: HashMap< 223 - String, 224 - Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>, 225 - > = HashMap::new(); 226 - ingestors.insert( 227 - "ns1".to_string(), 228 - Box::new(DummyIngestor { 229 - called: called_flag.clone(), 230 - }), 231 - ); 232 - 233 - let result = handle_message( 234 - Message::Text(event_json), 235 - &ingestors, 236 - reconnect_tx, 237 - cursor.clone(), 238 - ) 239 - .await; 240 - assert!(result.is_ok()); 241 - // Check that the ingestor was called. 242 - assert!(*called_flag.lock().unwrap()); 243 - // Verify that the cursor got updated. 244 - assert_eq!(*cursor.lock().unwrap(), Some(200)); 245 - } 246 - 247 - #[cfg(feature = "zstd")] 248 - #[tokio::test] 249 - async fn test_binary_valid_commit() { 250 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 251 - let cursor = Arc::new(Mutex::new(Some(100))); 252 - let called_flag = Arc::new(Mutex::new(false)); 253 - 254 - let uncompressed_json = json!({ 255 - "did": "did:example:123", 256 - "time_us": 200, 257 - "kind": "commit", 258 - "commit": { 259 - "rev": "1", 260 - "operation": "create", 261 - "collection": "ns1", 262 - "rkey": "rkey1", 263 - "record": { "foo": "bar" }, 264 - "cid": "cid123" 265 - }, 266 - }) 267 - .to_string(); 268 - 269 - let compressed_dest: IoCursor<Vec<u8>> = IoCursor::new(vec![]); 270 - let mut encoder = zstd::Encoder::with_prepared_dictionary( 271 - compressed_dest, 272 - &zstd::dict::EncoderDictionary::copy(include_bytes!("../zstd/dictionary"), 0), 273 - ) 274 - .unwrap(); 275 - std::io::copy( 276 - &mut IoCursor::new(uncompressed_json.as_bytes()), 277 - &mut encoder, 278 - ) 279 - .unwrap(); 280 - let compressed_dest = encoder.finish().unwrap(); 281 - 282 - let mut ingestors: HashMap< 283 - String, 284 - Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>, 285 - > = HashMap::new(); 286 - ingestors.insert( 287 - "ns1".to_string(), 288 - Box::new(DummyIngestor { 289 - called: called_flag.clone(), 290 - }), 291 - ); 292 - 293 - let result = handle_message( 294 - Message::Binary(compressed_dest.into_inner()), 295 - &ingestors, 296 - reconnect_tx, 297 - cursor.clone(), 298 - ) 299 - .await; 300 - 301 - assert!(result.is_ok()); 302 - // Check that the ingestor was called. 303 - assert!(*called_flag.lock().unwrap()); 304 - // Verify that the cursor got updated. 305 - assert_eq!(*cursor.lock().unwrap(), Some(200)); 306 - } 307 - 308 - #[tokio::test] 309 - async fn test_commit_ingest_failure() { 310 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 311 - let cursor = Arc::new(Mutex::new(Some(100))); 312 - 313 - // Valid commit event with an ingestor that fails. 314 - let event_json = json!({ 315 - "did": "did:example:123", 316 - "time_us": 300, 317 - "kind": "commit", 318 - "commit": { 319 - "rev": "1", 320 - "operation": "create", 321 - "collection": "ns_error", 322 - "rkey": "rkey1", 323 - "record": { "foo": "bar" }, 324 - "cid": "cid123" 325 - }, 326 - "identity": null 327 - }) 328 - .to_string(); 329 - 330 - let mut ingestors: HashMap< 331 - String, 332 - Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>, 333 - > = HashMap::new(); 334 - ingestors.insert("ns_error".to_string(), Box::new(ErrorIngestor)); 335 - 336 - // Even though ingestion fails, handle_message returns Ok(()). 337 - let result = handle_message( 338 - Message::Text(event_json), 339 - &ingestors, 340 - reconnect_tx, 341 - cursor.clone(), 342 - ) 343 - .await; 344 - assert!(result.is_ok()); 345 - // Cursor should still update because it comes before the ingest call. 346 - assert_eq!(*cursor.lock().unwrap(), Some(300)); 347 - } 348 - 349 - #[tokio::test] 350 - async fn test_identity_message() { 351 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 352 - let cursor = Arc::new(Mutex::new(None)); 353 - // Valid identity event. 354 - let event_json = json!({ 355 - "did": "did:example:123", 356 - "time_us": 150, 357 - "kind": "identity", 358 - "commit": null, 359 - "identity": { 360 - "did": "did:example:123", 361 - "handle": "user", 362 - "seq": 1, 363 - "time": "2025-01-01T00:00:00Z" 364 - } 365 - }) 366 - .to_string(); 367 - let ingestors: HashMap<String, Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>> = 368 - HashMap::new(); 369 - 370 - let result = 371 - handle_message(Message::Text(event_json), &ingestors, reconnect_tx, cursor).await; 372 - assert!(result.is_ok()); 373 - } 374 - 375 - #[tokio::test] 376 - async fn test_close_message() { 377 - let (reconnect_tx, reconnect_rx) = setup_reconnect_channel(); 378 - let cursor = Arc::new(Mutex::new(None)); 379 - let ingestors: HashMap<String, Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>> = 380 - HashMap::new(); 381 - 382 - let result = handle_message(Message::Close(None), &ingestors, reconnect_tx, cursor).await; 383 - // Should return an error due to connection close. 384 - assert!(result.is_err()); 385 - // Verify that a reconnect signal was sent. 386 - let signal = reconnect_rx.recv_async().await; 387 - assert!(signal.is_ok()); 388 - } 389 - 390 - #[tokio::test] 391 - async fn test_invalid_json() { 392 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 393 - let cursor = Arc::new(Mutex::new(None)); 394 - let ingestors: HashMap<String, Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>> = 395 - HashMap::new(); 396 - 397 - let invalid_json = "this is not json".to_string(); 398 - let result = handle_message( 399 - Message::Text(invalid_json), 400 - &ingestors, 401 - reconnect_tx, 402 - cursor, 403 - ) 404 - .await; 405 - assert!(result.is_err()); 406 - } 407 - 408 - #[tokio::test] 409 - async fn test_cursor_not_updated_if_lower() { 410 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 411 - // Set an initial cursor value. 412 - let cursor = Arc::new(Mutex::new(Some(300))); 413 - let event_json = json!({ 414 - "did": "did:example:123", 415 - "time_us": 200, 416 - "kind": "commit", 417 - "commit": { 418 - "rev": "1", 419 - "operation": "create", 420 - "collection": "ns1", 421 - "rkey": "rkey1", 422 - "record": { "foo": "bar" }, 423 - "cid": "cid123" 424 - }, 425 - "identity": null 426 - }) 427 - .to_string(); 428 - 429 - // Use a dummy ingestor that does nothing. 430 - let mut ingestors: HashMap< 431 - String, 432 - Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>, 433 - > = HashMap::new(); 434 - ingestors.insert( 435 - "ns1".to_string(), 436 - Box::new(DummyIngestor { 437 - called: Arc::new(Mutex::new(false)), 438 - }), 439 - ); 440 - 441 - let result = handle_message( 442 - Message::Text(event_json), 443 - &ingestors, 444 - reconnect_tx, 445 - cursor.clone(), 446 - ) 447 - .await; 448 - assert!(result.is_ok()); 449 - // Cursor should remain unchanged. 450 - assert_eq!(*cursor.lock().unwrap(), Some(300)); 451 - } 452 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-22

services/rocketman/src/ingestion.rs

··· 1 - use anyhow::Result; 2 - use async_trait::async_trait; 3 - use serde_json::Value; 4 - use tracing::info; 5 - 6 - use crate::types::event::Event; 7 - 8 - #[async_trait] 9 - pub trait LexiconIngestor { 10 - async fn ingest(&self, message: Event<Value>) -> Result<()>; 11 - } 12 - 13 - pub struct DefaultLexiconIngestor; 14 - 15 - #[async_trait] 16 - impl LexiconIngestor for DefaultLexiconIngestor { 17 - async fn ingest(&self, message: Event<Value>) -> Result<()> { 18 - info!("Default lexicon processing: {:?}", message); 19 - // Process message for default lexicon. 20 - Ok(()) 21 - } 22 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-8

services/rocketman/src/lib.rs

··· 1 - // lib.rs 2 - pub mod connection; 3 - pub mod endpoints; 4 - pub mod handler; 5 - pub mod ingestion; 6 - pub mod options; 7 - pub mod time; 8 - pub mod types;

··· 0 0 0 0 0 0 0 0

-40

services/rocketman/src/options.rs

··· 1 - use bon::Builder; 2 - 3 - use crate::endpoints::JetstreamEndpoints; 4 - 5 - #[derive(Builder, Debug)] 6 - pub struct JetstreamOptions { 7 - #[builder(default)] 8 - pub ws_url: JetstreamEndpoints, 9 - #[builder(default)] 10 - pub max_retry_interval_seconds: u64, 11 - #[builder(default)] 12 - pub connection_success_time_seconds: u64, 13 - #[builder(default)] 14 - pub bound: usize, 15 - #[builder(default)] 16 - pub timeout_time_sec: usize, 17 - #[cfg(feature = "zstd")] 18 - #[builder(default = true)] 19 - pub compress: bool, 20 - pub wanted_collections: Option<Vec<String>>, 21 - pub wanted_dids: Option<Vec<String>>, 22 - pub cursor: Option<String>, 23 - } 24 - 25 - impl Default for JetstreamOptions { 26 - fn default() -> Self { 27 - Self { 28 - ws_url: JetstreamEndpoints::default(), 29 - max_retry_interval_seconds: 120, 30 - connection_success_time_seconds: 60, 31 - bound: 65536, 32 - timeout_time_sec: 40, 33 - #[cfg(feature = "zstd")] 34 - compress: true, 35 - wanted_collections: None, 36 - wanted_dids: None, 37 - cursor: None, 38 - } 39 - } 40 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-11

services/rocketman/src/time/mod.rs

··· 1 - use std::time::{Duration, Instant, SystemTime}; 2 - 3 - pub mod system_time; 4 - 5 - pub trait TimeProvider { 6 - fn new() -> Self; 7 - fn now(&self) -> SystemTime; // Get the current time 8 - fn elapsed(&self, earlier: SystemTime) -> Duration; // Calculate the elapsed time. 9 - fn instant_now(&self) -> Instant; // For compatibility with your existing code (if needed) 10 - fn instant_elapsed(&self, earlier: Instant) -> Duration; 11 - }

··· 0 0 0 0 0 0 0 0 0 0 0

-28

services/rocketman/src/time/system_time.rs

··· 1 - use std::time::{Duration, Instant, SystemTime}; 2 - 3 - use super::TimeProvider; 4 - 5 - #[derive(Default, Clone, Copy)] // Add these derives for ease of use 6 - pub struct SystemTimeProvider; // No fields needed, just a marker type 7 - 8 - impl TimeProvider for SystemTimeProvider { 9 - fn new() -> Self { 10 - Self 11 - } 12 - 13 - fn now(&self) -> SystemTime { 14 - SystemTime::now() 15 - } 16 - 17 - fn elapsed(&self, earlier: SystemTime) -> Duration { 18 - earlier.elapsed().unwrap_or_else(|_| Duration::from_secs(0)) 19 - } 20 - 21 - fn instant_now(&self) -> Instant { 22 - Instant::now() 23 - } 24 - 25 - fn instant_elapsed(&self, earlier: Instant) -> Duration { 26 - earlier.elapsed() 27 - } 28 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-116

services/rocketman/src/types/event.rs

··· 1 - use serde::{Deserialize, Deserializer, Serialize}; 2 - 3 - #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] 4 - #[serde(rename_all = "lowercase")] 5 - pub enum Kind { 6 - Account, 7 - Identity, 8 - Commit, 9 - Unknown(String), 10 - } 11 - 12 - #[derive(Debug, Serialize, Deserialize)] 13 - #[serde(rename_all = "snake_case")] 14 - pub struct Event<T> { 15 - pub did: String, 16 - pub time_us: Option<u64>, 17 - pub kind: Kind, 18 - pub commit: Option<Commit<T>>, 19 - pub identity: Option<Identity>, 20 - } 21 - 22 - #[derive(Debug, Serialize, Deserialize)] 23 - pub struct Identity { 24 - did: String, 25 - handle: Option<String>, 26 - seq: u64, 27 - time: String, 28 - } 29 - 30 - #[derive(Debug, Serialize, Deserialize)] 31 - #[serde(rename_all = "lowercase")] 32 - enum AccountStatus { 33 - TakenDown, 34 - Suspended, 35 - Deleted, 36 - Activated, 37 - } 38 - 39 - #[derive(Debug, Serialize, Deserialize)] 40 - pub struct Account { 41 - did: String, 42 - handle: String, 43 - seq: u64, 44 - time: String, 45 - status: AccountStatus, 46 - } 47 - 48 - #[derive(Debug, Serialize)] 49 - #[serde(rename_all = "camelCase")] 50 - pub struct Commit<T> { 51 - pub rev: String, 52 - pub operation: Operation, 53 - pub collection: String, 54 - pub rkey: String, 55 - pub record: Option<T>, 56 - pub cid: Option<String>, 57 - } 58 - 59 - #[derive(Debug, Serialize, Deserialize)] 60 - #[serde(rename_all = "lowercase")] 61 - pub enum Operation { 62 - Create, 63 - Update, 64 - Delete, 65 - } 66 - 67 - /// Enforce that record is None only when operation is 'delete' 68 - impl<'de, T> Deserialize<'de> for Commit<T> 69 - where 70 - T: Deserialize<'de>, 71 - { 72 - fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> 73 - where 74 - D: Deserializer<'de>, 75 - { 76 - // Helper struct to perform the deserialization. 77 - #[derive(Deserialize)] 78 - #[serde(rename_all = "camelCase")] 79 - struct Helper<T> { 80 - rev: String, 81 - operation: Operation, 82 - collection: String, 83 - rkey: String, 84 - record: Option<T>, 85 - cid: Option<String>, 86 - } 87 - 88 - let helper = Helper::deserialize(deserializer)?; 89 - 90 - match helper.operation { 91 - Operation::Delete => { 92 - if helper.record.is_some() || helper.cid.is_some() { 93 - return Err(<D::Error as serde::de::Error>::custom( 94 - "record and cid must be null when operation is delete", 95 - )); 96 - } 97 - } 98 - _ => { 99 - if helper.record.is_none() || helper.cid.is_none() { 100 - return Err(<D::Error as serde::de::Error>::custom( 101 - "record and cid must be present unless operation is delete", 102 - )); 103 - } 104 - } 105 - } 106 - 107 - Ok(Commit { 108 - rev: helper.rev, 109 - operation: helper.operation, 110 - collection: helper.collection, 111 - rkey: helper.rkey, 112 - record: helper.record, 113 - cid: helper.cid, 114 - }) 115 - } 116 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-1

services/rocketman/src/types/mod.rs

··· 1 - pub mod event;

··· 0

services/rocketman/zstd/dictionary

This is a binary file and will not be displayed.