-110

.github/workflows/security.yml

··· 1 1 - # yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json 2 2 - 3 3 - name: Security 4 4 - 5 5 - on: 6 6 - push: 7 7 - branches: [main, develop] 8 8 - pull_request: 9 9 - branches: [main, develop] 10 10 - schedule: 11 11 - # Run security checks daily at 2 AM sunday 12 12 - - cron: "0 2 * * 0" 13 13 - 14 14 - jobs: 15 15 - codeql-analysis: 16 16 - name: CodeQL Analysis 17 17 - runs-on: ubuntu-latest 18 18 - permissions: 19 19 - actions: read 20 20 - contents: read 21 21 - security-events: write 22 22 - 23 23 - steps: 24 24 - - name: Checkout repository 25 25 - uses: actions/checkout@v4 26 26 - 27 27 - - name: Initialize CodeQL 28 28 - uses: github/codeql-action/init@v3 29 29 - with: 30 30 - languages: "javascript,typescript,rust" 31 31 - queries: security-extended,security-and-quality 32 32 - 33 33 - - name: Setup environment for all languages 34 34 - uses: ./.github/actions/setup 35 35 - with: 36 36 - setup-node: "true" 37 37 - setup-rust: "true" 38 38 - 39 39 - - name: Perform a full build for CodeQL 40 40 - run: | 41 41 - echo "Building Node.js projects..." 42 42 - pnpm build 43 43 - echo "Building Rust projects..." 44 44 - (cd services && cargo build --all-features) 45 45 - (cd apps/aqua && cargo build --all-features) 46 46 - 47 47 - - name: Perform CodeQL Analysis 48 48 - uses: github/codeql-action/analyze@v3 49 49 - 50 50 - docker-security-scan: 51 51 - name: Docker Security Scan 52 52 - runs-on: ubuntu-latest 53 53 - if: github.event_name == 'push' || github.event_name == 'schedule' 54 54 - strategy: 55 55 - matrix: 56 56 - service: [aqua, cadet] 57 57 - steps: 58 58 - - name: Checkout repository 59 59 - uses: actions/checkout@v4 60 60 - 61 61 - - name: Setup environment 62 62 - uses: ./.github/actions/setup 63 63 - with: 64 64 - setup-node: "true" 65 65 - lexicons-only-rust: "true" 66 66 - 67 67 - - name: Set up Docker Buildx 68 68 - uses: docker/setup-buildx-action@v3 69 69 - 70 70 - - name: Build Docker image 71 71 - uses: docker/build-push-action@v5 72 72 - with: 73 73 - context: . 74 74 - file: ${{ matrix.service == 'aqua' && './apps/aqua/Dockerfile' || './services/cadet/Dockerfile' }} 75 75 - load: true 76 76 - tags: ${{ matrix.service }}:latest 77 77 - cache-from: type=gha,scope=${{ matrix.service }} 78 78 - cache-to: type=gha,mode=max,scope=${{ matrix.service }} 79 79 - 80 80 - - name: Run Trivy vulnerability scanner 81 81 - uses: aquasecurity/trivy-action@master 82 82 - with: 83 83 - image-ref: "${{ matrix.service }}:latest" 84 84 - format: "sarif" 85 85 - output: "trivy-results-${{ matrix.service }}.sarif" 86 86 - severity: "CRITICAL,HIGH" 87 87 - exit-code: "1" 88 88 - 89 89 - - name: Upload Trivy scan results to GitHub Security tab 90 90 - uses: github/codeql-action/upload-sarif@v3 91 91 - if: always() 92 92 - with: 93 93 - sarif_file: "trivy-results-${{ matrix.service }}.sarif" 94 94 - 95 95 - secrets-scan: 96 96 - name: Secrets Scan 97 97 - runs-on: ubuntu-latest 98 98 - steps: 99 99 - - name: Checkout repository 100 100 - uses: actions/checkout@v4 101 101 - with: 102 102 - fetch-depth: 0 103 103 - 104 104 - - name: Run TruffleHog OSS 105 105 - uses: trufflesecurity/trufflehog@main 106 106 - with: 107 107 - path: ./ 108 108 - base: main 109 109 - head: HEAD 110 110 - extra_args: --debug --only-verified

+189 -42

Cargo.lock

··· 124 124 "atmst", 125 125 "atrium-api", 126 126 "axum", 127 127 - "base64", 127 127 + "base64 0.22.1", 128 128 "chrono", 129 129 "clap", 130 130 "dotenvy", ··· 247 247 "atrium-common", 248 248 "atrium-xrpc", 249 249 "chrono", 250 250 - "http", 250 250 + "http 1.3.1", 251 251 "ipld-core", 252 252 "langtag", 253 253 "regex", ··· 280 280 source = "registry+https://github.com/rust-lang/crates.io-index" 281 281 checksum = "0216ad50ce34e9ff982e171c3659e65dedaa2ed5ac2994524debdc9a9647ffa8" 282 282 dependencies = [ 283 283 - "http", 283 283 + "http 1.3.1", 284 284 "serde", 285 285 "serde_html_form", 286 286 "serde_json", ··· 328 328 "bytes", 329 329 "form_urlencoded", 330 330 "futures-util", 331 331 - "http", 331 331 + "http 1.3.1", 332 332 "http-body", 333 333 "http-body-util", 334 334 "hyper", ··· 361 361 dependencies = [ 362 362 "bytes", 363 363 "futures-core", 364 364 - "http", 364 364 + "http 1.3.1", 365 365 "http-body", 366 366 "http-body-util", 367 367 "mime", ··· 422 422 423 423 [[package]] 424 424 name = "base64" 425 425 + version = "0.21.7" 426 426 + source = "registry+https://github.com/rust-lang/crates.io-index" 427 427 + checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" 428 428 + 429 429 + [[package]] 430 430 + name = "base64" 425 431 version = "0.22.1" 426 432 source = "registry+https://github.com/rust-lang/crates.io-index" 427 433 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" ··· 568 574 "async-trait", 569 575 "atmst", 570 576 "atrium-api", 571 571 - "base64", 577 577 + "base64 0.22.1", 572 578 "chrono", 573 579 "cid 0.11.1", 574 580 "dotenvy", ··· 590 596 "sqlx", 591 597 "time", 592 598 "tokio", 593 593 - "tokio-tungstenite", 599 599 + "tokio-tungstenite 0.24.0", 594 600 "tracing", 595 601 "tracing-subscriber", 596 602 "types", ··· 826 832 827 833 [[package]] 828 834 name = "core-foundation" 835 835 + version = "0.9.4" 836 836 + source = "registry+https://github.com/rust-lang/crates.io-index" 837 837 + checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" 838 838 + dependencies = [ 839 839 + "core-foundation-sys", 840 840 + "libc", 841 841 + ] 842 842 + 843 843 + [[package]] 844 844 + name = "core-foundation" 829 845 version = "0.10.1" 830 846 source = "registry+https://github.com/rust-lang/crates.io-index" 831 847 checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" ··· 1504 1520 "fnv", 1505 1521 "futures-core", 1506 1522 "futures-sink", 1507 1507 - "http", 1523 1523 + "http 1.3.1", 1508 1524 "indexmap", 1509 1525 "slab", 1510 1526 "tokio", ··· 1579 1595 1580 1596 [[package]] 1581 1597 name = "http" 1598 1598 + version = "0.2.12" 1599 1599 + source = "registry+https://github.com/rust-lang/crates.io-index" 1600 1600 + checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" 1601 1601 + dependencies = [ 1602 1602 + "bytes", 1603 1603 + "fnv", 1604 1604 + "itoa", 1605 1605 + ] 1606 1606 + 1607 1607 + [[package]] 1608 1608 + name = "http" 1582 1609 version = "1.3.1" 1583 1610 source = "registry+https://github.com/rust-lang/crates.io-index" 1584 1611 checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" ··· 1595 1622 checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" 1596 1623 dependencies = [ 1597 1624 "bytes", 1598 1598 - "http", 1625 1625 + "http 1.3.1", 1599 1626 ] 1600 1627 1601 1628 [[package]] ··· 1606 1633 dependencies = [ 1607 1634 "bytes", 1608 1635 "futures-core", 1609 1609 - "http", 1636 1636 + "http 1.3.1", 1610 1637 "http-body", 1611 1638 "pin-project-lite", 1612 1639 ] ··· 1633 1660 "futures-channel", 1634 1661 "futures-util", 1635 1662 "h2", 1636 1636 - "http", 1663 1663 + "http 1.3.1", 1637 1664 "http-body", 1638 1665 "httparse", 1639 1666 "httpdate", ··· 1650 1677 source = "registry+https://github.com/rust-lang/crates.io-index" 1651 1678 checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" 1652 1679 dependencies = [ 1653 1653 - "http", 1680 1680 + "http 1.3.1", 1654 1681 "hyper", 1655 1682 "hyper-util", 1656 1656 - "rustls", 1657 1657 - "rustls-native-certs", 1683 1683 + "rustls 0.23.31", 1684 1684 + "rustls-native-certs 0.8.1", 1658 1685 "rustls-pki-types", 1659 1686 "tokio", 1660 1660 - "tokio-rustls", 1687 1687 + "tokio-rustls 0.26.2", 1661 1688 "tower-service", 1662 1689 "webpki-roots 1.0.2", 1663 1690 ] ··· 1668 1695 source = "registry+https://github.com/rust-lang/crates.io-index" 1669 1696 checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" 1670 1697 dependencies = [ 1671 1671 - "base64", 1698 1698 + "base64 0.22.1", 1672 1699 "bytes", 1673 1700 "futures-channel", 1674 1701 "futures-core", 1675 1702 "futures-util", 1676 1676 - "http", 1703 1703 + "http 1.3.1", 1677 1704 "http-body", 1678 1705 "hyper", 1679 1706 "ipnet", ··· 2225 2252 source = "registry+https://github.com/rust-lang/crates.io-index" 2226 2253 checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" 2227 2254 dependencies = [ 2228 2228 - "base64", 2255 2255 + "base64 0.22.1", 2229 2256 "http-body-util", 2230 2257 "hyper", 2231 2258 "hyper-rustls", ··· 2319 2346 "bytes", 2320 2347 "encoding_rs", 2321 2348 "futures-util", 2322 2322 - "http", 2349 2349 + "http 1.3.1", 2323 2350 "httparse", 2324 2351 "memchr", 2325 2352 "mime", ··· 2793 2820 "quinn-proto", 2794 2821 "quinn-udp", 2795 2822 "rustc-hash 2.1.1", 2796 2796 - "rustls", 2823 2823 + "rustls 0.23.31", 2797 2824 "socket2 0.5.10", 2798 2825 "thiserror 2.0.12", 2799 2826 "tokio", ··· 2813 2840 "rand 0.9.2", 2814 2841 "ring", 2815 2842 "rustc-hash 2.1.1", 2816 2816 - "rustls", 2843 2843 + "rustls 0.23.31", 2817 2844 "rustls-pki-types", 2818 2845 "slab", 2819 2846 "thiserror 2.0.12", ··· 3024 3051 checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" 3025 3052 dependencies = [ 3026 3053 "async-compression", 3027 3027 - "base64", 3054 3054 + "base64 0.22.1", 3028 3055 "bytes", 3029 3056 "futures-core", 3030 3057 "futures-util", 3031 3031 - "http", 3058 3058 + "http 1.3.1", 3032 3059 "http-body", 3033 3060 "http-body-util", 3034 3061 "hyper", ··· 3039 3066 "percent-encoding", 3040 3067 "pin-project-lite", 3041 3068 "quinn", 3042 3042 - "rustls", 3069 3069 + "rustls 0.23.31", 3043 3070 "rustls-pki-types", 3044 3071 "serde", 3045 3072 "serde_json", 3046 3073 "serde_urlencoded", 3047 3074 "sync_wrapper", 3048 3075 "tokio", 3049 3049 - "tokio-rustls", 3076 3076 + "tokio-rustls 0.26.2", 3050 3077 "tokio-util", 3051 3078 "tower", 3052 3079 "tower-http", ··· 3095 3122 [[package]] 3096 3123 name = "rocketman" 3097 3124 version = "0.2.3" 3125 3125 + source = "registry+https://github.com/rust-lang/crates.io-index" 3126 3126 + checksum = "9928fe43979c19ff1f46f7920c30b76dfcead7a4d571c9836c4d02da8587f844" 3098 3127 dependencies = [ 3099 3128 "anyhow", 3100 3129 "async-trait", ··· 3102 3131 "derive_builder", 3103 3132 "flume", 3104 3133 "futures-util", 3105 3105 - "metrics 0.23.1", 3134 3134 + "metrics 0.24.2", 3106 3135 "rand 0.8.5", 3107 3136 "serde", 3108 3137 "serde_json", 3109 3138 "tokio", 3110 3110 - "tokio-tungstenite", 3139 3139 + "tokio-tungstenite 0.20.1", 3111 3140 "tracing", 3112 3141 "tracing-subscriber", 3113 3142 "url", ··· 3189 3218 3190 3219 [[package]] 3191 3220 name = "rustls" 3221 3221 + version = "0.21.12" 3222 3222 + source = "registry+https://github.com/rust-lang/crates.io-index" 3223 3223 + checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" 3224 3224 + dependencies = [ 3225 3225 + "log", 3226 3226 + "ring", 3227 3227 + "rustls-webpki 0.101.7", 3228 3228 + "sct", 3229 3229 + ] 3230 3230 + 3231 3231 + [[package]] 3232 3232 + name = "rustls" 3192 3233 version = "0.23.31" 3193 3234 source = "registry+https://github.com/rust-lang/crates.io-index" 3194 3235 checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" ··· 3197 3238 "once_cell", 3198 3239 "ring", 3199 3240 "rustls-pki-types", 3200 3200 - "rustls-webpki", 3241 3241 + "rustls-webpki 0.103.4", 3201 3242 "subtle", 3202 3243 "zeroize", 3203 3244 ] 3204 3245 3205 3246 [[package]] 3206 3247 name = "rustls-native-certs" 3248 3248 + version = "0.6.3" 3249 3249 + source = "registry+https://github.com/rust-lang/crates.io-index" 3250 3250 + checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" 3251 3251 + dependencies = [ 3252 3252 + "openssl-probe", 3253 3253 + "rustls-pemfile", 3254 3254 + "schannel", 3255 3255 + "security-framework 2.11.1", 3256 3256 + ] 3257 3257 + 3258 3258 + [[package]] 3259 3259 + name = "rustls-native-certs" 3207 3260 version = "0.8.1" 3208 3261 source = "registry+https://github.com/rust-lang/crates.io-index" 3209 3262 checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" ··· 3211 3264 "openssl-probe", 3212 3265 "rustls-pki-types", 3213 3266 "schannel", 3214 3214 - "security-framework", 3267 3267 + "security-framework 3.2.0", 3268 3268 + ] 3269 3269 + 3270 3270 + [[package]] 3271 3271 + name = "rustls-pemfile" 3272 3272 + version = "1.0.4" 3273 3273 + source = "registry+https://github.com/rust-lang/crates.io-index" 3274 3274 + checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" 3275 3275 + dependencies = [ 3276 3276 + "base64 0.21.7", 3215 3277 ] 3216 3278 3217 3279 [[package]] ··· 3222 3284 dependencies = [ 3223 3285 "web-time", 3224 3286 "zeroize", 3287 3287 + ] 3288 3288 + 3289 3289 + [[package]] 3290 3290 + name = "rustls-webpki" 3291 3291 + version = "0.101.7" 3292 3292 + source = "registry+https://github.com/rust-lang/crates.io-index" 3293 3293 + checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" 3294 3294 + dependencies = [ 3295 3295 + "ring", 3296 3296 + "untrusted", 3225 3297 ] 3226 3298 3227 3299 [[package]] ··· 3270 3342 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 3271 3343 3272 3344 [[package]] 3345 3345 + name = "sct" 3346 3346 + version = "0.7.1" 3347 3347 + source = "registry+https://github.com/rust-lang/crates.io-index" 3348 3348 + checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" 3349 3349 + dependencies = [ 3350 3350 + "ring", 3351 3351 + "untrusted", 3352 3352 + ] 3353 3353 + 3354 3354 + [[package]] 3273 3355 name = "sec1" 3274 3356 version = "0.7.3" 3275 3357 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3285 3367 3286 3368 [[package]] 3287 3369 name = "security-framework" 3370 3370 + version = "2.11.1" 3371 3371 + source = "registry+https://github.com/rust-lang/crates.io-index" 3372 3372 + checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" 3373 3373 + dependencies = [ 3374 3374 + "bitflags 2.9.1", 3375 3375 + "core-foundation 0.9.4", 3376 3376 + "core-foundation-sys", 3377 3377 + "libc", 3378 3378 + "security-framework-sys", 3379 3379 + ] 3380 3380 + 3381 3381 + [[package]] 3382 3382 + name = "security-framework" 3288 3383 version = "3.2.0" 3289 3384 source = "registry+https://github.com/rust-lang/crates.io-index" 3290 3385 checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" 3291 3386 dependencies = [ 3292 3387 "bitflags 2.9.1", 3293 3293 - "core-foundation", 3388 3388 + "core-foundation 0.10.1", 3294 3389 "core-foundation-sys", 3295 3390 "libc", 3296 3391 "security-framework-sys", ··· 3565 3660 source = "registry+https://github.com/rust-lang/crates.io-index" 3566 3661 checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" 3567 3662 dependencies = [ 3568 3568 - "base64", 3663 3663 + "base64 0.22.1", 3569 3664 "bytes", 3570 3665 "crc", 3571 3666 "crossbeam-queue", ··· 3582 3677 "memchr", 3583 3678 "once_cell", 3584 3679 "percent-encoding", 3585 3585 - "rustls", 3680 3680 + "rustls 0.23.31", 3586 3681 "serde", 3587 3682 "serde_json", 3588 3683 "sha2", ··· 3642 3737 checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" 3643 3738 dependencies = [ 3644 3739 "atoi", 3645 3645 - "base64", 3740 3740 + "base64 0.22.1", 3646 3741 "bitflags 2.9.1", 3647 3742 "byteorder", 3648 3743 "bytes", ··· 3686 3781 checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" 3687 3782 dependencies = [ 3688 3783 "atoi", 3689 3689 - "base64", 3784 3784 + "base64 0.22.1", 3690 3785 "bitflags 2.9.1", 3691 3786 "byteorder", 3692 3787 "crc", ··· 4041 4136 4042 4137 [[package]] 4043 4138 name = "tokio-rustls" 4139 4139 + version = "0.24.1" 4140 4140 + source = "registry+https://github.com/rust-lang/crates.io-index" 4141 4141 + checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" 4142 4142 + dependencies = [ 4143 4143 + "rustls 0.21.12", 4144 4144 + "tokio", 4145 4145 + ] 4146 4146 + 4147 4147 + [[package]] 4148 4148 + name = "tokio-rustls" 4044 4149 version = "0.26.2" 4045 4150 source = "registry+https://github.com/rust-lang/crates.io-index" 4046 4151 checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" 4047 4152 dependencies = [ 4048 4048 - "rustls", 4153 4153 + "rustls 0.23.31", 4049 4154 "tokio", 4050 4155 ] 4051 4156 ··· 4062 4167 4063 4168 [[package]] 4064 4169 name = "tokio-tungstenite" 4170 4170 + version = "0.20.1" 4171 4171 + source = "registry+https://github.com/rust-lang/crates.io-index" 4172 4172 + checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" 4173 4173 + dependencies = [ 4174 4174 + "futures-util", 4175 4175 + "log", 4176 4176 + "rustls 0.21.12", 4177 4177 + "rustls-native-certs 0.6.3", 4178 4178 + "tokio", 4179 4179 + "tokio-rustls 0.24.1", 4180 4180 + "tungstenite 0.20.1", 4181 4181 + "webpki-roots 0.25.4", 4182 4182 + ] 4183 4183 + 4184 4184 + [[package]] 4185 4185 + name = "tokio-tungstenite" 4065 4186 version = "0.24.0" 4066 4187 source = "registry+https://github.com/rust-lang/crates.io-index" 4067 4188 checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" 4068 4189 dependencies = [ 4069 4190 "futures-util", 4070 4191 "log", 4071 4071 - "rustls", 4192 4192 + "rustls 0.23.31", 4072 4193 "rustls-pki-types", 4073 4194 "tokio", 4074 4074 - "tokio-rustls", 4075 4075 - "tungstenite", 4195 4195 + "tokio-rustls 0.26.2", 4196 4196 + "tungstenite 0.24.0", 4076 4197 "webpki-roots 0.26.11", 4077 4198 ] 4078 4199 ··· 4140 4261 "bitflags 2.9.1", 4141 4262 "bytes", 4142 4263 "futures-util", 4143 4143 - "http", 4264 4264 + "http 1.3.1", 4144 4265 "http-body", 4145 4266 "iri-string", 4146 4267 "pin-project-lite", ··· 4242 4363 4243 4364 [[package]] 4244 4365 name = "tungstenite" 4366 4366 + version = "0.20.1" 4367 4367 + source = "registry+https://github.com/rust-lang/crates.io-index" 4368 4368 + checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" 4369 4369 + dependencies = [ 4370 4370 + "byteorder", 4371 4371 + "bytes", 4372 4372 + "data-encoding", 4373 4373 + "http 0.2.12", 4374 4374 + "httparse", 4375 4375 + "log", 4376 4376 + "rand 0.8.5", 4377 4377 + "rustls 0.21.12", 4378 4378 + "sha1", 4379 4379 + "thiserror 1.0.69", 4380 4380 + "url", 4381 4381 + "utf-8", 4382 4382 + ] 4383 4383 + 4384 4384 + [[package]] 4385 4385 + name = "tungstenite" 4245 4386 version = "0.24.0" 4246 4387 source = "registry+https://github.com/rust-lang/crates.io-index" 4247 4388 checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" ··· 4249 4390 "byteorder", 4250 4391 "bytes", 4251 4392 "data-encoding", 4252 4252 - "http", 4393 4393 + "http 1.3.1", 4253 4394 "httparse", 4254 4395 "log", 4255 4396 "rand 0.8.5", 4256 4256 - "rustls", 4397 4397 + "rustls 0.23.31", 4257 4398 "rustls-pki-types", 4258 4399 "sha1", 4259 4400 "thiserror 1.0.69", ··· 4273 4414 "atrium-api", 4274 4415 "atrium-xrpc", 4275 4416 "chrono", 4276 4276 - "http", 4417 4417 + "http 1.3.1", 4277 4418 "ipld-core", 4278 4419 "langtag", 4279 4420 "regex", ··· 4569 4710 "js-sys", 4570 4711 "wasm-bindgen", 4571 4712 ] 4713 4713 + 4714 4714 + [[package]] 4715 4715 + name = "webpki-roots" 4716 4716 + version = "0.25.4" 4717 4717 + source = "registry+https://github.com/rust-lang/crates.io-index" 4718 4718 + checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" 4572 4719 4573 4720 [[package]] 4574 4721 name = "webpki-roots"

+2 -7

Cargo.toml

··· 1 1 [workspace] 2 2 - members = [ 3 3 - "apps/aqua", 4 4 - "services/cadet", 5 5 - "services/rocketman", 6 6 - "tools/teal-cli", 7 7 - ] 2 2 + members = ["apps/aqua", "services/cadet", "tools/teal-cli"] 8 3 resolver = "2" 9 4 10 5 [workspace.dependencies] ··· 51 46 chrono = "0.4" 52 47 uuid = { version = "1.0", features = ["v4", "serde"] } 53 48 types = { path = "services/types" } 54 54 - rocketman = { path = "services/rocketman" } 49 49 + rocketman = "0.2.3" 55 50 56 51 # CAR and IPLD dependencies 57 52 iroh-car = "0.5"

-24

compose.db-test.yml

··· 1 1 - version: "3.8" 2 2 - 3 3 - services: 4 4 - postgres: 5 5 - image: postgres:latest 6 6 - container_name: postgres_test_db 7 7 - environment: 8 8 - POSTGRES_USER: postgres 9 9 - POSTGRES_PASSWORD: testpass123 10 10 - POSTGRES_DB: teal_test 11 11 - ports: 12 12 - - "5433:5432" 13 13 - volumes: 14 14 - - postgres_test_data:/var/lib/postgresql/data 15 15 - networks: 16 16 - - test_network 17 17 - command: postgres -c log_statement=all -c log_destination=stderr 18 18 - 19 19 - networks: 20 20 - test_network: 21 21 - driver: bridge 22 22 - 23 23 - volumes: 24 24 - postgres_test_data:

+2 -2

services/Cargo.toml

··· 1 1 [workspace] 2 2 - members = ["cadet", "rocketman", "satellite", "types"] 2 2 + members = ["cadet", "satellite", "types"] 3 3 resolver = "2" 4 4 5 5 [workspace.dependencies] ··· 32 32 chrono = { version = "0.4", features = ["serde"] } 33 33 uuid = { version = "1.0", features = ["v4", "serde"] } 34 34 types = { path = "types" } 35 35 - rocketman = { path = "rocketman" } 35 35 + rocketman = "0.2.5" 36 36 37 37 # CAR and IPLD dependencies 38 38 iroh-car = "0.4"

-34

services/rocketman/Cargo.toml

··· 1 1 - [package] 2 2 - name = "rocketman" 3 3 - version = "0.2.3" 4 4 - edition = "2021" 5 5 - 6 6 - license = "MIT" 7 7 - authors = ["Natalie B. <nat@natalie.sh>"] 8 8 - repository = "https://github.com/espeon/cadet" 9 9 - 10 10 - readme = "readme.md" 11 11 - 12 12 - description = "A modular(ish) jetstream consumer." 13 13 - 14 14 - [dependencies] 15 15 - tokio.workspace = true 16 16 - tokio-tungstenite.workspace = true 17 17 - futures-util = "0.3" 18 18 - url.workspace = true 19 19 - rand.workspace = true 20 20 - tracing.workspace = true 21 21 - tracing-subscriber.workspace = true 22 22 - metrics.workspace = true 23 23 - derive_builder = "0.20.2" 24 24 - bon = "3.3.2" 25 25 - serde = { workspace = true, features = ["derive"] } 26 26 - serde_json.workspace = true 27 27 - flume.workspace = true 28 28 - anyhow.workspace = true 29 29 - async-trait.workspace = true 30 30 - zstd = { version = "0.13.3", optional = true } 31 31 - 32 32 - [features] 33 33 - default = ["zstd"] 34 34 - zstd = ["dep:zstd"]

-76

services/rocketman/examples/spew-bsky-posts.rs

··· 1 1 - use async_trait::async_trait; 2 2 - use rocketman::{ 3 3 - connection::JetstreamConnection, 4 4 - handler, 5 5 - ingestion::LexiconIngestor, 6 6 - options::JetstreamOptions, 7 7 - types::event::{Commit, Event}, 8 8 - }; 9 9 - use serde_json::Value; 10 10 - use std::{collections::HashMap, sync::Arc, sync::Mutex}; 11 11 - 12 12 - #[tokio::main] 13 13 - async fn main() { 14 14 - // init the builder 15 15 - let opts = JetstreamOptions::builder() 16 16 - // your EXACT nsids 17 17 - .wanted_collections(vec!["app.bsky.feed.post".to_string()]) 18 18 - .build(); 19 19 - // create the jetstream connector 20 20 - let jetstream = JetstreamConnection::new(opts); 21 21 - 22 22 - // create your ingestors 23 23 - let mut ingestors: HashMap<String, Box<dyn LexiconIngestor + Send + Sync>> = HashMap::new(); 24 24 - ingestors.insert( 25 25 - // your EXACT nsid 26 26 - "app.bsky.feed.post".to_string(), 27 27 - Box::new(MyCoolIngestor), 28 28 - ); 29 29 - 30 30 - // tracks the last message we've processed 31 31 - let cursor: Arc<Mutex<Option<u64>>> = Arc::new(Mutex::new(None)); 32 32 - 33 33 - // get channels 34 34 - let msg_rx = jetstream.get_msg_rx(); 35 35 - let reconnect_tx = jetstream.get_reconnect_tx(); 36 36 - 37 37 - // spawn a task to process messages from the queue. 38 38 - // this is a simple implementation, you can use a more complex one based on needs. 39 39 - let c_cursor = cursor.clone(); 40 40 - tokio::spawn(async move { 41 41 - while let Ok(message) = msg_rx.recv_async().await { 42 42 - if let Err(e) = 43 43 - handler::handle_message(message, &ingestors, reconnect_tx.clone(), c_cursor.clone()) 44 44 - .await 45 45 - { 46 46 - eprintln!("Error processing message: {}", e); 47 47 - }; 48 48 - } 49 49 - }); 50 50 - 51 51 - // connect to jetstream 52 52 - // retries internally, but may fail if there is an extreme error. 53 53 - if let Err(e) = jetstream.connect(cursor.clone()).await { 54 54 - eprintln!("Failed to connect to Jetstream: {}", e); 55 55 - std::process::exit(1); 56 56 - } 57 57 - } 58 58 - 59 59 - pub struct MyCoolIngestor; 60 60 - 61 61 - /// A cool ingestor implementation. Will just print the message. Does not do verification. 62 62 - #[async_trait] 63 63 - impl LexiconIngestor for MyCoolIngestor { 64 64 - async fn ingest(&self, message: Event<Value>) -> anyhow::Result<()> { 65 65 - if let Some(Commit { 66 66 - record: Some(record), 67 67 - .. 68 68 - }) = message.commit 69 69 - { 70 70 - if let Some(Value::String(text)) = record.get("text") { 71 71 - println!("{text:?}"); 72 72 - } 73 73 - } 74 74 - Ok(()) 75 75 - } 76 76 - }

-11

services/rocketman/package.json

··· 1 1 - { 2 2 - "name": "@repo/rocketman", 3 3 - "private": true, 4 4 - "scripts": { 5 5 - "build": "cargo build --release", 6 6 - "build:rust": "cargo build --release", 7 7 - "dev": "cargo watch -x 'run'", 8 8 - "test": "cargo test", 9 9 - "test:rust": "cargo test" 10 10 - } 11 11 - }

-74

services/rocketman/readme.md

··· 1 1 - ## Rocketman 2 2 - 3 3 - A modular(ish) jetstream consumer. Backed by Tungstenite. 4 4 - 5 5 - 6 6 - ### Installation 7 7 - ```toml 8 8 - [dependencies] 9 9 - rocketman = "latest" # pyt the latest version here 10 10 - tokio = { version = "1", features = ["macros", "rt-multi-thread"] } 11 11 - ``` 12 12 - ### Usage 13 13 - ```rs 14 14 - #[tokio::main] 15 15 - async fn main() { 16 16 - // init the builder 17 17 - let opts = JetstreamOptions::builder() 18 18 - // your EXACT nsids 19 19 - .wanted_collections(vec!["com.example.cool.nsid".to_string()]) 20 20 - .build(); 21 21 - // create the jetstream connector 22 22 - let jetstream = JetstreamConnection::new(opts); 23 23 - 24 24 - // create your ingestors 25 25 - let mut ingestors: HashMap<String, Box<dyn LexiconIngestor + Send + Sync>> = HashMap::new(); 26 26 - ingestors.insert( 27 27 - // your EXACT nsid 28 28 - "com.example.cool.nsid".to_string(), 29 29 - Box::new(MyCoolIngestor), 30 30 - ); 31 31 - 32 32 - 33 33 - // tracks the last message we've processed 34 34 - let cursor: Arc<Mutex<Option<u64>>> = Arc::new(Mutex::new(None)); 35 35 - 36 36 - // get channels 37 37 - let msg_rx = jetstream.get_msg_rx(); 38 38 - let reconnect_tx = jetstream.get_reconnect_tx(); 39 39 - 40 40 - // spawn a task to process messages from the queue. 41 41 - // this is a simple implementation, you can use a more complex one based on needs. 42 42 - let c_cursor = cursor.clone(); 43 43 - tokio::spawn(async move { 44 44 - while let Ok(message) = msg_rx.recv_async().await { 45 45 - if let Err(e) = 46 46 - handler::handle_message(message, &ingestors, reconnect_tx.clone(), c_cursor.clone()) 47 47 - .await 48 48 - { 49 49 - error!("Error processing message: {}", e); 50 50 - }; 51 51 - } 52 52 - }); 53 53 - 54 54 - // connect to jetstream 55 55 - // retries internally, but may fail if there is an extreme error. 56 56 - if let Err(e) = jetstream.connect(cursor.clone()).await { 57 57 - error!("Failed to connect to Jetstream: {}", e); 58 58 - std::process::exit(1); 59 59 - } 60 60 - } 61 61 - 62 62 - pub struct MyCoolIngestor; 63 63 - 64 64 - /// A cool ingestor implementation. Will just print the message. Does not do verification. 65 65 - impl LexiconIngestor for MyCoolIngestor { 66 66 - async fn ingest(&self, message: Event<Value>) -> Result<()> { 67 67 - info!("{:?}", message); 68 68 - // Process message for default lexicon. 69 69 - Ok(()) 70 70 - } 71 71 - } 72 72 - ``` 73 73 - ### gratz 74 74 - Based heavily on [phil's jetstream consumer on atcosm constellation.](https://github.com/atcosm/links/blob/main/constellation/src/consumer/jetstream.rs)

-335

services/rocketman/src/connection.rs

··· 1 1 - use flume::{Receiver, Sender}; 2 2 - use futures_util::StreamExt; 3 3 - use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; 4 4 - use std::cmp::{max, min}; 5 5 - use std::sync::{Arc, Mutex}; 6 6 - use std::time::Instant; 7 7 - use tokio::time::{sleep, Duration}; 8 8 - use tokio_tungstenite::{connect_async, tungstenite::Message}; 9 9 - use tracing::{error, info}; 10 10 - use url::Url; 11 11 - 12 12 - use crate::options::JetstreamOptions; 13 13 - use crate::time::system_time::SystemTimeProvider; 14 14 - use crate::time::TimeProvider; 15 15 - 16 16 - pub struct JetstreamConnection { 17 17 - pub opts: JetstreamOptions, 18 18 - reconnect_tx: flume::Sender<()>, 19 19 - reconnect_rx: flume::Receiver<()>, 20 20 - msg_tx: flume::Sender<Message>, 21 21 - msg_rx: flume::Receiver<Message>, 22 22 - } 23 23 - 24 24 - impl JetstreamConnection { 25 25 - pub fn new(opts: JetstreamOptions) -> Self { 26 26 - let (reconnect_tx, reconnect_rx) = flume::bounded(opts.bound); 27 27 - let (msg_tx, msg_rx) = flume::bounded(opts.bound); 28 28 - Self { 29 29 - opts, 30 30 - reconnect_tx, 31 31 - reconnect_rx, 32 32 - msg_tx, 33 33 - msg_rx, 34 34 - } 35 35 - } 36 36 - 37 37 - pub fn get_reconnect_tx(&self) -> Sender<()> { 38 38 - self.reconnect_tx.clone() 39 39 - } 40 40 - 41 41 - pub fn get_msg_rx(&self) -> Receiver<Message> { 42 42 - self.msg_rx.clone() 43 43 - } 44 44 - 45 45 - fn build_ws_url(&self, cursor: Arc<Mutex<Option<u64>>>) -> String { 46 46 - let mut url = Url::parse(&self.opts.ws_url.to_string()).unwrap(); 47 47 - 48 48 - // Append query params 49 49 - if let Some(ref cols) = self.opts.wanted_collections { 50 50 - for col in cols { 51 51 - url.query_pairs_mut().append_pair("wantedCollections", col); 52 52 - } 53 53 - } 54 54 - if let Some(ref dids) = self.opts.wanted_dids { 55 55 - for did in dids { 56 56 - url.query_pairs_mut().append_pair("wantedDids", did); 57 57 - } 58 58 - } 59 59 - if let Some(cursor) = cursor.lock().unwrap().as_ref() { 60 60 - url.query_pairs_mut() 61 61 - .append_pair("cursor", &cursor.to_string()); 62 62 - } 63 63 - #[cfg(feature = "zstd")] 64 64 - if self.opts.compress { 65 65 - url.query_pairs_mut().append_pair("compress", "true"); 66 66 - } 67 67 - 68 68 - url.to_string() 69 69 - } 70 70 - 71 71 - pub async fn connect( 72 72 - &self, 73 73 - cursor: Arc<Mutex<Option<u64>>>, 74 74 - ) -> Result<(), Box<dyn std::error::Error>> { 75 75 - describe_counter!( 76 76 - "jetstream.connection.attempt", 77 77 - Unit::Count, 78 78 - "attempts to connect to jetstream service" 79 79 - ); 80 80 - describe_counter!( 81 81 - "jetstream.connection.error", 82 82 - Unit::Count, 83 83 - "errors connecting to jetstream service" 84 84 - ); 85 85 - describe_histogram!( 86 86 - "jetstream.connection.duration", 87 87 - Unit::Seconds, 88 88 - "Time connected to jetstream service" 89 89 - ); 90 90 - describe_counter!( 91 91 - "jetstream.connection.reconnect", 92 92 - Unit::Count, 93 93 - "reconnects to jetstream service" 94 94 - ); 95 95 - let mut retry_interval = 1; 96 96 - 97 97 - let time_provider = SystemTimeProvider::new(); 98 98 - 99 99 - let mut start_time = time_provider.now(); 100 100 - 101 101 - loop { 102 102 - counter!("jetstream.connection.attempt").increment(1); 103 103 - info!("Connecting to {}", self.opts.ws_url); 104 104 - let start = Instant::now(); 105 105 - 106 106 - let ws_url = self.build_ws_url(cursor.clone()); 107 107 - 108 108 - match connect_async(ws_url).await { 109 109 - Ok((ws_stream, response)) => { 110 110 - let elapsed = start.elapsed(); 111 111 - info!("Connected. HTTP status: {}", response.status()); 112 112 - 113 113 - let (_, mut read) = ws_stream.split(); 114 114 - 115 115 - loop { 116 116 - // Inner loop to handle messages, reconnect signals, and receive timeout 117 117 - let receive_timeout = 118 118 - sleep(Duration::from_secs(self.opts.timeout_time_sec as u64)); 119 119 - tokio::pin!(receive_timeout); 120 120 - 121 121 - loop { 122 122 - tokio::select! { 123 123 - message_result = read.next() => { 124 124 - match message_result { 125 125 - Some(message) => { 126 126 - // Reset timeout on message received 127 127 - receive_timeout.as_mut().reset(tokio::time::Instant::now() + Duration::from_secs(self.opts.timeout_time_sec as u64)); 128 128 - 129 129 - histogram!("jetstream.connection.duration").record(elapsed.as_secs_f64()); 130 130 - match message { 131 131 - Ok(message) => { 132 132 - if let Err(err) = self.msg_tx.send_async(message).await { 133 133 - counter!("jetstream.error").increment(1); 134 134 - error!("Failed to queue message: {}", err); 135 135 - } 136 136 - } 137 137 - Err(e) => { 138 138 - counter!("jetstream.error").increment(1); 139 139 - error!("Error: {}", e); 140 140 - } 141 141 - } 142 142 - } 143 143 - None => { 144 144 - info!("Stream closed by server."); 145 145 - counter!("jetstream.connection.reconnect").increment(1); 146 146 - break; // Stream ended, break inner loop to reconnect 147 147 - } 148 148 - } 149 149 - } 150 150 - _ = self.reconnect_rx.recv_async() => { 151 151 - info!("Reconnect signal received."); 152 152 - counter!("jetstream.connection.reconnect").increment(1); 153 153 - break; 154 154 - } 155 155 - _ = &mut receive_timeout => { 156 156 - // last final poll, just in case 157 157 - match read.next().await { 158 158 - Some(Ok(message)) => { 159 159 - if let Err(err) = self.msg_tx.send_async(message).await { 160 160 - counter!("jetstream.error").increment(1); 161 161 - error!("Failed to queue message: {}", err); 162 162 - } 163 163 - // Reset timeout to continue 164 164 - receive_timeout.as_mut().reset(tokio::time::Instant::now() + Duration::from_secs(self.opts.timeout_time_sec as u64)); 165 165 - } 166 166 - Some(Err(e)) => { 167 167 - counter!("jetstream.error").increment(1); 168 168 - error!("Error receiving message during final poll: {}", e); 169 169 - counter!("jetstream.connection.reconnect").increment(1); 170 170 - break; 171 171 - } 172 172 - None => { 173 173 - info!("No commits received in {} seconds, reconnecting.", self.opts.timeout_time_sec); 174 174 - counter!("jetstream.connection.reconnect").increment(1); 175 175 - break; 176 176 - } 177 177 - } 178 178 - } 179 179 - } 180 180 - } 181 181 - } 182 182 - } 183 183 - Err(e) => { 184 184 - let elapsed_time = time_provider.elapsed(start_time); 185 185 - // reset if time connected > the time we set 186 186 - if elapsed_time.as_secs() > self.opts.max_retry_interval_seconds { 187 187 - retry_interval = 0; 188 188 - start_time = time_provider.now(); 189 189 - } 190 190 - counter!("jetstream.connection.error").increment(1); 191 191 - error!("Connection error: {}", e); 192 192 - } 193 193 - } 194 194 - 195 195 - let sleep_time = max(1, min(self.opts.max_retry_interval_seconds, retry_interval)); 196 196 - info!("Reconnecting in {} seconds...", sleep_time); 197 197 - sleep(Duration::from_secs(sleep_time)).await; 198 198 - 199 199 - if retry_interval > self.opts.max_retry_interval_seconds { 200 200 - retry_interval = self.opts.max_retry_interval_seconds; 201 201 - } else { 202 202 - retry_interval *= 2; 203 203 - } 204 204 - } 205 205 - } 206 206 - 207 207 - pub fn force_reconnect(&self) -> Result<(), flume::SendError<()>> { 208 208 - info!("Force reconnect requested."); 209 209 - self.reconnect_tx.send(()) // Send a reconnect signal 210 210 - } 211 211 - } 212 212 - 213 213 - #[cfg(test)] 214 214 - mod tests { 215 215 - use super::*; 216 216 - use std::sync::{Arc, Mutex}; 217 217 - use tokio::task; 218 218 - use tokio::time::{timeout, Duration}; 219 219 - use tokio_tungstenite::tungstenite::Message; 220 220 - 221 221 - #[test] 222 222 - fn test_build_ws_url() { 223 223 - let opts = JetstreamOptions { 224 224 - wanted_collections: Some(vec!["col1".to_string(), "col2".to_string()]), 225 225 - wanted_dids: Some(vec!["did1".to_string()]), 226 226 - ..Default::default() 227 227 - }; 228 228 - let connection = JetstreamConnection::new(opts); 229 229 - 230 230 - let test = Arc::new(Mutex::new(Some(8373))); 231 231 - 232 232 - let url = connection.build_ws_url(test); 233 233 - 234 234 - assert!(url.starts_with("wss://")); 235 235 - assert!(url.contains("cursor=8373")); 236 236 - assert!(url.contains("wantedCollections=col1")); 237 237 - assert!(url.contains("wantedCollections=col2")); 238 238 - assert!(url.contains("wantedDids=did1")); 239 239 - } 240 240 - 241 241 - #[tokio::test] 242 242 - async fn test_force_reconnect() { 243 243 - let opts = JetstreamOptions::default(); 244 244 - let connection = JetstreamConnection::new(opts); 245 245 - 246 246 - // Spawn a task to listen for the reconnect signal 247 247 - let reconnect_rx = connection.reconnect_rx.clone(); 248 248 - let recv_task = task::spawn(async move { 249 249 - reconnect_rx 250 250 - .recv_async() 251 251 - .await 252 252 - .expect("Failed to receive reconnect signal"); 253 253 - }); 254 254 - 255 255 - connection 256 256 - .force_reconnect() 257 257 - .expect("Failed to send reconnect signal"); 258 258 - 259 259 - // Ensure reconnect signal was received 260 260 - assert!(recv_task.await.is_ok()); 261 261 - } 262 262 - 263 263 - #[tokio::test] 264 264 - async fn test_message_queue() { 265 265 - let opts = JetstreamOptions::default(); 266 266 - let connection = JetstreamConnection::new(opts); 267 267 - 268 268 - let msg_rx = connection.get_msg_rx(); 269 269 - let msg = Message::Text("test message".into()); 270 270 - 271 271 - // Send a message to the queue 272 272 - connection 273 273 - .msg_tx 274 274 - .send_async(msg.clone()) 275 275 - .await 276 276 - .expect("Failed to send message"); 277 277 - 278 278 - // Receive and verify the message 279 279 - let received = msg_rx 280 280 - .recv_async() 281 281 - .await 282 282 - .expect("Failed to receive message"); 283 283 - assert_eq!(received, msg); 284 284 - } 285 285 - 286 286 - #[tokio::test] 287 287 - async fn test_connection_retries_on_failure() { 288 288 - let opts = JetstreamOptions::default(); 289 289 - let connection = Arc::new(JetstreamConnection::new(opts)); 290 290 - 291 291 - let cursor = Arc::new(Mutex::new(None)); 292 292 - 293 293 - // Timeout to prevent infinite loop 294 294 - let result = timeout(Duration::from_secs(3), connection.connect(cursor)).await; 295 295 - 296 296 - assert!(result.is_err(), "Expected timeout due to retry logic"); 297 297 - } 298 298 - 299 299 - #[tokio::test] 300 300 - async fn test_reconnect_after_receive_timeout() { 301 301 - use tokio::net::TcpListener; 302 302 - use tokio_tungstenite::accept_async; 303 303 - 304 304 - let opts = JetstreamOptions { 305 305 - ws_url: crate::endpoints::JetstreamEndpoints::Custom("ws://127.0.0.1:9001".to_string()), 306 306 - bound: 5, 307 307 - max_retry_interval_seconds: 1, 308 308 - ..Default::default() 309 309 - }; 310 310 - let connection = JetstreamConnection::new(opts); 311 311 - let cursor = Arc::new(Mutex::new(None)); 312 312 - 313 313 - // set up dummy "websocket" 314 314 - let listener = TcpListener::bind("127.0.0.1:9001") 315 315 - .await 316 316 - .expect("Failed to bind"); 317 317 - let server_handle = tokio::spawn(async move { 318 318 - if let Ok((stream, _)) = listener.accept().await { 319 319 - let ws_stream = accept_async(stream).await.expect("Failed to accept"); 320 320 - // send nothing 321 321 - tokio::time::sleep(Duration::from_secs(6)).await; 322 322 - drop(ws_stream); 323 323 - } 324 324 - }); 325 325 - 326 326 - // spawn, then run for >30 seconds to trigger reconnect 327 327 - let connect_handle = tokio::spawn(async move { 328 328 - tokio::time::timeout(Duration::from_secs(5), connection.connect(cursor)) 329 329 - .await 330 330 - .ok(); 331 331 - }); 332 332 - 333 333 - let _ = tokio::join!(server_handle, connect_handle); 334 334 - } 335 335 - }

-65

services/rocketman/src/endpoints.rs

··· 1 1 - use std::fmt::{Display, Formatter, Result}; 2 2 - 3 3 - #[derive(Debug, Clone, PartialEq, Eq, Hash)] 4 4 - pub enum JetstreamEndpointLocations { 5 5 - UsEast, 6 6 - UsWest, 7 7 - } 8 8 - 9 9 - impl Display for JetstreamEndpointLocations { 10 10 - fn fmt(&self, f: &mut Formatter<'_>) -> Result { 11 11 - write!( 12 12 - f, 13 13 - "{}", 14 14 - match self { 15 15 - Self::UsEast => "us-east", 16 16 - Self::UsWest => "us-west", 17 17 - } 18 18 - ) 19 19 - } 20 20 - } 21 21 - 22 22 - #[derive(Debug, Clone, PartialEq, Eq, Hash)] 23 23 - pub enum JetstreamEndpoints { 24 24 - Public(JetstreamEndpointLocations, i8), 25 25 - Custom(String), 26 26 - } 27 27 - 28 28 - impl Display for JetstreamEndpoints { 29 29 - fn fmt(&self, f: &mut Formatter<'_>) -> Result { 30 30 - match self { 31 31 - Self::Public(location, id) => write!( 32 32 - f, 33 33 - "wss://jetstream{}.{}.bsky.network/subscribe", 34 34 - id, location 35 35 - ), 36 36 - Self::Custom(url) => write!(f, "{}", url), 37 37 - } 38 38 - } 39 39 - } 40 40 - 41 41 - impl Default for JetstreamEndpoints { 42 42 - fn default() -> Self { 43 43 - Self::Public(JetstreamEndpointLocations::UsEast, 2) 44 44 - } 45 45 - } 46 46 - 47 47 - #[cfg(test)] 48 48 - mod tests { 49 49 - use super::*; 50 50 - 51 51 - #[test] 52 52 - fn test_display_public() { 53 53 - let endpoint = JetstreamEndpoints::Public(JetstreamEndpointLocations::UsEast, 2); 54 54 - assert_eq!( 55 55 - endpoint.to_string(), 56 56 - "wss://jetstream2.us-east.bsky.network/subscribe" 57 57 - ); 58 58 - } 59 59 - 60 60 - #[test] 61 61 - fn test_display_custom() { 62 62 - let endpoint = JetstreamEndpoints::Custom("wss://custom.bsky.network/subscribe".into()); 63 63 - assert_eq!(endpoint.to_string(), "wss://custom.bsky.network/subscribe"); 64 64 - } 65 65 - }

-1

services/rocketman/src/err.rs

··· 1 1 - // TODO: error types instead of using anyhow

-452

services/rocketman/src/handler.rs

··· 1 1 - use anyhow::Result; 2 2 - use flume::Sender; 3 3 - use metrics::{counter, describe_counter, Unit}; 4 4 - use serde_json::Value; 5 5 - use std::{ 6 6 - collections::HashMap, 7 7 - sync::{Arc, Mutex}, 8 8 - }; 9 9 - use tokio_tungstenite::tungstenite::{Error, Message}; 10 10 - use tracing::{debug, error}; 11 11 - 12 12 - #[cfg(feature = "zstd")] 13 13 - use std::io::Cursor as IoCursor; 14 14 - #[cfg(feature = "zstd")] 15 15 - use std::sync::LazyLock; 16 16 - #[cfg(feature = "zstd")] 17 17 - use zstd::dict::DecoderDictionary; 18 18 - 19 19 - use crate::{ 20 20 - ingestion::LexiconIngestor, 21 21 - types::event::{Event, Kind}, 22 22 - }; 23 23 - 24 24 - /// The custom `zstd` dictionary used for decoding compressed Jetstream messages. 25 25 - /// 26 26 - /// Sourced from the [official Bluesky Jetstream repo.](https://github.com/bluesky-social/jetstream/tree/main/pkg/models) 27 27 - #[cfg(feature = "zstd")] 28 28 - static ZSTD_DICTIONARY: LazyLock<DecoderDictionary> = 29 29 - LazyLock::new(|| DecoderDictionary::copy(include_bytes!("../zstd/dictionary"))); 30 30 - 31 31 - pub async fn handle_message( 32 32 - message: Message, 33 33 - ingestors: &HashMap<String, Box<dyn LexiconIngestor + Send + Sync>>, 34 34 - reconnect_tx: Sender<()>, 35 35 - cursor: Arc<Mutex<Option<u64>>>, 36 36 - ) -> Result<()> { 37 37 - describe_counter!( 38 38 - "jetstream.event", 39 39 - Unit::Count, 40 40 - "number of event ingest attempts" 41 41 - ); 42 42 - describe_counter!( 43 43 - "jetstream.event.parse", 44 44 - Unit::Count, 45 45 - "events that were successfully processed" 46 46 - ); 47 47 - describe_counter!( 48 48 - "jetstream.event.fail", 49 49 - Unit::Count, 50 50 - "events that could not be read" 51 51 - ); 52 52 - describe_counter!("jetstream.error", Unit::Count, "errors encountered"); 53 53 - match message { 54 54 - Message::Text(text) => { 55 55 - debug!("Text message received"); 56 56 - counter!("jetstream.event").increment(1); 57 57 - let envelope: Event<Value> = serde_json::from_str(&text).map_err(|e| { 58 58 - anyhow::anyhow!("Failed to parse message: {} with json string {}", e, text) 59 59 - })?; 60 60 - debug!("envelope: {:?}", envelope); 61 61 - handle_envelope(envelope, cursor, ingestors).await?; 62 62 - Ok(()) 63 63 - } 64 64 - #[cfg(feature = "zstd")] 65 65 - Message::Binary(bytes) => { 66 66 - debug!("Binary message received"); 67 67 - counter!("jetstream.event").increment(1); 68 68 - let decoder = zstd::stream::Decoder::with_prepared_dictionary( 69 69 - IoCursor::new(bytes), 70 70 - &ZSTD_DICTIONARY, 71 71 - )?; 72 72 - let envelope: Event<Value> = serde_json::from_reader(decoder) 73 73 - .map_err(|e| anyhow::anyhow!("Failed to parse binary message: {}", e))?; 74 74 - debug!("envelope: {:?}", envelope); 75 75 - handle_envelope(envelope, cursor, ingestors).await?; 76 76 - Ok(()) 77 77 - } 78 78 - #[cfg(not(feature = "zstd"))] 79 79 - Message::Binary(_) => { 80 80 - debug!("Binary message received"); 81 81 - Err(anyhow::anyhow!( 82 82 - "binary message received but zstd feature is not enabled" 83 83 - )) 84 84 - } 85 85 - Message::Close(_) => { 86 86 - debug!("Server closed connection"); 87 87 - if let Err(e) = reconnect_tx.send(()) { 88 88 - counter!("jetstream.event.parse.error", "error" => "failed_to_send_reconnect_signal").increment(1); 89 89 - error!("Failed to send reconnect signal: {}", e); 90 90 - } 91 91 - Err(Error::ConnectionClosed.into()) 92 92 - } 93 93 - _ => Ok(()), 94 94 - } 95 95 - } 96 96 - 97 97 - async fn handle_envelope( 98 98 - envelope: Event<Value>, 99 99 - cursor: Arc<Mutex<Option<u64>>>, 100 100 - ingestors: &HashMap<String, Box<dyn LexiconIngestor + Send + Sync>>, 101 101 - ) -> Result<()> { 102 102 - if let Some(ref time_us) = envelope.time_us { 103 103 - debug!("Time: {}", time_us); 104 104 - if let Some(cursor) = cursor.lock().unwrap().as_mut() { 105 105 - debug!("Cursor: {}", cursor); 106 106 - if time_us > cursor { 107 107 - debug!("Cursor is behind, resetting"); 108 108 - *cursor = *time_us; 109 109 - } 110 110 - } 111 111 - } 112 112 - 113 113 - match envelope.kind { 114 114 - Kind::Commit => match extract_commit_nsid(&envelope) { 115 115 - Ok(nsid) => { 116 116 - if let Some(fun) = ingestors.get(&nsid) { 117 117 - match fun.ingest(envelope).await { 118 118 - Ok(_) => { 119 119 - counter!("jetstream.event.parse.commit", "nsid" => nsid).increment(1) 120 120 - } 121 121 - Err(e) => { 122 122 - error!("Error ingesting commit with nsid {}: {}", nsid, e); 123 123 - counter!("jetstream.error").increment(1); 124 124 - counter!("jetstream.event.fail").increment(1); 125 125 - } 126 126 - } 127 127 - } 128 128 - } 129 129 - Err(e) => error!("Error parsing commit: {}", e), 130 130 - }, 131 131 - Kind::Identity => { 132 132 - counter!("jetstream.event.parse.identity").increment(1); 133 133 - } 134 134 - Kind::Account => { 135 135 - counter!("jetstream.event.parse.account").increment(1); 136 136 - } 137 137 - Kind::Unknown(kind) => { 138 138 - counter!("jetstream.event.parse.unknown", "kind" => kind).increment(1); 139 139 - } 140 140 - } 141 141 - Ok(()) 142 142 - } 143 143 - 144 144 - fn extract_commit_nsid(envelope: &Event<Value>) -> anyhow::Result<String> { 145 145 - // if the type is not a commit 146 146 - if envelope.commit.is_none() { 147 147 - return Err(anyhow::anyhow!( 148 148 - "Message has no commit, so there is no nsid attached." 149 149 - )); 150 150 - } else if let Some(ref commit) = envelope.commit { 151 151 - return Ok(commit.collection.clone()); 152 152 - } 153 153 - 154 154 - Err(anyhow::anyhow!("Failed to extract nsid: unknown error")) 155 155 - } 156 156 - 157 157 - #[cfg(test)] 158 158 - mod tests { 159 159 - use super::*; 160 160 - use crate::types::event::Event; 161 161 - use anyhow::Result; 162 162 - use async_trait::async_trait; 163 163 - use flume::{Receiver, Sender}; 164 164 - use serde_json::json; 165 165 - use std::{ 166 166 - collections::HashMap, 167 167 - sync::{Arc, Mutex}, 168 168 - }; 169 169 - use tokio_tungstenite::tungstenite::Message; 170 170 - 171 171 - // Dummy ingestor that records if it was called. 172 172 - struct DummyIngestor { 173 173 - pub called: Arc<Mutex<bool>>, 174 174 - } 175 175 - 176 176 - #[async_trait] 177 177 - impl crate::ingestion::LexiconIngestor for DummyIngestor { 178 178 - async fn ingest(&self, _event: Event<serde_json::Value>) -> Result<(), anyhow::Error> { 179 179 - let mut called = self.called.lock().unwrap(); 180 180 - *called = true; 181 181 - Ok(()) 182 182 - } 183 183 - } 184 184 - 185 185 - // Dummy ingestor that always returns an error. 186 186 - struct ErrorIngestor; 187 187 - 188 188 - #[async_trait] 189 189 - impl crate::ingestion::LexiconIngestor for ErrorIngestor { 190 190 - async fn ingest(&self, _event: Event<serde_json::Value>) -> Result<(), anyhow::Error> { 191 191 - Err(anyhow::anyhow!("Ingest error")) 192 192 - } 193 193 - } 194 194 - 195 195 - // Helper to create a reconnect channel. 196 196 - fn setup_reconnect_channel() -> (Sender<()>, Receiver<()>) { 197 197 - flume::unbounded() 198 198 - } 199 199 - 200 200 - #[tokio::test] 201 201 - async fn test_valid_commit_success() { 202 202 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 203 203 - let cursor = Arc::new(Mutex::new(Some(100))); 204 204 - let called_flag = Arc::new(Mutex::new(false)); 205 205 - 206 206 - // Create a valid commit event JSON. 207 207 - let event_json = json!({ 208 208 - "did": "did:example:123", 209 209 - "time_us": 200, 210 210 - "kind": "commit", 211 211 - "commit": { 212 212 - "rev": "1", 213 213 - "operation": "create", 214 214 - "collection": "ns1", 215 215 - "rkey": "rkey1", 216 216 - "record": { "foo": "bar" }, 217 217 - "cid": "cid123" 218 218 - }, 219 219 - }) 220 220 - .to_string(); 221 221 - 222 222 - let mut ingestors: HashMap< 223 223 - String, 224 224 - Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>, 225 225 - > = HashMap::new(); 226 226 - ingestors.insert( 227 227 - "ns1".to_string(), 228 228 - Box::new(DummyIngestor { 229 229 - called: called_flag.clone(), 230 230 - }), 231 231 - ); 232 232 - 233 233 - let result = handle_message( 234 234 - Message::Text(event_json), 235 235 - &ingestors, 236 236 - reconnect_tx, 237 237 - cursor.clone(), 238 238 - ) 239 239 - .await; 240 240 - assert!(result.is_ok()); 241 241 - // Check that the ingestor was called. 242 242 - assert!(*called_flag.lock().unwrap()); 243 243 - // Verify that the cursor got updated. 244 244 - assert_eq!(*cursor.lock().unwrap(), Some(200)); 245 245 - } 246 246 - 247 247 - #[cfg(feature = "zstd")] 248 248 - #[tokio::test] 249 249 - async fn test_binary_valid_commit() { 250 250 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 251 251 - let cursor = Arc::new(Mutex::new(Some(100))); 252 252 - let called_flag = Arc::new(Mutex::new(false)); 253 253 - 254 254 - let uncompressed_json = json!({ 255 255 - "did": "did:example:123", 256 256 - "time_us": 200, 257 257 - "kind": "commit", 258 258 - "commit": { 259 259 - "rev": "1", 260 260 - "operation": "create", 261 261 - "collection": "ns1", 262 262 - "rkey": "rkey1", 263 263 - "record": { "foo": "bar" }, 264 264 - "cid": "cid123" 265 265 - }, 266 266 - }) 267 267 - .to_string(); 268 268 - 269 269 - let compressed_dest: IoCursor<Vec<u8>> = IoCursor::new(vec![]); 270 270 - let mut encoder = zstd::Encoder::with_prepared_dictionary( 271 271 - compressed_dest, 272 272 - &zstd::dict::EncoderDictionary::copy(include_bytes!("../zstd/dictionary"), 0), 273 273 - ) 274 274 - .unwrap(); 275 275 - std::io::copy( 276 276 - &mut IoCursor::new(uncompressed_json.as_bytes()), 277 277 - &mut encoder, 278 278 - ) 279 279 - .unwrap(); 280 280 - let compressed_dest = encoder.finish().unwrap(); 281 281 - 282 282 - let mut ingestors: HashMap< 283 283 - String, 284 284 - Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>, 285 285 - > = HashMap::new(); 286 286 - ingestors.insert( 287 287 - "ns1".to_string(), 288 288 - Box::new(DummyIngestor { 289 289 - called: called_flag.clone(), 290 290 - }), 291 291 - ); 292 292 - 293 293 - let result = handle_message( 294 294 - Message::Binary(compressed_dest.into_inner()), 295 295 - &ingestors, 296 296 - reconnect_tx, 297 297 - cursor.clone(), 298 298 - ) 299 299 - .await; 300 300 - 301 301 - assert!(result.is_ok()); 302 302 - // Check that the ingestor was called. 303 303 - assert!(*called_flag.lock().unwrap()); 304 304 - // Verify that the cursor got updated. 305 305 - assert_eq!(*cursor.lock().unwrap(), Some(200)); 306 306 - } 307 307 - 308 308 - #[tokio::test] 309 309 - async fn test_commit_ingest_failure() { 310 310 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 311 311 - let cursor = Arc::new(Mutex::new(Some(100))); 312 312 - 313 313 - // Valid commit event with an ingestor that fails. 314 314 - let event_json = json!({ 315 315 - "did": "did:example:123", 316 316 - "time_us": 300, 317 317 - "kind": "commit", 318 318 - "commit": { 319 319 - "rev": "1", 320 320 - "operation": "create", 321 321 - "collection": "ns_error", 322 322 - "rkey": "rkey1", 323 323 - "record": { "foo": "bar" }, 324 324 - "cid": "cid123" 325 325 - }, 326 326 - "identity": null 327 327 - }) 328 328 - .to_string(); 329 329 - 330 330 - let mut ingestors: HashMap< 331 331 - String, 332 332 - Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>, 333 333 - > = HashMap::new(); 334 334 - ingestors.insert("ns_error".to_string(), Box::new(ErrorIngestor)); 335 335 - 336 336 - // Even though ingestion fails, handle_message returns Ok(()). 337 337 - let result = handle_message( 338 338 - Message::Text(event_json), 339 339 - &ingestors, 340 340 - reconnect_tx, 341 341 - cursor.clone(), 342 342 - ) 343 343 - .await; 344 344 - assert!(result.is_ok()); 345 345 - // Cursor should still update because it comes before the ingest call. 346 346 - assert_eq!(*cursor.lock().unwrap(), Some(300)); 347 347 - } 348 348 - 349 349 - #[tokio::test] 350 350 - async fn test_identity_message() { 351 351 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 352 352 - let cursor = Arc::new(Mutex::new(None)); 353 353 - // Valid identity event. 354 354 - let event_json = json!({ 355 355 - "did": "did:example:123", 356 356 - "time_us": 150, 357 357 - "kind": "identity", 358 358 - "commit": null, 359 359 - "identity": { 360 360 - "did": "did:example:123", 361 361 - "handle": "user", 362 362 - "seq": 1, 363 363 - "time": "2025-01-01T00:00:00Z" 364 364 - } 365 365 - }) 366 366 - .to_string(); 367 367 - let ingestors: HashMap<String, Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>> = 368 368 - HashMap::new(); 369 369 - 370 370 - let result = 371 371 - handle_message(Message::Text(event_json), &ingestors, reconnect_tx, cursor).await; 372 372 - assert!(result.is_ok()); 373 373 - } 374 374 - 375 375 - #[tokio::test] 376 376 - async fn test_close_message() { 377 377 - let (reconnect_tx, reconnect_rx) = setup_reconnect_channel(); 378 378 - let cursor = Arc::new(Mutex::new(None)); 379 379 - let ingestors: HashMap<String, Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>> = 380 380 - HashMap::new(); 381 381 - 382 382 - let result = handle_message(Message::Close(None), &ingestors, reconnect_tx, cursor).await; 383 383 - // Should return an error due to connection close. 384 384 - assert!(result.is_err()); 385 385 - // Verify that a reconnect signal was sent. 386 386 - let signal = reconnect_rx.recv_async().await; 387 387 - assert!(signal.is_ok()); 388 388 - } 389 389 - 390 390 - #[tokio::test] 391 391 - async fn test_invalid_json() { 392 392 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 393 393 - let cursor = Arc::new(Mutex::new(None)); 394 394 - let ingestors: HashMap<String, Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>> = 395 395 - HashMap::new(); 396 396 - 397 397 - let invalid_json = "this is not json".to_string(); 398 398 - let result = handle_message( 399 399 - Message::Text(invalid_json), 400 400 - &ingestors, 401 401 - reconnect_tx, 402 402 - cursor, 403 403 - ) 404 404 - .await; 405 405 - assert!(result.is_err()); 406 406 - } 407 407 - 408 408 - #[tokio::test] 409 409 - async fn test_cursor_not_updated_if_lower() { 410 410 - let (reconnect_tx, _reconnect_rx) = setup_reconnect_channel(); 411 411 - // Set an initial cursor value. 412 412 - let cursor = Arc::new(Mutex::new(Some(300))); 413 413 - let event_json = json!({ 414 414 - "did": "did:example:123", 415 415 - "time_us": 200, 416 416 - "kind": "commit", 417 417 - "commit": { 418 418 - "rev": "1", 419 419 - "operation": "create", 420 420 - "collection": "ns1", 421 421 - "rkey": "rkey1", 422 422 - "record": { "foo": "bar" }, 423 423 - "cid": "cid123" 424 424 - }, 425 425 - "identity": null 426 426 - }) 427 427 - .to_string(); 428 428 - 429 429 - // Use a dummy ingestor that does nothing. 430 430 - let mut ingestors: HashMap< 431 431 - String, 432 432 - Box<dyn crate::ingestion::LexiconIngestor + Send + Sync>, 433 433 - > = HashMap::new(); 434 434 - ingestors.insert( 435 435 - "ns1".to_string(), 436 436 - Box::new(DummyIngestor { 437 437 - called: Arc::new(Mutex::new(false)), 438 438 - }), 439 439 - ); 440 440 - 441 441 - let result = handle_message( 442 442 - Message::Text(event_json), 443 443 - &ingestors, 444 444 - reconnect_tx, 445 445 - cursor.clone(), 446 446 - ) 447 447 - .await; 448 448 - assert!(result.is_ok()); 449 449 - // Cursor should remain unchanged. 450 450 - assert_eq!(*cursor.lock().unwrap(), Some(300)); 451 451 - } 452 452 - }

-22

services/rocketman/src/ingestion.rs

··· 1 1 - use anyhow::Result; 2 2 - use async_trait::async_trait; 3 3 - use serde_json::Value; 4 4 - use tracing::info; 5 5 - 6 6 - use crate::types::event::Event; 7 7 - 8 8 - #[async_trait] 9 9 - pub trait LexiconIngestor { 10 10 - async fn ingest(&self, message: Event<Value>) -> Result<()>; 11 11 - } 12 12 - 13 13 - pub struct DefaultLexiconIngestor; 14 14 - 15 15 - #[async_trait] 16 16 - impl LexiconIngestor for DefaultLexiconIngestor { 17 17 - async fn ingest(&self, message: Event<Value>) -> Result<()> { 18 18 - info!("Default lexicon processing: {:?}", message); 19 19 - // Process message for default lexicon. 20 20 - Ok(()) 21 21 - } 22 22 - }

-8

services/rocketman/src/lib.rs

··· 1 1 - // lib.rs 2 2 - pub mod connection; 3 3 - pub mod endpoints; 4 4 - pub mod handler; 5 5 - pub mod ingestion; 6 6 - pub mod options; 7 7 - pub mod time; 8 8 - pub mod types;

-40

services/rocketman/src/options.rs

··· 1 1 - use bon::Builder; 2 2 - 3 3 - use crate::endpoints::JetstreamEndpoints; 4 4 - 5 5 - #[derive(Builder, Debug)] 6 6 - pub struct JetstreamOptions { 7 7 - #[builder(default)] 8 8 - pub ws_url: JetstreamEndpoints, 9 9 - #[builder(default)] 10 10 - pub max_retry_interval_seconds: u64, 11 11 - #[builder(default)] 12 12 - pub connection_success_time_seconds: u64, 13 13 - #[builder(default)] 14 14 - pub bound: usize, 15 15 - #[builder(default)] 16 16 - pub timeout_time_sec: usize, 17 17 - #[cfg(feature = "zstd")] 18 18 - #[builder(default = true)] 19 19 - pub compress: bool, 20 20 - pub wanted_collections: Option<Vec<String>>, 21 21 - pub wanted_dids: Option<Vec<String>>, 22 22 - pub cursor: Option<String>, 23 23 - } 24 24 - 25 25 - impl Default for JetstreamOptions { 26 26 - fn default() -> Self { 27 27 - Self { 28 28 - ws_url: JetstreamEndpoints::default(), 29 29 - max_retry_interval_seconds: 120, 30 30 - connection_success_time_seconds: 60, 31 31 - bound: 65536, 32 32 - timeout_time_sec: 40, 33 33 - #[cfg(feature = "zstd")] 34 34 - compress: true, 35 35 - wanted_collections: None, 36 36 - wanted_dids: None, 37 37 - cursor: None, 38 38 - } 39 39 - } 40 40 - }

-11

services/rocketman/src/time/mod.rs

··· 1 1 - use std::time::{Duration, Instant, SystemTime}; 2 2 - 3 3 - pub mod system_time; 4 4 - 5 5 - pub trait TimeProvider { 6 6 - fn new() -> Self; 7 7 - fn now(&self) -> SystemTime; // Get the current time 8 8 - fn elapsed(&self, earlier: SystemTime) -> Duration; // Calculate the elapsed time. 9 9 - fn instant_now(&self) -> Instant; // For compatibility with your existing code (if needed) 10 10 - fn instant_elapsed(&self, earlier: Instant) -> Duration; 11 11 - }

-28

services/rocketman/src/time/system_time.rs

··· 1 1 - use std::time::{Duration, Instant, SystemTime}; 2 2 - 3 3 - use super::TimeProvider; 4 4 - 5 5 - #[derive(Default, Clone, Copy)] // Add these derives for ease of use 6 6 - pub struct SystemTimeProvider; // No fields needed, just a marker type 7 7 - 8 8 - impl TimeProvider for SystemTimeProvider { 9 9 - fn new() -> Self { 10 10 - Self 11 11 - } 12 12 - 13 13 - fn now(&self) -> SystemTime { 14 14 - SystemTime::now() 15 15 - } 16 16 - 17 17 - fn elapsed(&self, earlier: SystemTime) -> Duration { 18 18 - earlier.elapsed().unwrap_or_else(|_| Duration::from_secs(0)) 19 19 - } 20 20 - 21 21 - fn instant_now(&self) -> Instant { 22 22 - Instant::now() 23 23 - } 24 24 - 25 25 - fn instant_elapsed(&self, earlier: Instant) -> Duration { 26 26 - earlier.elapsed() 27 27 - } 28 28 - }

-116

services/rocketman/src/types/event.rs

··· 1 1 - use serde::{Deserialize, Deserializer, Serialize}; 2 2 - 3 3 - #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] 4 4 - #[serde(rename_all = "lowercase")] 5 5 - pub enum Kind { 6 6 - Account, 7 7 - Identity, 8 8 - Commit, 9 9 - Unknown(String), 10 10 - } 11 11 - 12 12 - #[derive(Debug, Serialize, Deserialize)] 13 13 - #[serde(rename_all = "snake_case")] 14 14 - pub struct Event<T> { 15 15 - pub did: String, 16 16 - pub time_us: Option<u64>, 17 17 - pub kind: Kind, 18 18 - pub commit: Option<Commit<T>>, 19 19 - pub identity: Option<Identity>, 20 20 - } 21 21 - 22 22 - #[derive(Debug, Serialize, Deserialize)] 23 23 - pub struct Identity { 24 24 - did: String, 25 25 - handle: Option<String>, 26 26 - seq: u64, 27 27 - time: String, 28 28 - } 29 29 - 30 30 - #[derive(Debug, Serialize, Deserialize)] 31 31 - #[serde(rename_all = "lowercase")] 32 32 - enum AccountStatus { 33 33 - TakenDown, 34 34 - Suspended, 35 35 - Deleted, 36 36 - Activated, 37 37 - } 38 38 - 39 39 - #[derive(Debug, Serialize, Deserialize)] 40 40 - pub struct Account { 41 41 - did: String, 42 42 - handle: String, 43 43 - seq: u64, 44 44 - time: String, 45 45 - status: AccountStatus, 46 46 - } 47 47 - 48 48 - #[derive(Debug, Serialize)] 49 49 - #[serde(rename_all = "camelCase")] 50 50 - pub struct Commit<T> { 51 51 - pub rev: String, 52 52 - pub operation: Operation, 53 53 - pub collection: String, 54 54 - pub rkey: String, 55 55 - pub record: Option<T>, 56 56 - pub cid: Option<String>, 57 57 - } 58 58 - 59 59 - #[derive(Debug, Serialize, Deserialize)] 60 60 - #[serde(rename_all = "lowercase")] 61 61 - pub enum Operation { 62 62 - Create, 63 63 - Update, 64 64 - Delete, 65 65 - } 66 66 - 67 67 - /// Enforce that record is None only when operation is 'delete' 68 68 - impl<'de, T> Deserialize<'de> for Commit<T> 69 69 - where 70 70 - T: Deserialize<'de>, 71 71 - { 72 72 - fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> 73 73 - where 74 74 - D: Deserializer<'de>, 75 75 - { 76 76 - // Helper struct to perform the deserialization. 77 77 - #[derive(Deserialize)] 78 78 - #[serde(rename_all = "camelCase")] 79 79 - struct Helper<T> { 80 80 - rev: String, 81 81 - operation: Operation, 82 82 - collection: String, 83 83 - rkey: String, 84 84 - record: Option<T>, 85 85 - cid: Option<String>, 86 86 - } 87 87 - 88 88 - let helper = Helper::deserialize(deserializer)?; 89 89 - 90 90 - match helper.operation { 91 91 - Operation::Delete => { 92 92 - if helper.record.is_some() || helper.cid.is_some() { 93 93 - return Err(<D::Error as serde::de::Error>::custom( 94 94 - "record and cid must be null when operation is delete", 95 95 - )); 96 96 - } 97 97 - } 98 98 - _ => { 99 99 - if helper.record.is_none() || helper.cid.is_none() { 100 100 - return Err(<D::Error as serde::de::Error>::custom( 101 101 - "record and cid must be present unless operation is delete", 102 102 - )); 103 103 - } 104 104 - } 105 105 - } 106 106 - 107 107 - Ok(Commit { 108 108 - rev: helper.rev, 109 109 - operation: helper.operation, 110 110 - collection: helper.collection, 111 111 - rkey: helper.rkey, 112 112 - record: helper.record, 113 113 - cid: helper.cid, 114 114 - }) 115 115 - } 116 116 - }

-1

services/rocketman/src/types/mod.rs

··· 1 1 - pub mod event;

services/rocketman/zstd/dictionary

This is a binary file and will not be displayed.