ptr.pet
at protocol indexer with flexible filtering, xrpc queries, and a cursor-backed event stream, built on fjall
at-protocol
atproto
indexer
rust
fjall
1#!/usr/bin/env nu
2use common.nu *
3
4def main [] {
5 # 1. ensure http-nu is installed
6 if (which http-nu | is-empty) {
7 print "http-nu not found, installing..."
8 cargo install http-nu
9 }
10
11 # 2. setup ports and paths
12 let port = 3006
13 let mock_port = 3008
14 let url = $"http://localhost:($port)"
15 let debug_url = $"http://localhost:($port + 1)"
16 let mock_url = $"http://localhost:($mock_port)"
17 let db_path = (mktemp -d -t hydrant_full_net.XXXXXX)
18
19 print $"testing full network crawler..."
20 print $"database path: ($db_path)"
21
22 # 3. start mock relay
23 print $"starting mock relay on ($mock_port)..."
24 let mock_pid = (
25 bash -c $"http-nu :($mock_port) tests/mock_relay.nu > ($db_path)/mock.log 2>&1 & echo $!"
26 | str trim
27 | into int
28 )
29 print $"mock relay pid: ($mock_pid)"
30
31 # give mock relay a moment
32 sleep 1sec
33
34 # 4. start hydrant in full network mode, firehose disabled
35 let binary = build-hydrant
36
37 let log_file = $"($db_path)/hydrant.log"
38 print $"starting hydrant - logs at ($log_file)..."
39
40 let hydrant_pid = (
41 with-env {
42 HYDRANT_DATABASE_PATH: ($db_path),
43 HYDRANT_FULL_NETWORK: "true",
44 HYDRANT_RELAY_HOST: ($mock_url),
45 HYDRANT_DISABLE_FIREHOSE: "true",
46 HYDRANT_DISABLE_BACKFILL: "true",
47 HYDRANT_API_PORT: ($port | into string),
48 HYDRANT_ENABLE_DEBUG: "true", # for stats checking
49 HYDRANT_DEBUG_PORT: ($port + 1 | into string),
50 HYDRANT_LOG_LEVEL: "debug",
51 HYDRANT_CURSOR_SAVE_INTERVAL: "1" # faster save
52 } {
53 sh -c $"($binary) >($log_file) 2>&1 & echo $!" | str trim | into int
54 }
55 )
56 print $"hydrant started with pid: ($hydrant_pid)"
57
58 mut success = false
59
60 try {
61 if (wait-for-api $url) {
62 print "hydrant api is up."
63
64 # wait for crawler to run (it runs on startup)
65 print "waiting for crawler to fetch repos..."
66
67 # retry check for 30s
68 for i in 1..30 {
69 let stats = (http get $"($url)/stats?accurate=true").counts
70 let pending = ($stats.pending | into int)
71 let repos = ($stats.repos | default 0 | into int)
72
73 # we expect 5 repos from the mock
74 print $"[($i)/30] pending: ($pending), known_repos: ($repos)"
75
76 if $repos >= 5 {
77 print "crawler successfully discovered repos!"
78 $success = true
79 break
80 }
81
82 sleep 1sec
83 }
84
85 if not $success {
86 print "timeout waiting for crawler."
87 }
88
89 # check cursor persistence
90 print "verifying crawler cursor persistence..."
91 let cursor_check = try {
92 # cursor key format is now: crawler_cursor|{relay_id_hash}
93 let cursor_iter = (http get $"($debug_url)/debug/iter?partition=cursors")
94 print $"cursors in db: ($cursor_iter | to json)"
95
96 let has_cursor = ($cursor_iter.items | any { |it| ($it.0 | str starts-with "crawler_cursor") })
97
98 if $has_cursor {
99 print "cursor verified."
100 true
101 } else {
102 print "cursor missing or empty."
103 false
104 }
105 } catch {
106 print "failed to get cursor from debug endpoint"
107 false
108 }
109 if not $cursor_check { $success = false }
110
111 } else {
112 print "hydrant failed to start."
113 }
114 } catch { |e|
115 print $"test failed with error: ($e)"
116 }
117
118 # cleanup
119 print "stopping processes..."
120 try { kill $hydrant_pid }
121 try { kill $mock_pid }
122
123 if $success {
124 print "test passed!"
125 exit 0
126 } else {
127 print "test failed!"
128 print "hydrant logs:"
129 open $log_file | tail -n 20
130 print "mock logs:"
131 open $"($db_path)/mock.log"
132 exit 1
133 }
134}