evan.jarrett.net
A container registry that uses the AT Protocol for manifest storage and S3 for blob storage.
atcr.io
docker
container
atproto
go
1package main
2
3import (
4 "bytes"
5 "fmt"
6 "io"
7 "os"
8 "os/exec"
9 "path/filepath"
10 "strings"
11 "time"
12
13 "github.com/spf13/cobra"
14)
15
16var updateCmd = &cobra.Command{
17 Use: "update [target]",
18 Short: "Deploy updates to servers",
19 Args: cobra.MaximumNArgs(1),
20 ValidArgs: []string{"all", "appview", "hold"},
21 RunE: func(cmd *cobra.Command, args []string) error {
22 target := "all"
23 if len(args) > 0 {
24 target = args[0]
25 }
26 withScanner, _ := cmd.Flags().GetBool("with-scanner")
27 return cmdUpdate(target, withScanner)
28 },
29}
30
31var sshCmd = &cobra.Command{
32 Use: "ssh <target>",
33 Short: "SSH into a server",
34 Args: cobra.ExactArgs(1),
35 ValidArgs: []string{"appview", "hold"},
36 RunE: func(cmd *cobra.Command, args []string) error {
37 return cmdSSH(args[0])
38 },
39}
40
41func init() {
42 updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold")
43 rootCmd.AddCommand(updateCmd)
44 rootCmd.AddCommand(sshCmd)
45}
46
47func cmdUpdate(target string, withScanner bool) error {
48 state, err := loadState()
49 if err != nil {
50 return err
51 }
52
53 naming := state.Naming()
54 rootDir := projectRoot()
55
56 // Enable scanner retroactively via --with-scanner on update
57 if withScanner && !state.ScannerEnabled {
58 state.ScannerEnabled = true
59 if state.ScannerSecret == "" {
60 secret, err := generateScannerSecret()
61 if err != nil {
62 return fmt.Errorf("generate scanner secret: %w", err)
63 }
64 state.ScannerSecret = secret
65 fmt.Printf("Generated scanner shared secret\n")
66 }
67 _ = saveState(state)
68 }
69
70 vals := configValsFromState(state)
71
72 targets := map[string]struct {
73 ip string
74 binaryName string
75 buildCmd string
76 localBinary string
77 serviceName string
78 healthURL string
79 configTmpl string
80 configPath string
81 unitTmpl string
82 }{
83 "appview": {
84 ip: state.Appview.PublicIP,
85 binaryName: naming.Appview(),
86 buildCmd: "appview",
87 localBinary: "atcr-appview",
88 serviceName: naming.Appview(),
89 healthURL: "http://localhost:5000/health",
90 configTmpl: appviewConfigTmpl,
91 configPath: naming.AppviewConfigPath(),
92 unitTmpl: appviewServiceTmpl,
93 },
94 "hold": {
95 ip: state.Hold.PublicIP,
96 binaryName: naming.Hold(),
97 buildCmd: "hold",
98 localBinary: "atcr-hold",
99 serviceName: naming.Hold(),
100 healthURL: "http://localhost:8080/xrpc/_health",
101 configTmpl: holdConfigTmpl,
102 configPath: naming.HoldConfigPath(),
103 unitTmpl: holdServiceTmpl,
104 },
105 }
106
107 var toUpdate []string
108 switch target {
109 case "all":
110 toUpdate = []string{"appview", "hold"}
111 case "appview", "hold":
112 toUpdate = []string{target}
113 default:
114 return fmt.Errorf("unknown target: %s (use: all, appview, hold)", target)
115 }
116
117 // Run go generate before building
118 if err := runGenerate(rootDir); err != nil {
119 return fmt.Errorf("go generate: %w", err)
120 }
121
122 // Build all binaries locally before touching servers
123 fmt.Println("Building locally (GOOS=linux GOARCH=amd64)...")
124 for _, name := range toUpdate {
125 t := targets[name]
126 outputPath := filepath.Join(rootDir, "bin", t.localBinary)
127 if err := buildLocal(rootDir, outputPath, "./cmd/"+t.buildCmd); err != nil {
128 return fmt.Errorf("build %s: %w", name, err)
129 }
130 }
131
132 // Build scanner locally if needed
133 needScanner := false
134 for _, name := range toUpdate {
135 if name == "hold" && state.ScannerEnabled {
136 needScanner = true
137 break
138 }
139 }
140 if needScanner {
141 outputPath := filepath.Join(rootDir, "bin", "atcr-scanner")
142 if err := buildLocal(filepath.Join(rootDir, "scanner"), outputPath, "./cmd/scanner"); err != nil {
143 return fmt.Errorf("build scanner: %w", err)
144 }
145 }
146
147 // Deploy each target
148 for _, name := range toUpdate {
149 t := targets[name]
150 fmt.Printf("\nDeploying %s (%s)...\n", name, t.ip)
151
152 // Sync config keys (adds missing keys from template, never overwrites)
153 configYAML, err := renderConfig(t.configTmpl, vals)
154 if err != nil {
155 return fmt.Errorf("render %s config: %w", name, err)
156 }
157 if err := syncConfigKeys(name, t.ip, t.configPath, configYAML); err != nil {
158 return fmt.Errorf("%s config sync: %w", name, err)
159 }
160
161 // Sync systemd service unit
162 renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{
163 DisplayName: naming.DisplayName(),
164 User: naming.SystemUser(),
165 BinaryPath: naming.InstallDir() + "/bin/" + t.binaryName,
166 ConfigPath: t.configPath,
167 DataDir: naming.BasePath(),
168 ServiceName: t.serviceName,
169 })
170 if err != nil {
171 return fmt.Errorf("render %s service unit: %w", name, err)
172 }
173 unitChanged, err := syncServiceUnit(name, t.ip, t.serviceName, renderedUnit)
174 if err != nil {
175 return fmt.Errorf("%s service unit sync: %w", name, err)
176 }
177
178 // Upload binary
179 localPath := filepath.Join(rootDir, "bin", t.localBinary)
180 remotePath := naming.InstallDir() + "/bin/" + t.binaryName
181 if err := scpFile(localPath, t.ip, remotePath); err != nil {
182 return fmt.Errorf("upload %s: %w", name, err)
183 }
184
185 daemonReload := ""
186 if unitChanged {
187 daemonReload = "systemctl daemon-reload"
188 }
189
190 // Scanner additions for hold server
191 scannerRestart := ""
192 scannerHealthCheck := ""
193 if name == "hold" && state.ScannerEnabled {
194 // Sync scanner config keys
195 scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
196 if err != nil {
197 return fmt.Errorf("render scanner config: %w", err)
198 }
199 if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML); err != nil {
200 return fmt.Errorf("scanner config sync: %w", err)
201 }
202
203 // Sync scanner service unit
204 scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
205 DisplayName: naming.DisplayName(),
206 User: naming.SystemUser(),
207 BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
208 ConfigPath: naming.ScannerConfigPath(),
209 DataDir: naming.BasePath(),
210 ServiceName: naming.Scanner(),
211 HoldServiceName: naming.Hold(),
212 })
213 if err != nil {
214 return fmt.Errorf("render scanner service unit: %w", err)
215 }
216 scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit)
217 if err != nil {
218 return fmt.Errorf("scanner service unit sync: %w", err)
219 }
220 if scannerUnitChanged {
221 daemonReload = "systemctl daemon-reload"
222 }
223
224 // Upload scanner binary
225 scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
226 scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
227 if err := scpFile(scannerLocal, t.ip, scannerRemote); err != nil {
228 return fmt.Errorf("upload scanner: %w", err)
229 }
230
231 // Ensure scanner data dirs exist on server
232 scannerSetup := fmt.Sprintf(`mkdir -p %s/vulndb %s/tmp
233chown -R %s:%s %s`,
234 naming.ScannerDataDir(), naming.ScannerDataDir(),
235 naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir())
236 if _, err := runSSH(t.ip, scannerSetup, false); err != nil {
237 return fmt.Errorf("scanner dir setup: %w", err)
238 }
239
240 scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner())
241 scannerHealthCheck = `
242sleep 2
243curl -sf http://localhost:9090/healthz > /dev/null && echo "SCANNER_HEALTH_OK" || echo "SCANNER_HEALTH_FAIL"
244`
245 }
246
247 // Restart services and health check
248 restartScript := fmt.Sprintf(`set -euo pipefail
249%s
250systemctl restart %s%s
251sleep 2
252curl -sf %s > /dev/null && echo "HEALTH_OK" || echo "HEALTH_FAIL"
253%s`, daemonReload, t.serviceName, scannerRestart, t.healthURL, scannerHealthCheck)
254
255 output, err := runSSH(t.ip, restartScript, true)
256 if err != nil {
257 fmt.Printf(" ERROR: %v\n", err)
258 fmt.Printf(" Output: %s\n", output)
259 return fmt.Errorf("restart %s failed", name)
260 }
261
262 if strings.Contains(output, "HEALTH_OK") {
263 fmt.Printf(" %s: updated and healthy\n", name)
264 } else if strings.Contains(output, "HEALTH_FAIL") {
265 fmt.Printf(" %s: updated but health check failed!\n", name)
266 fmt.Printf(" Check: ssh root@%s journalctl -u %s -n 50\n", t.ip, t.serviceName)
267 } else {
268 fmt.Printf(" %s: updated (health check inconclusive)\n", name)
269 }
270
271 // Scanner health reporting
272 if name == "hold" && state.ScannerEnabled {
273 if strings.Contains(output, "SCANNER_HEALTH_OK") {
274 fmt.Printf(" scanner: updated and healthy\n")
275 } else if strings.Contains(output, "SCANNER_HEALTH_FAIL") {
276 fmt.Printf(" scanner: updated but health check failed!\n")
277 fmt.Printf(" Check: ssh root@%s journalctl -u %s -n 50\n", t.ip, naming.Scanner())
278 }
279 }
280 }
281
282 return nil
283}
284
285// configValsFromState builds ConfigValues from persisted state.
286// S3SecretKey is intentionally left empty — syncConfigKeys only adds missing
287// keys and never overwrites, so the server's existing secret is preserved.
288func configValsFromState(state *InfraState) *ConfigValues {
289 naming := state.Naming()
290 _, baseDomain, _, _ := extractFromAppviewTemplate()
291 holdDomain := state.Zone + ".cove." + baseDomain
292
293 return &ConfigValues{
294 S3Endpoint: state.ObjectStorage.Endpoint,
295 S3Region: state.ObjectStorage.Region,
296 S3Bucket: state.ObjectStorage.Bucket,
297 S3AccessKey: state.ObjectStorage.AccessKeyID,
298 S3SecretKey: "", // not persisted in state; existing value on server is preserved
299 Zone: state.Zone,
300 HoldDomain: holdDomain,
301 HoldDid: "did:web:" + holdDomain,
302 BasePath: naming.BasePath(),
303 ScannerSecret: state.ScannerSecret,
304 }
305}
306
307// runGenerate runs go generate ./... in the given directory using host OS/arch
308// (no cross-compilation env vars — generate tools must run on the build machine).
309func runGenerate(dir string) error {
310 fmt.Println("Running go generate ./...")
311 cmd := exec.Command("go", "generate", "./...")
312 cmd.Dir = dir
313 cmd.Stdout = os.Stdout
314 cmd.Stderr = os.Stderr
315 return cmd.Run()
316}
317
318// buildLocal compiles a Go binary locally with cross-compilation flags for linux/amd64.
319func buildLocal(dir, outputPath, buildPkg string) error {
320 fmt.Printf(" building %s...\n", filepath.Base(outputPath))
321 cmd := exec.Command("go", "build",
322 "-ldflags=-s -w",
323 "-trimpath",
324 "-o", outputPath,
325 buildPkg,
326 )
327 cmd.Dir = dir
328 cmd.Env = append(os.Environ(),
329 "GOOS=linux",
330 "GOARCH=amd64",
331 "CGO_ENABLED=1",
332 )
333 cmd.Stdout = os.Stdout
334 cmd.Stderr = os.Stderr
335 return cmd.Run()
336}
337
338// scpFile uploads a local file to a remote server via SCP.
339// Removes the remote file first to avoid ETXTBSY when overwriting a running binary.
340func scpFile(localPath, ip, remotePath string) error {
341 fmt.Printf(" uploading %s → %s:%s\n", filepath.Base(localPath), ip, remotePath)
342 _, _ = runSSH(ip, fmt.Sprintf("rm -f %s", remotePath), false)
343 cmd := exec.Command("scp",
344 "-o", "StrictHostKeyChecking=accept-new",
345 "-o", "ConnectTimeout=10",
346 localPath,
347 "root@"+ip+":"+remotePath,
348 )
349 cmd.Stdout = os.Stdout
350 cmd.Stderr = os.Stderr
351 return cmd.Run()
352}
353
354func cmdSSH(target string) error {
355 state, err := loadState()
356 if err != nil {
357 return err
358 }
359
360 var ip string
361 switch target {
362 case "appview":
363 ip = state.Appview.PublicIP
364 case "hold":
365 ip = state.Hold.PublicIP
366 default:
367 return fmt.Errorf("unknown target: %s (use: appview, hold)", target)
368 }
369
370 fmt.Printf("Connecting to %s (%s)...\n", target, ip)
371 cmd := exec.Command("ssh",
372 "-o", "StrictHostKeyChecking=accept-new",
373 "root@"+ip,
374 )
375 cmd.Stdin = os.Stdin
376 cmd.Stdout = os.Stdout
377 cmd.Stderr = os.Stderr
378 return cmd.Run()
379}
380
381func runSSH(ip, script string, stream bool) (string, error) {
382 cmd := exec.Command("ssh",
383 "-o", "StrictHostKeyChecking=accept-new",
384 "-o", "ConnectTimeout=10",
385 "root@"+ip,
386 "bash -s",
387 )
388 cmd.Stdin = strings.NewReader(script)
389
390 var buf bytes.Buffer
391 if stream {
392 cmd.Stdout = io.MultiWriter(os.Stdout, &buf)
393 cmd.Stderr = io.MultiWriter(os.Stderr, &buf)
394 } else {
395 cmd.Stdout = &buf
396 cmd.Stderr = &buf
397 }
398
399 // Give deploys up to 5 minutes (SCP + restart, much faster than remote builds)
400 done := make(chan error, 1)
401 go func() { done <- cmd.Run() }()
402
403 select {
404 case err := <-done:
405 return buf.String(), err
406 case <-time.After(5 * time.Minute):
407 _ = cmd.Process.Kill()
408 return buf.String(), fmt.Errorf("SSH command timed out after 5 minutes")
409 }
410}