evan.jarrett.net
A container registry that uses the AT Protocol for manifest storage and S3 for blob storage.
atcr.io
docker
container
atproto
go
1package main
2
3import (
4 "bufio"
5 "context"
6 crypto_rand "crypto/rand"
7 "crypto/sha256"
8 "encoding/base64"
9 "encoding/hex"
10 "fmt"
11 "os"
12 "path/filepath"
13 "strings"
14 "time"
15
16 "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud"
17 "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/request"
18 "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/service"
19 "github.com/spf13/cobra"
20)
21
22var provisionCmd = &cobra.Command{
23 Use: "provision",
24 Short: "Create all infrastructure (servers, network, LB, firewall)",
25 RunE: func(cmd *cobra.Command, args []string) error {
26 token, _ := cmd.Root().PersistentFlags().GetString("token")
27 zone, _ := cmd.Flags().GetString("zone")
28 plan, _ := cmd.Flags().GetString("plan")
29 sshKey, _ := cmd.Flags().GetString("ssh-key")
30 s3Secret, _ := cmd.Flags().GetString("s3-secret")
31 withScanner, _ := cmd.Flags().GetBool("with-scanner")
32 return cmdProvision(token, zone, plan, sshKey, s3Secret, withScanner)
33 },
34}
35
36func init() {
37 provisionCmd.Flags().String("zone", "", "UpCloud zone (interactive picker if omitted)")
38 provisionCmd.Flags().String("plan", "", "Server plan (interactive picker if omitted)")
39 provisionCmd.Flags().String("ssh-key", "", "Path to SSH public key file (required)")
40 provisionCmd.Flags().String("s3-secret", "", "S3 secret access key (for existing object storage)")
41 provisionCmd.Flags().Bool("with-scanner", false, "Deploy vulnerability scanner alongside hold")
42 _ = provisionCmd.MarkFlagRequired("ssh-key")
43 rootCmd.AddCommand(provisionCmd)
44}
45
46func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner bool) error {
47 cfg, err := loadConfig(zone, plan, sshKeyPath, s3Secret)
48 if err != nil {
49 return err
50 }
51
52 naming := cfg.Naming()
53
54 svc, err := newService(token)
55 if err != nil {
56 return err
57 }
58
59 ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
60 defer cancel()
61
62 // Load existing state or start fresh
63 state, err := loadState()
64 if err != nil {
65 state = &InfraState{}
66 }
67
68 // Use zone from state if not provided via flags
69 if cfg.Zone == "" && state.Zone != "" {
70 cfg.Zone = state.Zone
71 }
72
73 // Only need interactive picker if we still need to create resources
74 needsServers := state.Appview.UUID == "" || state.Hold.UUID == ""
75 if cfg.Zone == "" || (needsServers && cfg.Plan == "") {
76 if err := resolveInteractive(ctx, svc, cfg); err != nil {
77 return err
78 }
79 }
80
81 if state.Zone == "" {
82 state.Zone = cfg.Zone
83 }
84 state.ClientName = cfg.ClientName
85 state.RepoBranch = cfg.RepoBranch
86
87 // Scanner setup
88 if withScanner {
89 state.ScannerEnabled = true
90 if state.ScannerSecret == "" {
91 secret, err := generateScannerSecret()
92 if err != nil {
93 return fmt.Errorf("generate scanner secret: %w", err)
94 }
95 state.ScannerSecret = secret
96 fmt.Printf("Generated scanner shared secret\n")
97 }
98 _ = saveState(state)
99 }
100
101 fmt.Printf("Provisioning %s infrastructure in zone %s...\n", naming.DisplayName(), cfg.Zone)
102 if needsServers {
103 fmt.Printf("Server plan: %s\n", cfg.Plan)
104 }
105 fmt.Println()
106
107 // S3 secret key — from flag for existing storage, from API for new
108 s3SecretKey := cfg.S3SecretKey
109
110 // 1. Object storage
111 if state.ObjectStorage.UUID != "" {
112 fmt.Printf("Object storage: %s (exists)\n", state.ObjectStorage.UUID)
113 // Refresh discoverable fields if missing (e.g. pre-seeded UUID only)
114 if state.ObjectStorage.Endpoint == "" || state.ObjectStorage.Bucket == "" {
115 fmt.Println(" Discovering endpoint, bucket, access key...")
116 discovered, err := lookupObjectStorage(ctx, svc, state.ObjectStorage.UUID)
117 if err != nil {
118 return err
119 }
120 state.ObjectStorage.Endpoint = discovered.Endpoint
121 state.ObjectStorage.Region = discovered.Region
122 if discovered.Bucket != "" {
123 state.ObjectStorage.Bucket = discovered.Bucket
124 }
125 if discovered.AccessKeyID != "" {
126 state.ObjectStorage.AccessKeyID = discovered.AccessKeyID
127 }
128 _ = saveState(state)
129 }
130 } else {
131 fmt.Println("Creating object storage...")
132 objState, secretKey, err := provisionObjectStorage(ctx, svc, cfg.Zone, naming.S3Name())
133 if err != nil {
134 return fmt.Errorf("object storage: %w", err)
135 }
136 state.ObjectStorage = objState
137 s3SecretKey = secretKey
138 _ = saveState(state)
139 fmt.Printf(" S3 Secret Key: %s\n", secretKey)
140 }
141
142 fmt.Printf(" Endpoint: %s\n", state.ObjectStorage.Endpoint)
143 fmt.Printf(" Region: %s\n", state.ObjectStorage.Region)
144 fmt.Printf(" Bucket: %s\n", state.ObjectStorage.Bucket)
145 fmt.Printf(" Access Key: %s\n\n", state.ObjectStorage.AccessKeyID)
146
147 // Hold domain is zone-based (e.g. us-chi1.cove.seamark.dev)
148 holdDomain := cfg.Zone + ".cove." + cfg.BaseDomain
149
150 // Build config template values
151 vals := &ConfigValues{
152 S3Endpoint: state.ObjectStorage.Endpoint,
153 S3Region: state.ObjectStorage.Region,
154 S3Bucket: state.ObjectStorage.Bucket,
155 S3AccessKey: state.ObjectStorage.AccessKeyID,
156 S3SecretKey: s3SecretKey,
157 Zone: cfg.Zone,
158 HoldDomain: holdDomain,
159 HoldDid: "did:web:" + holdDomain,
160 BasePath: naming.BasePath(),
161 ScannerSecret: state.ScannerSecret,
162 }
163
164 // 2. Private network
165 if state.Network.UUID != "" {
166 fmt.Printf("Network: %s (exists)\n", state.Network.UUID)
167 } else {
168 fmt.Println("Creating private network...")
169 network, err := svc.CreateNetwork(ctx, &request.CreateNetworkRequest{
170 Name: naming.NetworkName(),
171 Zone: cfg.Zone,
172 IPNetworks: upcloud.IPNetworkSlice{
173 {
174 Address: privateNetworkCIDR,
175 DHCP: upcloud.True,
176 DHCPDefaultRoute: upcloud.False,
177 DHCPDns: []string{"8.8.8.8", "1.1.1.1"},
178 Family: upcloud.IPAddressFamilyIPv4,
179 Gateway: "",
180 },
181 },
182 })
183 if err != nil {
184 return fmt.Errorf("create network: %w", err)
185 }
186 state.Network = StateRef{UUID: network.UUID}
187 _ = saveState(state)
188 fmt.Printf(" Network: %s (%s)\n", network.UUID, privateNetworkCIDR)
189 }
190
191 // Find Debian template (needed for server creation)
192 templateUUID, err := findDebianTemplate(ctx, svc)
193 if err != nil {
194 return err
195 }
196
197 // 3. Appview server
198 appviewCreated := false
199 if state.Appview.UUID != "" {
200 fmt.Printf("Appview: %s (exists)\n", state.Appview.UUID)
201 appviewScript, err := generateAppviewCloudInit(cfg, vals)
202 if err != nil {
203 return err
204 }
205 if err := syncCloudInit("appview", state.Appview.PublicIP, appviewScript); err != nil {
206 return err
207 }
208 appviewConfigYAML, err := renderConfig(appviewConfigTmpl, vals)
209 if err != nil {
210 return fmt.Errorf("render appview config: %w", err)
211 }
212 if err := syncConfigKeys("appview", state.Appview.PublicIP, naming.AppviewConfigPath(), appviewConfigYAML); err != nil {
213 return fmt.Errorf("appview config sync: %w", err)
214 }
215 } else {
216 fmt.Println("Creating appview server...")
217 appviewUserData, err := generateAppviewCloudInit(cfg, vals)
218 if err != nil {
219 return err
220 }
221 appview, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData)
222 if err != nil {
223 return fmt.Errorf("create appview: %w", err)
224 }
225 state.Appview = *appview
226 _ = saveState(state)
227 appviewCreated = true
228 fmt.Printf(" Appview: %s (public: %s, private: %s)\n", appview.UUID, appview.PublicIP, appview.PrivateIP)
229 }
230
231 // 4. Hold server
232 holdCreated := false
233 if state.Hold.UUID != "" {
234 fmt.Printf("Hold: %s (exists)\n", state.Hold.UUID)
235 holdScript, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled)
236 if err != nil {
237 return err
238 }
239 if err := syncCloudInit("hold", state.Hold.PublicIP, holdScript); err != nil {
240 return err
241 }
242 holdConfigYAML, err := renderConfig(holdConfigTmpl, vals)
243 if err != nil {
244 return fmt.Errorf("render hold config: %w", err)
245 }
246 if err := syncConfigKeys("hold", state.Hold.PublicIP, naming.HoldConfigPath(), holdConfigYAML); err != nil {
247 return fmt.Errorf("hold config sync: %w", err)
248 }
249 if state.ScannerEnabled {
250 scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
251 if err != nil {
252 return fmt.Errorf("render scanner config: %w", err)
253 }
254 if err := syncConfigKeys("scanner", state.Hold.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML); err != nil {
255 return fmt.Errorf("scanner config sync: %w", err)
256 }
257 }
258 } else {
259 fmt.Println("Creating hold server...")
260 holdUserData, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled)
261 if err != nil {
262 return err
263 }
264 hold, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Hold(), holdUserData)
265 if err != nil {
266 return fmt.Errorf("create hold: %w", err)
267 }
268 state.Hold = *hold
269 _ = saveState(state)
270 holdCreated = true
271 fmt.Printf(" Hold: %s (public: %s, private: %s)\n", hold.UUID, hold.PublicIP, hold.PrivateIP)
272 }
273
274 // 5. Firewall rules (idempotent — replaces all rules)
275 fmt.Println("Configuring firewall rules...")
276 for _, s := range []struct {
277 name string
278 uuid string
279 }{
280 {"appview", state.Appview.UUID},
281 {"hold", state.Hold.UUID},
282 } {
283 if err := createFirewallRules(ctx, svc, s.uuid, privateNetworkCIDR); err != nil {
284 return fmt.Errorf("firewall %s: %w", s.name, err)
285 }
286 }
287
288 // 6. Load balancer
289 if state.LB.UUID != "" {
290 fmt.Printf("Load balancer: %s (exists)\n", state.LB.UUID)
291 } else {
292 fmt.Println("Creating load balancer (Essentials tier)...")
293 lb, err := createLoadBalancer(ctx, svc, cfg, naming, state.Network.UUID, state.Appview.PrivateIP, state.Hold.PrivateIP, holdDomain)
294 if err != nil {
295 return fmt.Errorf("create LB: %w", err)
296 }
297 state.LB = StateRef{UUID: lb.UUID}
298 _ = saveState(state)
299 }
300
301 // Always reconcile forwarded headers rule (handles existing LBs)
302 if err := ensureLBForwardedHeaders(ctx, svc, state.LB.UUID); err != nil {
303 return fmt.Errorf("LB forwarded headers: %w", err)
304 }
305
306 // Always reconcile TLS certs (handles partial failures and re-runs)
307 tlsDomains := []string{cfg.BaseDomain}
308 tlsDomains = append(tlsDomains, cfg.RegistryDomains...)
309 tlsDomains = append(tlsDomains, holdDomain)
310 if err := ensureLBCertificates(ctx, svc, state.LB.UUID, tlsDomains); err != nil {
311 return fmt.Errorf("LB certificates: %w", err)
312 }
313
314 // Fetch LB DNS name for output
315 lbDNS := ""
316 if state.LB.UUID != "" {
317 lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: state.LB.UUID})
318 if err == nil {
319 for _, n := range lb.Networks {
320 if n.Type == upcloud.LoadBalancerNetworkTypePublic {
321 lbDNS = n.DNSName
322 }
323 }
324 }
325 }
326
327 // 7. Build locally and deploy binaries to new servers
328 if appviewCreated || holdCreated {
329 rootDir := projectRoot()
330
331 if err := runGenerate(rootDir); err != nil {
332 return fmt.Errorf("go generate: %w", err)
333 }
334
335 fmt.Println("\nBuilding locally (GOOS=linux GOARCH=amd64)...")
336 if appviewCreated {
337 outputPath := filepath.Join(rootDir, "bin", "atcr-appview")
338 if err := buildLocal(rootDir, outputPath, "./cmd/appview"); err != nil {
339 return fmt.Errorf("build appview: %w", err)
340 }
341 }
342 if holdCreated {
343 outputPath := filepath.Join(rootDir, "bin", "atcr-hold")
344 if err := buildLocal(rootDir, outputPath, "./cmd/hold"); err != nil {
345 return fmt.Errorf("build hold: %w", err)
346 }
347 if state.ScannerEnabled {
348 outputPath := filepath.Join(rootDir, "bin", "atcr-scanner")
349 if err := buildLocal(filepath.Join(rootDir, "scanner"), outputPath, "./cmd/scanner"); err != nil {
350 return fmt.Errorf("build scanner: %w", err)
351 }
352 }
353 }
354
355 fmt.Println("\nWaiting for cloud-init to complete on new servers...")
356 if appviewCreated {
357 if err := waitForSetup(state.Appview.PublicIP, "appview"); err != nil {
358 return err
359 }
360 }
361 if holdCreated {
362 if err := waitForSetup(state.Hold.PublicIP, "hold"); err != nil {
363 return err
364 }
365 }
366
367 fmt.Println("\nDeploying binaries...")
368 if appviewCreated {
369 localPath := filepath.Join(rootDir, "bin", "atcr-appview")
370 remotePath := naming.InstallDir() + "/bin/" + naming.Appview()
371 if err := scpFile(localPath, state.Appview.PublicIP, remotePath); err != nil {
372 return fmt.Errorf("upload appview: %w", err)
373 }
374 }
375 if holdCreated {
376 localPath := filepath.Join(rootDir, "bin", "atcr-hold")
377 remotePath := naming.InstallDir() + "/bin/" + naming.Hold()
378 if err := scpFile(localPath, state.Hold.PublicIP, remotePath); err != nil {
379 return fmt.Errorf("upload hold: %w", err)
380 }
381 if state.ScannerEnabled {
382 scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
383 scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
384 if err := scpFile(scannerLocal, state.Hold.PublicIP, scannerRemote); err != nil {
385 return fmt.Errorf("upload scanner: %w", err)
386 }
387 }
388 }
389 }
390
391 fmt.Println("\n=== Provisioning Complete ===")
392 fmt.Println()
393 fmt.Println("DNS records needed:")
394 if lbDNS != "" {
395 fmt.Printf(" CNAME %-24s → %s\n", cfg.BaseDomain, lbDNS)
396 for _, rd := range cfg.RegistryDomains {
397 fmt.Printf(" CNAME %-24s → %s\n", rd, lbDNS)
398 }
399 fmt.Printf(" CNAME %-24s → %s\n", holdDomain, lbDNS)
400 } else {
401 fmt.Println(" (LB DNS name not yet available — check 'status' in a few minutes)")
402 }
403 fmt.Println()
404 fmt.Println("SSH access:")
405 fmt.Printf(" ssh root@%s # appview\n", state.Appview.PublicIP)
406 fmt.Printf(" ssh root@%s # hold\n", state.Hold.PublicIP)
407 fmt.Println()
408 fmt.Println("Next steps:")
409 if appviewCreated || holdCreated {
410 fmt.Println(" 1. Edit configs if needed, then start services:")
411 } else {
412 fmt.Println(" 1. Start services:")
413 }
414 if state.ScannerEnabled {
415 fmt.Printf(" systemctl start %s / %s / %s\n", naming.Appview(), naming.Hold(), naming.Scanner())
416 } else {
417 fmt.Printf(" systemctl start %s / %s\n", naming.Appview(), naming.Hold())
418 }
419 fmt.Println(" 2. Configure DNS records above")
420
421 return nil
422}
423
424// provisionObjectStorage creates a new Managed Object Storage with a user, access key, and bucket.
425// Returns the state and the secret key separately (only available at creation time).
426func provisionObjectStorage(ctx context.Context, svc *service.Service, zone, s3Name string) (ObjectStorageState, string, error) {
427 // Map compute zone to object storage region (e.g. us-chi1 → us-east-1)
428 region := objectStorageRegion(zone)
429
430 storage, err := svc.CreateManagedObjectStorage(ctx, &request.CreateManagedObjectStorageRequest{
431 Name: s3Name,
432 Region: region,
433 ConfiguredStatus: upcloud.ManagedObjectStorageConfiguredStatusStarted,
434 Networks: []upcloud.ManagedObjectStorageNetwork{
435 {
436 Family: upcloud.IPAddressFamilyIPv4,
437 Name: "public",
438 Type: "public",
439 },
440 },
441 })
442 if err != nil {
443 return ObjectStorageState{}, "", fmt.Errorf("create storage: %w", err)
444 }
445 fmt.Printf(" Created: %s (region: %s)\n", storage.UUID, region)
446
447 // Find endpoint
448 var endpoint string
449 for _, ep := range storage.Endpoints {
450 if ep.DomainName != "" {
451 endpoint = "https://" + ep.DomainName
452 break
453 }
454 }
455
456 // Create user
457 _, err = svc.CreateManagedObjectStorageUser(ctx, &request.CreateManagedObjectStorageUserRequest{
458 ServiceUUID: storage.UUID,
459 Username: s3Name,
460 })
461 if err != nil {
462 return ObjectStorageState{}, "", fmt.Errorf("create user: %w", err)
463 }
464
465 // Attach admin policy
466 err = svc.AttachManagedObjectStorageUserPolicy(ctx, &request.AttachManagedObjectStorageUserPolicyRequest{
467 ServiceUUID: storage.UUID,
468 Username: s3Name,
469 Name: "admin",
470 })
471 if err != nil {
472 return ObjectStorageState{}, "", fmt.Errorf("attach policy: %w", err)
473 }
474
475 // Create access key (secret is only returned here)
476 accessKey, err := svc.CreateManagedObjectStorageUserAccessKey(ctx, &request.CreateManagedObjectStorageUserAccessKeyRequest{
477 ServiceUUID: storage.UUID,
478 Username: s3Name,
479 })
480 if err != nil {
481 return ObjectStorageState{}, "", fmt.Errorf("create access key: %w", err)
482 }
483
484 secretKey := ""
485 if accessKey.SecretAccessKey != nil {
486 secretKey = *accessKey.SecretAccessKey
487 }
488
489 // Create bucket
490 _, err = svc.CreateManagedObjectStorageBucket(ctx, &request.CreateManagedObjectStorageBucketRequest{
491 ServiceUUID: storage.UUID,
492 Name: s3Name,
493 })
494 if err != nil {
495 return ObjectStorageState{}, "", fmt.Errorf("create bucket: %w", err)
496 }
497
498 return ObjectStorageState{
499 UUID: storage.UUID,
500 Endpoint: endpoint,
501 Region: region,
502 Bucket: s3Name,
503 AccessKeyID: accessKey.AccessKeyID,
504 }, secretKey, nil
505}
506
507// objectStorageRegion maps a compute zone to the nearest object storage region.
508func objectStorageRegion(zone string) string {
509 switch {
510 case strings.HasPrefix(zone, "us-"):
511 return "us-east-1"
512 case strings.HasPrefix(zone, "de-"):
513 return "europe-1"
514 case strings.HasPrefix(zone, "fi-"):
515 return "europe-1"
516 case strings.HasPrefix(zone, "nl-"):
517 return "europe-1"
518 case strings.HasPrefix(zone, "es-"):
519 return "europe-1"
520 case strings.HasPrefix(zone, "pl-"):
521 return "europe-1"
522 case strings.HasPrefix(zone, "se-"):
523 return "europe-1"
524 case strings.HasPrefix(zone, "au-"):
525 return "australia-1"
526 case strings.HasPrefix(zone, "sg-"):
527 return "singapore-1"
528 default:
529 return "us-east-1"
530 }
531}
532
533func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, templateUUID, networkUUID, title, userData string) (*ServerState, error) {
534 storageTier := "maxiops"
535 if strings.HasPrefix(strings.ToUpper(cfg.Plan), "DEV-") {
536 storageTier = "standard"
537 }
538
539 // Look up the plan's storage size from the API instead of hardcoding.
540 diskSize := 25 // fallback
541 plans, err := svc.GetPlans(ctx)
542 if err == nil {
543 for _, p := range plans.Plans {
544 if p.Name == cfg.Plan {
545 diskSize = p.StorageSize
546 break
547 }
548 }
549 }
550
551 details, err := svc.CreateServer(ctx, &request.CreateServerRequest{
552 Zone: cfg.Zone,
553 Title: title,
554 Hostname: title,
555 Plan: cfg.Plan,
556 Metadata: upcloud.True,
557 UserData: userData,
558 Firewall: "on",
559 PasswordDelivery: "none",
560 StorageDevices: request.CreateServerStorageDeviceSlice{
561 {
562 Action: "clone",
563 Storage: templateUUID,
564 Title: title + "-disk",
565 Size: diskSize,
566 Tier: storageTier,
567 },
568 },
569 Networking: &request.CreateServerNetworking{
570 Interfaces: request.CreateServerInterfaceSlice{
571 {
572 Index: 1,
573 Type: upcloud.IPAddressAccessPublic,
574 IPAddresses: request.CreateServerIPAddressSlice{
575 {Family: upcloud.IPAddressFamilyIPv4},
576 },
577 },
578 {
579 Index: 2,
580 Type: upcloud.IPAddressAccessPrivate,
581 Network: networkUUID,
582 IPAddresses: request.CreateServerIPAddressSlice{
583 {Family: upcloud.IPAddressFamilyIPv4},
584 },
585 },
586 },
587 },
588 LoginUser: &request.LoginUser{
589 CreatePassword: "no",
590 SSHKeys: request.SSHKeySlice{cfg.SSHPublicKey},
591 },
592 })
593 if err != nil {
594 return nil, err
595 }
596
597 fmt.Printf(" Waiting for server %s to start...\n", details.UUID)
598 details, err = svc.WaitForServerState(ctx, &request.WaitForServerStateRequest{
599 UUID: details.UUID,
600 DesiredState: upcloud.ServerStateStarted,
601 })
602 if err != nil {
603 return nil, fmt.Errorf("wait for server: %w", err)
604 }
605
606 s := &ServerState{UUID: details.UUID}
607 for _, iface := range details.Networking.Interfaces {
608 for _, addr := range iface.IPAddresses {
609 if addr.Family == upcloud.IPAddressFamilyIPv4 {
610 switch iface.Type {
611 case upcloud.IPAddressAccessPublic:
612 s.PublicIP = addr.Address
613 case upcloud.IPAddressAccessPrivate:
614 s.PrivateIP = addr.Address
615 }
616 }
617 }
618 }
619
620 return s, nil
621}
622
623func createFirewallRules(ctx context.Context, svc *service.Service, serverUUID, privateCIDR string) error {
624 networkBase := strings.TrimSuffix(privateCIDR, "/24")
625 networkBase = strings.TrimSuffix(networkBase, ".0")
626
627 return svc.CreateFirewallRules(ctx, &request.CreateFirewallRulesRequest{
628 ServerUUID: serverUUID,
629 FirewallRules: request.FirewallRuleSlice{
630 {
631 Direction: upcloud.FirewallRuleDirectionIn,
632 Action: upcloud.FirewallRuleActionAccept,
633 Family: upcloud.IPAddressFamilyIPv4,
634 Protocol: upcloud.FirewallRuleProtocolTCP,
635 DestinationPortStart: "22",
636 DestinationPortEnd: "22",
637 Position: 1,
638 Comment: "Allow SSH",
639 },
640 {
641 Direction: upcloud.FirewallRuleDirectionIn,
642 Action: upcloud.FirewallRuleActionAccept,
643 Family: upcloud.IPAddressFamilyIPv4,
644 SourceAddressStart: networkBase + ".0",
645 SourceAddressEnd: networkBase + ".255",
646 Position: 2,
647 Comment: "Allow private network",
648 },
649 {
650 Direction: upcloud.FirewallRuleDirectionIn,
651 Action: upcloud.FirewallRuleActionAccept,
652 Family: upcloud.IPAddressFamilyIPv4,
653 Protocol: upcloud.FirewallRuleProtocolUDP,
654 SourcePortStart: "123",
655 SourcePortEnd: "123",
656 Position: 3,
657 Comment: "Allow NTP replies",
658 },
659 {
660 Direction: upcloud.FirewallRuleDirectionIn,
661 Action: upcloud.FirewallRuleActionDrop,
662 Position: 4,
663 Comment: "Drop all other inbound",
664 },
665 },
666 })
667}
668
669func createLoadBalancer(ctx context.Context, svc *service.Service, cfg *InfraConfig, naming Naming, networkUUID, appviewIP, holdIP, holdDomain string) (*upcloud.LoadBalancer, error) {
670 lb, err := svc.CreateLoadBalancer(ctx, &request.CreateLoadBalancerRequest{
671 Name: naming.LBName(),
672 Plan: "essentials",
673 Zone: cfg.Zone,
674 ConfiguredStatus: upcloud.LoadBalancerConfiguredStatusStarted,
675 Networks: []request.LoadBalancerNetwork{
676 {
677 Name: "public",
678 Type: upcloud.LoadBalancerNetworkTypePublic,
679 Family: upcloud.LoadBalancerAddressFamilyIPv4,
680 },
681 {
682 Name: "private",
683 Type: upcloud.LoadBalancerNetworkTypePrivate,
684 Family: upcloud.LoadBalancerAddressFamilyIPv4,
685 UUID: networkUUID,
686 },
687 },
688 Frontends: []request.LoadBalancerFrontend{
689 {
690 Name: "https",
691 Mode: upcloud.LoadBalancerModeHTTP,
692 Port: 443,
693 DefaultBackend: "appview",
694 Networks: []upcloud.LoadBalancerFrontendNetwork{
695 {Name: "public"},
696 },
697 Rules: []request.LoadBalancerFrontendRule{
698 {
699 Name: "set-forwarded-headers",
700 Priority: 1,
701 Matchers: []upcloud.LoadBalancerMatcher{},
702 Actions: []upcloud.LoadBalancerAction{
703 request.NewLoadBalancerSetForwardedHeadersAction(),
704 },
705 },
706 {
707 Name: "route-hold",
708 Priority: 10,
709 Matchers: []upcloud.LoadBalancerMatcher{
710 {
711 Type: upcloud.LoadBalancerMatcherTypeHost,
712 Host: &upcloud.LoadBalancerMatcherHost{
713 Value: holdDomain,
714 },
715 },
716 },
717 Actions: []upcloud.LoadBalancerAction{
718 {
719 Type: upcloud.LoadBalancerActionTypeUseBackend,
720 UseBackend: &upcloud.LoadBalancerActionUseBackend{
721 Backend: "hold",
722 },
723 },
724 },
725 },
726 },
727 },
728 {
729 Name: "http-redirect",
730 Mode: upcloud.LoadBalancerModeHTTP,
731 Port: 80,
732 DefaultBackend: "appview",
733 Networks: []upcloud.LoadBalancerFrontendNetwork{
734 {Name: "public"},
735 },
736 Rules: []request.LoadBalancerFrontendRule{
737 {
738 Name: "redirect-https",
739 Priority: 10,
740 Matchers: []upcloud.LoadBalancerMatcher{
741 {
742 Type: upcloud.LoadBalancerMatcherTypeSrcPort,
743 SrcPort: &upcloud.LoadBalancerMatcherInteger{
744 Method: upcloud.LoadBalancerIntegerMatcherMethodEqual,
745 Value: 80,
746 },
747 },
748 },
749 Actions: []upcloud.LoadBalancerAction{
750 {
751 Type: upcloud.LoadBalancerActionTypeHTTPRedirect,
752 HTTPRedirect: &upcloud.LoadBalancerActionHTTPRedirect{
753 Scheme: upcloud.LoadBalancerActionHTTPRedirectSchemeHTTPS,
754 },
755 },
756 },
757 },
758 },
759 },
760 },
761 Resolvers: []request.LoadBalancerResolver{},
762 Backends: []request.LoadBalancerBackend{
763 {
764 Name: "appview",
765 Members: []request.LoadBalancerBackendMember{
766 {
767 Name: "appview-1",
768 Type: upcloud.LoadBalancerBackendMemberTypeStatic,
769 IP: appviewIP,
770 Port: 5000,
771 Weight: 100,
772 MaxSessions: 1000,
773 Enabled: true,
774 },
775 },
776 Properties: &upcloud.LoadBalancerBackendProperties{
777 HealthCheckType: upcloud.LoadBalancerHealthCheckTypeHTTP,
778 HealthCheckURL: "/health",
779 },
780 },
781 {
782 Name: "hold",
783 Members: []request.LoadBalancerBackendMember{
784 {
785 Name: "hold-1",
786 Type: upcloud.LoadBalancerBackendMemberTypeStatic,
787 IP: holdIP,
788 Port: 8080,
789 Weight: 100,
790 MaxSessions: 1000,
791 Enabled: true,
792 },
793 },
794 Properties: &upcloud.LoadBalancerBackendProperties{
795 HealthCheckType: upcloud.LoadBalancerHealthCheckTypeHTTP,
796 HealthCheckURL: "/xrpc/_health",
797 },
798 },
799 },
800 })
801 if err != nil {
802 return nil, err
803 }
804
805 return lb, nil
806}
807
808// ensureLBCertificates reconciles TLS certificate bundles on the load balancer.
809// It skips domains that already have a TLS config attached and creates missing ones.
810func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID string, tlsDomains []string) error {
811 lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: lbUUID})
812 if err != nil {
813 return fmt.Errorf("get load balancer: %w", err)
814 }
815
816 // Build set of existing TLS config names on the "https" frontend
817 existing := make(map[string]bool)
818 for _, fe := range lb.Frontends {
819 if fe.Name == "https" {
820 for _, tc := range fe.TLSConfigs {
821 existing[tc.Name] = true
822 }
823 }
824 }
825
826 for _, domain := range tlsDomains {
827 certName := "tls-" + strings.ReplaceAll(domain, ".", "-")
828 if existing[certName] {
829 fmt.Printf(" TLS certificate: %s (exists)\n", domain)
830 continue
831 }
832
833 bundle, err := svc.CreateLoadBalancerCertificateBundle(ctx, &request.CreateLoadBalancerCertificateBundleRequest{
834 Type: upcloud.LoadBalancerCertificateBundleTypeDynamic,
835 Name: certName,
836 KeyType: "ecdsa",
837 Hostnames: []string{domain},
838 })
839 if err != nil {
840 return fmt.Errorf("create TLS cert for %s: %w", domain, err)
841 }
842
843 _, err = svc.CreateLoadBalancerFrontendTLSConfig(ctx, &request.CreateLoadBalancerFrontendTLSConfigRequest{
844 ServiceUUID: lbUUID,
845 FrontendName: "https",
846 Config: request.LoadBalancerFrontendTLSConfig{
847 Name: certName,
848 CertificateBundleUUID: bundle.UUID,
849 },
850 })
851 if err != nil {
852 return fmt.Errorf("attach TLS cert %s to frontend: %w", domain, err)
853 }
854 fmt.Printf(" TLS certificate: %s\n", domain)
855 }
856
857 return nil
858}
859
860// ensureLBForwardedHeaders ensures the "https" frontend has a set_forwarded_headers rule.
861// This makes the LB set X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers,
862// overwriting any pre-existing values (prevents spoofing).
863func ensureLBForwardedHeaders(ctx context.Context, svc *service.Service, lbUUID string) error {
864 rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
865 ServiceUUID: lbUUID,
866 FrontendName: "https",
867 })
868 if err != nil {
869 return fmt.Errorf("get frontend rules: %w", err)
870 }
871
872 for _, r := range rules {
873 if r.Name == "set-forwarded-headers" {
874 fmt.Println(" Forwarded headers rule: exists")
875 return nil
876 }
877 }
878
879 _, err = svc.CreateLoadBalancerFrontendRule(ctx, &request.CreateLoadBalancerFrontendRuleRequest{
880 ServiceUUID: lbUUID,
881 FrontendName: "https",
882 Rule: request.LoadBalancerFrontendRule{
883 Name: "set-forwarded-headers",
884 Priority: 1,
885 Matchers: []upcloud.LoadBalancerMatcher{},
886 Actions: []upcloud.LoadBalancerAction{
887 request.NewLoadBalancerSetForwardedHeadersAction(),
888 },
889 },
890 })
891 if err != nil {
892 return fmt.Errorf("create forwarded headers rule: %w", err)
893 }
894 fmt.Println(" Forwarded headers rule: created")
895
896 return nil
897}
898
899// lookupObjectStorage discovers details of an existing Managed Object Storage.
900func lookupObjectStorage(ctx context.Context, svc *service.Service, uuid string) (ObjectStorageState, error) {
901 storage, err := svc.GetManagedObjectStorage(ctx, &request.GetManagedObjectStorageRequest{
902 UUID: uuid,
903 })
904 if err != nil {
905 return ObjectStorageState{}, fmt.Errorf("get object storage %s: %w", uuid, err)
906 }
907
908 var endpoint string
909 for _, ep := range storage.Endpoints {
910 if ep.DomainName != "" {
911 endpoint = "https://" + ep.DomainName
912 break
913 }
914 }
915
916 var bucket string
917 buckets, err := svc.GetManagedObjectStorageBucketMetrics(ctx, &request.GetManagedObjectStorageBucketMetricsRequest{
918 ServiceUUID: uuid,
919 })
920 if err == nil {
921 for _, b := range buckets {
922 if !b.Deleted {
923 bucket = b.Name
924 break
925 }
926 }
927 }
928
929 var accessKeyID string
930 users, err := svc.GetManagedObjectStorageUsers(ctx, &request.GetManagedObjectStorageUsersRequest{
931 ServiceUUID: uuid,
932 })
933 if err == nil {
934 for _, u := range users {
935 for _, k := range u.AccessKeys {
936 if k.Status == "Active" {
937 accessKeyID = k.AccessKeyID
938 break
939 }
940 }
941 if accessKeyID != "" {
942 break
943 }
944 }
945 }
946
947 return ObjectStorageState{
948 UUID: uuid,
949 Endpoint: endpoint,
950 Region: storage.Region,
951 Bucket: bucket,
952 AccessKeyID: accessKeyID,
953 }, nil
954}
955
956func findDebianTemplate(ctx context.Context, svc *service.Service) (string, error) {
957 storages, err := svc.GetStorages(ctx, &request.GetStoragesRequest{
958 Type: "template",
959 })
960 if err != nil {
961 return "", fmt.Errorf("list templates: %w", err)
962 }
963
964 var debian13, debian12 string
965 for _, s := range storages.Storages {
966 title := strings.ToLower(s.Title)
967 if strings.Contains(title, "debian") {
968 if strings.Contains(title, "13") || strings.Contains(title, "trixie") {
969 debian13 = s.UUID
970 } else if strings.Contains(title, "12") || strings.Contains(title, "bookworm") {
971 debian12 = s.UUID
972 }
973 }
974 }
975
976 if debian13 != "" {
977 return debian13, nil
978 }
979 if debian12 != "" {
980 fmt.Println(" Debian 13 not available, using Debian 12")
981 return debian12, nil
982 }
983
984 return "", fmt.Errorf("no Debian template found — check UpCloud template list")
985}
986
987const cloudInitPath = "/var/lib/cloud/instance/scripts/part-001"
988
989// syncCloudInit compares a locally-generated cloud-init script against what's
990// on the server. If they differ (or the remote is missing), it prompts the
991// user and re-runs the script over SSH.
992func syncCloudInit(name, ip, localScript string) error {
993 // Fetch the remote script
994 remoteScript, err := runSSH(ip, fmt.Sprintf("cat %s 2>/dev/null || echo '__MISSING__'", cloudInitPath), false)
995 if err != nil {
996 fmt.Printf(" cloud-init: could not reach %s (%v)\n", name, err)
997 return nil
998 }
999 remoteScript = strings.TrimSpace(remoteScript)
1000
1001 if remoteScript == "__MISSING__" {
1002 fmt.Printf(" cloud-init: not found on %s (server may need initial setup)\n", name)
1003 } else {
1004 localHash := fmt.Sprintf("%x", sha256.Sum256([]byte(strings.TrimSpace(localScript))))
1005 remoteHash := fmt.Sprintf("%x", sha256.Sum256([]byte(remoteScript)))
1006 if localHash == remoteHash {
1007 fmt.Printf(" cloud-init: up to date\n")
1008 return nil
1009 }
1010 fmt.Printf(" cloud-init: differs from local\n")
1011 }
1012
1013 fmt.Printf(" Re-run cloud-init on %s? [Y/n] ", name)
1014 scanner := bufio.NewScanner(os.Stdin)
1015 scanner.Scan()
1016 answer := strings.TrimSpace(strings.ToLower(scanner.Text()))
1017 if answer != "" && answer != "y" && answer != "yes" {
1018 fmt.Printf(" Skipped\n")
1019 // Still update the remote reference so next provision sees an accurate diff
1020 if err := writeRemoteCloudInit(ip, localScript); err != nil {
1021 fmt.Printf(" WARNING: could not update remote cloud-init reference: %v\n", err)
1022 }
1023 return nil
1024 }
1025
1026 // Write the reference file first so next provision can detect real diffs,
1027 // regardless of whether the script execution succeeds or fails.
1028 if err := writeRemoteCloudInit(ip, localScript); err != nil {
1029 fmt.Printf(" WARNING: could not update remote cloud-init reference: %v\n", err)
1030 }
1031
1032 fmt.Printf(" Running cloud-init on %s (%s)... (this may take several minutes)\n", name, ip)
1033 output, err := runSSH(ip, localScript, true)
1034 if err != nil {
1035 fmt.Printf(" ERROR: %v\n", err)
1036 fmt.Printf(" Output:\n%s\n", output)
1037 return fmt.Errorf("cloud-init %s failed", name)
1038 }
1039
1040 fmt.Printf(" %s: cloud-init complete\n", name)
1041 return nil
1042}
1043
1044// generateScannerSecret generates a random 32-byte hex-encoded shared secret
1045// for authenticating scanner-to-hold WebSocket connections.
1046func generateScannerSecret() (string, error) {
1047 b := make([]byte, 32)
1048 if _, err := crypto_rand.Read(b); err != nil {
1049 return "", err
1050 }
1051 return hex.EncodeToString(b), nil
1052}
1053
1054// writeRemoteCloudInit writes the local cloud-init script to the remote server
1055// so that subsequent provision runs can accurately detect real changes.
1056// Uses base64 encoding to avoid heredoc nesting issues (the cloud-init script
1057// itself contains heredocs like CFGEOF and SVCEOF).
1058func writeRemoteCloudInit(ip, script string) error {
1059 encoded := base64.StdEncoding.EncodeToString([]byte(script))
1060 cmd := fmt.Sprintf("mkdir -p $(dirname %s) && echo '%s' | base64 -d > %s", cloudInitPath, encoded, cloudInitPath)
1061 _, err := runSSH(ip, cmd, false)
1062 return err
1063}
1064
1065// waitForSetup polls SSH availability on a newly created server, then waits
1066// for cloud-init to complete before returning.
1067func waitForSetup(ip, name string) error {
1068 fmt.Printf(" %s (%s): waiting for SSH...\n", name, ip)
1069 for i := 0; i < 30; i++ {
1070 _, err := runSSH(ip, "echo ssh_ready", false)
1071 if err == nil {
1072 break
1073 }
1074 if i == 29 {
1075 return fmt.Errorf("SSH not available after 5 minutes on %s (%s)", name, ip)
1076 }
1077 time.Sleep(10 * time.Second)
1078 }
1079
1080 fmt.Printf(" %s: waiting for cloud-init...\n", name)
1081 _, err := runSSH(ip, "cloud-init status --wait 2>/dev/null || true", false)
1082 if err != nil {
1083 return fmt.Errorf("cloud-init wait on %s: %w", name, err)
1084 }
1085 fmt.Printf(" %s: ready\n", name)
1086 return nil
1087}