-1

pkg/appview/public/icons.svg

··· 49 49 <symbol id="user" viewBox="0 0 24 24"><path d="M19 21v-2a4 4 0 0 0-4-4H9a4 4 0 0 0-4 4v2"/><circle cx="12" cy="7" r="4"/></symbol> 50 50 <symbol id="user-plus" viewBox="0 0 24 24"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"/><circle cx="9" cy="7" r="4"/><line x1="19" x2="19" y1="8" y2="14"/><line x1="22" x2="16" y1="11" y2="11"/></symbol> 51 51 <symbol id="x-circle" viewBox="0 0 24 24"><circle cx="12" cy="12" r="10"/><path d="m15 9-6 6"/><path d="m9 9 6 6"/></symbol> 52 52 - <symbol id="zap" viewBox="0 0 24 24"><path d="M4 14a1 1 0 0 1-.78-1.63l9.9-10.2a.5.5 0 0 1 .86.46l-1.92 6.02A1 1 0 0 0 13 10h7a1 1 0 0 1 .78 1.63l-9.9 10.2a.5.5 0 0 1-.86-.46l1.92-6.02A1 1 0 0 0 11 14z"/></symbol> 53 52 <symbol id="helm" viewBox="0 0 24 24"><path d="M12.337 0c-.475 0-.861 1.016-.861 2.269 0 .527.069 1.011.183 1.396a8.514 8.514 0 0 0-3.961 1.22 5.229 5.229 0 0 0-.595-1.093c-.606-.866-1.34-1.436-1.79-1.43a.381.381 0 0 0-.217.066c-.39.273-.123 1.326.596 2.353.267.381.559.705.84.948a8.683 8.683 0 0 0-1.528 1.716h1.734a7.179 7.179 0 0 1 5.381-2.421 7.18 7.18 0 0 1 5.382 2.42h1.733a8.687 8.687 0 0 0-1.32-1.53c.35-.249.735-.643 1.078-1.133.719-1.027.986-2.08.596-2.353a.382.382 0 0 0-.217-.065c-.45-.007-1.184.563-1.79 1.43a4.897 4.897 0 0 0-.676 1.325 8.52 8.52 0 0 0-3.899-1.42c.12-.39.193-.887.193-1.429 0-1.253-.386-2.269-.862-2.269zM1.624 9.443v5.162h1.358v-1.968h1.64v1.968h1.357V9.443H4.62v1.838H2.98V9.443zm5.912 0v5.162h3.21v-1.108H8.893v-.95h1.64v-1.142h-1.64v-.84h1.853V9.443zm4.698 0v5.162h3.218v-1.362h-1.86v-3.8zm4.706 0v5.162h1.364v-2.643l1.357 1.225 1.35-1.232v2.65h1.365V9.443h-.614l-2.1 1.914-2.109-1.914zm-11.82 7.28a8.688 8.688 0 0 0 1.412 1.548 5.206 5.206 0 0 0-.841.948c-.719 1.027-.985 2.08-.596 2.353.39.273 1.289-.338 2.007-1.364a5.23 5.23 0 0 0 .595-1.092 8.514 8.514 0 0 0 3.961 1.219 5.01 5.01 0 0 0-.183 1.396c0 1.253.386 2.269.861 2.269.476 0 .862-1.016.862-2.269 0-.542-.072-1.04-.193-1.43a8.52 8.52 0 0 0 3.9-1.42c.121.4.352.865.675 1.327.719 1.026 1.617 1.637 2.007 1.364.39-.273.123-1.326-.596-2.353-.343-.49-.727-.885-1.077-1.135a8.69 8.69 0 0 0 1.202-1.36h-1.771a7.174 7.174 0 0 1-5.227 2.252 7.174 7.174 0 0 1-5.226-2.252z" fill="currentColor" stroke="none"/></symbol> 54 53 </svg>

+3

pkg/hold/admin/admin.go

··· 414 414 // GC POSTs 415 415 r.Post("/admin/api/gc/preview", ui.handleGCPreview) 416 416 r.Post("/admin/api/gc/run", ui.handleGCRun) 417 417 + r.Post("/admin/api/gc/reconcile", ui.handleGCReconcile) 418 418 + r.Post("/admin/api/gc/delete-records", ui.handleGCDeleteRecords) 419 419 + r.Post("/admin/api/gc/delete-blobs", ui.handleGCDeleteBlobs) 417 420 418 421 // API endpoints (for HTMX) 419 422 r.Get("/admin/api/stats", ui.handleStatsAPI)

+77

pkg/hold/admin/handlers_gc.go

··· 99 99 }{Result: result}) 100 100 } 101 101 102 102 + // handleGCReconcile creates missing layer records without deleting anything 103 103 + func (ui *AdminUI) handleGCReconcile(w http.ResponseWriter, r *http.Request) { 104 104 + if ui.gc == nil { 105 105 + ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"}) 106 106 + return 107 107 + } 108 108 + 109 109 + result, err := ui.gc.Reconcile(r.Context()) 110 110 + if err != nil { 111 111 + slog.Error("GC reconcile failed", "error", err) 112 112 + ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()}) 113 113 + return 114 114 + } 115 115 + 116 116 + session := getSessionFromContext(r.Context()) 117 117 + slog.Info("GC reconcile completed via admin panel", 118 118 + "recordsReconciled", result.RecordsReconciled, 119 119 + "duration", result.Duration, 120 120 + "by", session.DID) 121 121 + 122 122 + ui.renderTemplate(w, "partials/gc_result.html", struct { 123 123 + Result *gc.GCResult 124 124 + }{Result: result}) 125 125 + } 126 126 + 127 127 + // handleGCDeleteRecords deletes orphaned layer records (no blob deletion) 128 128 + func (ui *AdminUI) handleGCDeleteRecords(w http.ResponseWriter, r *http.Request) { 129 129 + if ui.gc == nil { 130 130 + ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"}) 131 131 + return 132 132 + } 133 133 + 134 134 + result, err := ui.gc.DeleteOrphanedRecords(r.Context()) 135 135 + if err != nil { 136 136 + slog.Error("GC delete records failed", "error", err) 137 137 + ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()}) 138 138 + return 139 139 + } 140 140 + 141 141 + session := getSessionFromContext(r.Context()) 142 142 + slog.Info("GC delete orphaned records completed via admin panel", 143 143 + "recordsDeleted", result.RecordsDeleted, 144 144 + "orphanedRecords", result.OrphanedRecords, 145 145 + "duration", result.Duration, 146 146 + "by", session.DID) 147 147 + 148 148 + ui.renderTemplate(w, "partials/gc_result.html", struct { 149 149 + Result *gc.GCResult 150 150 + }{Result: result}) 151 151 + } 152 152 + 153 153 + // handleGCDeleteBlobs walks S3 and deletes unreferenced blobs 154 154 + func (ui *AdminUI) handleGCDeleteBlobs(w http.ResponseWriter, r *http.Request) { 155 155 + if ui.gc == nil { 156 156 + ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{"GC not available"}) 157 157 + return 158 158 + } 159 159 + 160 160 + result, err := ui.gc.DeleteOrphanedBlobs(r.Context()) 161 161 + if err != nil { 162 162 + slog.Error("GC delete blobs failed", "error", err) 163 163 + ui.renderTemplate(w, "partials/gc_error.html", struct{ Error string }{err.Error()}) 164 164 + return 165 165 + } 166 166 + 167 167 + session := getSessionFromContext(r.Context()) 168 168 + slog.Info("GC delete orphaned blobs completed via admin panel", 169 169 + "blobsDeleted", result.BlobsDeleted, 170 170 + "bytesReclaimed", result.BytesReclaimed, 171 171 + "duration", result.Duration, 172 172 + "by", session.DID) 173 173 + 174 174 + ui.renderTemplate(w, "partials/gc_result.html", struct { 175 175 + Result *gc.GCResult 176 176 + }{Result: result}) 177 177 + } 178 178 + 102 179 // timeAgo returns a human-readable relative time string 103 180 func timeAgo(t time.Time) string { 104 181 if t.IsZero() {

-1

pkg/hold/admin/public/icons.svg

··· 49 49 <symbol id="user" viewBox="0 0 24 24"><path d="M19 21v-2a4 4 0 0 0-4-4H9a4 4 0 0 0-4 4v2"/><circle cx="12" cy="7" r="4"/></symbol> 50 50 <symbol id="user-plus" viewBox="0 0 24 24"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"/><circle cx="9" cy="7" r="4"/><line x1="19" x2="19" y1="8" y2="14"/><line x1="22" x2="16" y1="11" y2="11"/></symbol> 51 51 <symbol id="x-circle" viewBox="0 0 24 24"><circle cx="12" cy="12" r="10"/><path d="m15 9-6 6"/><path d="m9 9 6 6"/></symbol> 52 52 - <symbol id="zap" viewBox="0 0 24 24"><path d="M4 14a1 1 0 0 1-.78-1.63l9.9-10.2a.5.5 0 0 1 .86.46l-1.92 6.02A1 1 0 0 0 13 10h7a1 1 0 0 1 .78 1.63l-9.9 10.2a.5.5 0 0 1-.86-.46l1.92-6.02A1 1 0 0 0 11 14z"/></symbol> 53 52 <symbol id="helm" viewBox="0 0 24 24"><path d="M12.337 0c-.475 0-.861 1.016-.861 2.269 0 .527.069 1.011.183 1.396a8.514 8.514 0 0 0-3.961 1.22 5.229 5.229 0 0 0-.595-1.093c-.606-.866-1.34-1.436-1.79-1.43a.381.381 0 0 0-.217.066c-.39.273-.123 1.326.596 2.353.267.381.559.705.84.948a8.683 8.683 0 0 0-1.528 1.716h1.734a7.179 7.179 0 0 1 5.381-2.421 7.18 7.18 0 0 1 5.382 2.42h1.733a8.687 8.687 0 0 0-1.32-1.53c.35-.249.735-.643 1.078-1.133.719-1.027.986-2.08.596-2.353a.382.382 0 0 0-.217-.065c-.45-.007-1.184.563-1.79 1.43a4.897 4.897 0 0 0-.676 1.325 8.52 8.52 0 0 0-3.899-1.42c.12-.39.193-.887.193-1.429 0-1.253-.386-2.269-.862-2.269zM1.624 9.443v5.162h1.358v-1.968h1.64v1.968h1.357V9.443H4.62v1.838H2.98V9.443zm5.912 0v5.162h3.21v-1.108H8.893v-.95h1.64v-1.142h-1.64v-.84h1.853V9.443zm4.698 0v5.162h3.218v-1.362h-1.86v-3.8zm4.706 0v5.162h1.364v-2.643l1.357 1.225 1.35-1.232v2.65h1.365V9.443h-.614l-2.1 1.914-2.109-1.914zm-11.82 7.28a8.688 8.688 0 0 0 1.412 1.548 5.206 5.206 0 0 0-.841.948c-.719 1.027-.985 2.08-.596 2.353.39.273 1.289-.338 2.007-1.364a5.23 5.23 0 0 0 .595-1.092 8.514 8.514 0 0 0 3.961 1.219 5.01 5.01 0 0 0-.183 1.396c0 1.253.386 2.269.861 2.269.476 0 .862-1.016.862-2.269 0-.542-.072-1.04-.193-1.43a8.52 8.52 0 0 0 3.9-1.42c.121.4.352.865.675 1.327.719 1.026 1.617 1.637 2.007 1.364.39-.273.123-1.326-.596-2.353-.343-.49-.727-.885-1.077-1.135a8.69 8.69 0 0 0 1.202-1.36h-1.771a7.174 7.174 0 0 1-5.227 2.252 7.174 7.174 0 0 1-5.226-2.252z" fill="currentColor" stroke="none"/></symbol> 54 53 </svg>

+30 -6

pkg/hold/admin/templates/partials/gc_preview.html

··· 135 135 </div> 136 136 {{end}} 137 137 138 138 - <!-- Run GC button (only if there are actions to take) --> 138 138 + <!-- Individual action buttons --> 139 139 {{if or .Preview.OrphanedRecords .Preview.OrphanedBlobs .Preview.MissingRecords}} 140 140 - <div class="flex items-center gap-3 mt-6"> 140 140 + <div class="flex flex-wrap items-center gap-3 mt-6"> 141 141 + {{if .Preview.MissingRecords}} 142 142 + <button class="btn btn-warning gap-2" 143 143 + hx-post="/admin/api/gc/reconcile" 144 144 + hx-target="#gc-results" 145 145 + hx-swap="innerHTML" 146 146 + hx-indicator="#gc-loading"> 147 147 + {{ icon "file-plus" "size-4" }} 148 148 + Reconcile {{len .Preview.MissingRecords}} Records 149 149 + </button> 150 150 + {{end}} 151 151 + {{if .Preview.OrphanedRecords}} 141 152 <button class="btn btn-error gap-2" 142 142 - hx-post="/admin/api/gc/run" 153 153 + hx-post="/admin/api/gc/delete-records" 143 154 hx-target="#gc-results" 144 155 hx-swap="innerHTML" 145 145 - hx-confirm="Are you sure you want to run garbage collection?" 156 156 + hx-confirm="Delete {{len .Preview.OrphanedRecords}} orphaned layer records?" 146 157 hx-indicator="#gc-loading"> 147 147 - {{ icon "zap" "size-4" }} 148 148 - Run GC 158 158 + {{ icon "file-x" "size-4" }} 159 159 + Delete {{len .Preview.OrphanedRecords}} Orphaned Records 149 160 </button> 161 161 + {{end}} 162 162 + {{if .Preview.OrphanedBlobs}} 163 163 + <button class="btn btn-error gap-2" 164 164 + hx-post="/admin/api/gc/delete-blobs" 165 165 + hx-target="#gc-results" 166 166 + hx-swap="innerHTML" 167 167 + hx-confirm="Delete {{len .Preview.OrphanedBlobs}} orphaned blobs from S3? This cannot be undone." 168 168 + hx-indicator="#gc-loading"> 169 169 + {{ icon "trash-2" "size-4" }} 170 170 + Delete {{len .Preview.OrphanedBlobs}} Orphaned Blobs 171 171 + </button> 172 172 + {{end}} 150 173 </div> 174 174 + <p class="text-sm text-base-content/50 mt-2">Run Scan again after each operation to see updated counts.</p> 151 175 {{end}} 152 176 153 177 <!-- Nothing to clean -->

+2 -2

pkg/hold/config.go

··· 141 141 Secret string `yaml:"secret" comment:"Shared secret for scanner WebSocket auth. Empty disables scanning."` 142 142 143 143 // Minimum interval between re-scans of the same manifest. 0 disables proactive scanning. 144 144 - RescanInterval time.Duration `yaml:"rescan_interval" comment:"Minimum interval between re-scans of the same manifest. When set, the hold proactively scans manifests when the scanner is idle. Default: 24h. Set to 0 to disable."` 144 144 + RescanInterval time.Duration `yaml:"rescan_interval" comment:"Minimum interval between re-scans of the same manifest. When set, the hold proactively scans manifests when the scanner is idle. Default: 168h (7 days). Set to 0 to disable."` 145 145 } 146 146 147 147 // DatabaseConfig defines embedded PDS database settings ··· 223 223 v.SetDefault("gc.enabled", false) 224 224 // Scanner defaults 225 225 v.SetDefault("scanner.secret", "") 226 226 - v.SetDefault("scanner.rescan_interval", "24h") 226 226 + v.SetDefault("scanner.rescan_interval", "168h") // 7 days 227 227 228 228 // Log shipper defaults 229 229 v.SetDefault("log_shipper.batch_size", 100)

+112

pkg/hold/gc/gc.go

··· 296 296 return preview, nil 297 297 } 298 298 299 299 + // Reconcile creates missing layer records without deleting anything. 300 300 + // Requires a prior Preview() to identify missing records. 301 301 + func (gc *GarbageCollector) Reconcile(ctx context.Context) (*GCResult, error) { 302 302 + if !gc.tryStart() { 303 303 + return nil, fmt.Errorf("GC operation already in progress") 304 304 + } 305 305 + defer gc.finish() 306 306 + 307 307 + gc.mu.Lock() 308 308 + preview := gc.lastPreview 309 309 + gc.mu.Unlock() 310 310 + 311 311 + if preview == nil { 312 312 + return nil, fmt.Errorf("no preview available — run Scan first") 313 313 + } 314 314 + if len(preview.MissingRecords) == 0 { 315 315 + return &GCResult{}, nil 316 316 + } 317 317 + 318 318 + start := time.Now() 319 319 + result := &GCResult{} 320 320 + 321 321 + gc.logger.Info("Starting reconciliation", "missingRecords", len(preview.MissingRecords)) 322 322 + gc.reconcileMissingRecords(ctx, preview.MissingRecords, result) 323 323 + result.Duration = time.Since(start) 324 324 + 325 325 + gc.mu.Lock() 326 326 + gc.lastResult = result 327 327 + gc.lastResultAt = time.Now() 328 328 + gc.mu.Unlock() 329 329 + 330 330 + return result, nil 331 331 + } 332 332 + 333 333 + // DeleteOrphanedRecords deletes layer records whose manifests no longer exist. 334 334 + // Requires a prior Preview() to identify orphaned records. 335 335 + func (gc *GarbageCollector) DeleteOrphanedRecords(ctx context.Context) (*GCResult, error) { 336 336 + if !gc.tryStart() { 337 337 + return nil, fmt.Errorf("GC operation already in progress") 338 338 + } 339 339 + defer gc.finish() 340 340 + 341 341 + gc.mu.Lock() 342 342 + preview := gc.lastPreview 343 343 + gc.mu.Unlock() 344 344 + 345 345 + if preview == nil { 346 346 + return nil, fmt.Errorf("no preview available — run Scan first") 347 347 + } 348 348 + if len(preview.OrphanedRecords) == 0 { 349 349 + return &GCResult{}, nil 350 350 + } 351 351 + 352 352 + start := time.Now() 353 353 + result := &GCResult{ 354 354 + OrphanedRecords: int64(len(preview.OrphanedRecords)), 355 355 + } 356 356 + 357 357 + rkeys := make([]string, len(preview.OrphanedRecords)) 358 358 + for i, r := range preview.OrphanedRecords { 359 359 + rkeys[i] = r.Rkey 360 360 + } 361 361 + 362 362 + gc.logger.Info("Deleting orphaned records", "count", len(rkeys)) 363 363 + if err := gc.deleteOrphanedRecords(ctx, rkeys, result); err != nil { 364 364 + return nil, fmt.Errorf("delete orphaned records: %w", err) 365 365 + } 366 366 + result.Duration = time.Since(start) 367 367 + 368 368 + gc.mu.Lock() 369 369 + gc.lastResult = result 370 370 + gc.lastResultAt = time.Now() 371 371 + gc.mu.Unlock() 372 372 + 373 373 + return result, nil 374 374 + } 375 375 + 376 376 + // DeleteOrphanedBlobs walks S3 and deletes blobs not referenced by any manifest. 377 377 + // Runs a fresh analysis to build the current referenced set (reflects any reconciliation 378 378 + // done since the last preview). 379 379 + func (gc *GarbageCollector) DeleteOrphanedBlobs(ctx context.Context) (*GCResult, error) { 380 380 + if !gc.tryStart() { 381 381 + return nil, fmt.Errorf("GC operation already in progress") 382 382 + } 383 383 + defer gc.finish() 384 384 + 385 385 + start := time.Now() 386 386 + result := &GCResult{} 387 387 + 388 388 + gc.logger.Info("Starting orphaned blob deletion (fresh analysis)") 389 389 + 390 390 + // Fresh analysis so the referenced set includes any records reconciled since preview 391 391 + analysis, err := gc.analyzeRecords(ctx) 392 392 + if err != nil { 393 393 + return nil, fmt.Errorf("analyze records: %w", err) 394 394 + } 395 395 + 396 396 + result.ReferencedBlobs = int64(len(analysis.referenced)) 397 397 + 398 398 + if err := gc.deleteOrphanedBlobs(ctx, analysis.referenced, result); err != nil { 399 399 + return nil, fmt.Errorf("delete orphaned blobs: %w", err) 400 400 + } 401 401 + result.Duration = time.Since(start) 402 402 + 403 403 + gc.mu.Lock() 404 404 + gc.lastResult = result 405 405 + gc.lastResultAt = time.Now() 406 406 + gc.mu.Unlock() 407 407 + 408 408 + return result, nil 409 409 + } 410 410 + 299 411 // analyzeRecords performs Phase 1 analysis: builds referenced set, finds orphaned records, 300 412 // and identifies missing layer records. Pure analysis — no mutations. 301 413 // Discovers users, fetches manifests, scans records, identifies missing records.

+2 -2

pkg/hold/oci/xrpc.go

··· 380 380 } 381 381 } 382 382 383 383 - // Enqueue scan job if scanner is connected 384 384 - if h.scanBroadcaster != nil { 383 383 + // Enqueue scan job if scanner is connected (skip manifest lists — children get their own jobs) 384 384 + if h.scanBroadcaster != nil && !isMultiArch { 385 385 tier := "deckhand" 386 386 if stats != nil && stats.Tier != "" { 387 387 tier = stats.Tier

+146 -48

pkg/hold/pds/scan_broadcaster.go

··· 551 551 "total", msg.Summary.Total) 552 552 } 553 553 554 554 - // handleError marks a job as failed 554 554 + // handleError marks a job as failed and creates a scan record so the proactive 555 555 + // scanner treats it as "stale" rather than "never scanned" (avoids retry loops). 555 556 func (sb *ScanBroadcaster) handleError(sub *ScanSubscriber, msg ScannerMessage) { 556 556 - _, err := sb.db.Exec(` 557 557 + ctx := context.Background() 558 558 + 559 559 + // Get job details to create failure scan record 560 560 + var manifestDigest, repository, userDID string 561 561 + err := sb.db.QueryRow(` 562 562 + SELECT manifest_digest, repository, user_did 563 563 + FROM scan_jobs WHERE seq = ? 564 564 + `, msg.Seq).Scan(&manifestDigest, &repository, &userDID) 565 565 + if err != nil { 566 566 + slog.Error("Failed to get job details for failure record", 567 567 + "seq", msg.Seq, "error", err) 568 568 + } else { 569 569 + // Create a scan record with zero counts and nil blobs — marks it as 570 570 + // "scanned" so the proactive scheduler won't retry until rescan interval 571 571 + scanRecord := atproto.NewScanRecord( 572 572 + manifestDigest, repository, userDID, 573 573 + nil, nil, // no SBOM or vuln report 574 574 + 0, 0, 0, 0, 0, 575 575 + "failed: "+truncateError(msg.Error, 200), 576 576 + ) 577 577 + if _, _, err := sb.pds.CreateScanRecord(ctx, scanRecord); err != nil { 578 578 + slog.Error("Failed to store failure scan record", 579 579 + "seq", msg.Seq, "error", err) 580 580 + } 581 581 + } 582 582 + 583 583 + // Mark job as failed 584 584 + _, err = sb.db.Exec(` 557 585 UPDATE scan_jobs SET status = 'failed', completed_at = ? 558 586 WHERE seq = ? 559 587 `, time.Now(), msg.Seq) ··· 567 595 "seq", msg.Seq, 568 596 "subscriberId", sub.id, 569 597 "error", msg.Error) 598 598 + } 599 599 + 600 600 + func truncateError(s string, maxLen int) string { 601 601 + if len(s) <= maxLen { 602 602 + return s 603 603 + } 604 604 + return s[:maxLen] 570 605 } 571 606 572 607 // drainPendingJobs sends pending/timed-out jobs to a newly connected scanner. ··· 650 685 } 651 686 } 652 687 653 653 - // reDispatchTimedOut finds jobs that were assigned but not acked/completed within timeout. 688 688 + // reDispatchTimedOut finds jobs that were assigned but not acked/completed within timeout, 689 689 + // and also marks stuck processing jobs as failed. 654 690 // Collects timed-out rows first, closes cursor, then resets and re-dispatches 655 691 // to avoid holding a SELECT cursor open during UPDATEs (prevents SQLite BUSY). 656 692 func (sb *ScanBroadcaster) reDispatchTimedOut() { 657 693 timeout := time.Now().Add(-sb.ackTimeout) 658 694 695 695 + // Fail processing jobs stuck for >10 minutes (scanner likely crashed mid-scan) 696 696 + processingTimeout := time.Now().Add(-10 * time.Minute) 697 697 + res, err := sb.db.Exec(` 698 698 + UPDATE scan_jobs SET status = 'failed', completed_at = ? 699 699 + WHERE status = 'processing' AND assigned_at < ? 700 700 + `, time.Now(), processingTimeout) 701 701 + if err != nil { 702 702 + slog.Error("Failed to clean up stuck processing jobs", "error", err) 703 703 + } else if n, _ := res.RowsAffected(); n > 0 { 704 704 + slog.Warn("Cleaned up stuck processing jobs", "count", n) 705 705 + } 706 706 + 659 707 rows, err := sb.db.Query(` 660 708 SELECT seq, manifest_digest, repository, tag, user_did, user_handle, hold_did, hold_endpoint, tier, config_json, layers_json 661 709 FROM scan_jobs ··· 798 846 func (sb *ScanBroadcaster) proactiveScanLoop() { 799 847 defer sb.wg.Done() 800 848 801 801 - // Wait a bit before starting to let the system settle 849 849 + // Wait for the system to settle and DID list to populate 802 850 select { 803 851 case <-sb.stopCh: 804 852 return 805 805 - case <-time.After(30 * time.Second): 853 853 + case <-time.After(45 * time.Second): 806 854 } 807 855 856 856 + // Run immediately on startup, then every 60s 857 857 + slog.Info("Proactive scan loop started") 858 858 + sb.tryEnqueueProactiveScan() 859 859 + 808 860 ticker := time.NewTicker(60 * time.Second) 809 861 defer ticker.Stop() 810 862 ··· 824 876 // Uses the cached DID list from the relay (refreshed by refreshManifestDIDsLoop). 825 877 func (sb *ScanBroadcaster) tryEnqueueProactiveScan() { 826 878 if !sb.hasConnectedScanners() { 879 879 + slog.Debug("Proactive scan: no scanners connected, skipping") 827 880 return 828 881 } 829 882 if sb.hasActiveJobs() { 883 883 + slog.Debug("Proactive scan: active jobs in queue, skipping") 830 884 return 831 885 } 832 886 ··· 839 893 sb.manifestDIDsMu.RUnlock() 840 894 841 895 if len(userDIDs) == 0 { 896 896 + slog.Debug("Proactive scan: no manifest DIDs cached from relay, skipping") 842 897 return 843 898 } 844 899 ··· 854 909 } 855 910 } 856 911 912 912 + // scanCandidate is a manifest that needs scanning, with its scan freshness. 913 913 + type scanCandidate struct { 914 914 + manifest atproto.ManifestRecord 915 915 + userDID string 916 916 + userHandle string 917 917 + scannedAt time.Time // zero value = never scanned 918 918 + } 919 919 + 857 920 // tryEnqueueForUser fetches manifests from a user's PDS and enqueues a scan for the 858 858 - // first one that needs scanning. Returns true if a job was enqueued. 921 921 + // one that most needs it: never-scanned manifests first, then the stalest scan. 922 922 + // Returns true if a job was enqueued. 859 923 func (sb *ScanBroadcaster) tryEnqueueForUser(ctx context.Context, userDID string) bool { 860 924 // Resolve user DID to PDS endpoint and handle 861 925 did, userHandle, pdsEndpoint, err := atproto.ResolveIdentity(ctx, userDID) ··· 865 929 return false 866 930 } 867 931 868 868 - // Fetch manifest records from user's PDS 932 932 + // Collect all scannable manifests with their scan age 933 933 + var unscanned []scanCandidate 934 934 + var oldest *scanCandidate 935 935 + 869 936 client := atproto.NewClient(pdsEndpoint, did, "") 870 937 var cursor string 871 938 for { ··· 879 946 for _, record := range records { 880 947 var manifest atproto.ManifestRecord 881 948 if err := json.Unmarshal(record.Value, &manifest); err != nil { 882 882 - slog.Debug("Proactive scan: failed to unmarshal manifest record", 883 883 - "uri", record.URI, "error", err) 884 949 continue 885 950 } 886 951 ··· 898 963 continue 899 964 } 900 965 901 901 - // Skip if config is nil (shouldn't happen for image manifests, but be safe) 966 966 + // Skip if config is nil 902 967 if manifest.Config == nil { 903 968 continue 904 969 } 905 970 906 906 - // Check if already scanned recently 907 907 - if sb.isRecentlyScanned(ctx, manifest.Digest) { 971 971 + // Check scan status 972 972 + _, scanRecord, err := sb.pds.GetScanRecord(ctx, manifest.Digest) 973 973 + if err != nil { 974 974 + // No scan record — never scanned 975 975 + unscanned = append(unscanned, scanCandidate{ 976 976 + manifest: manifest, 977 977 + userDID: did, 978 978 + userHandle: userHandle, 979 979 + }) 908 980 continue 909 981 } 910 982 911 911 - // Construct and enqueue scan job 912 912 - configJSON, _ := json.Marshal(manifest.Config) 913 913 - layersJSON, _ := json.Marshal(manifest.Layers) 983 983 + scannedAt, err := time.Parse(time.RFC3339, scanRecord.ScannedAt) 984 984 + if err != nil { 985 985 + // Can't parse timestamp — treat as never scanned 986 986 + unscanned = append(unscanned, scanCandidate{ 987 987 + manifest: manifest, 988 988 + userDID: did, 989 989 + userHandle: userHandle, 990 990 + }) 991 991 + continue 992 992 + } 914 993 915 915 - slog.Info("Enqueuing proactive scan", 916 916 - "manifestDigest", manifest.Digest, 917 917 - "repository", manifest.Repository, 918 918 - "userDID", did) 994 994 + // Skip if scanned recently 995 995 + if time.Since(scannedAt) < sb.rescanInterval { 996 996 + continue 997 997 + } 919 998 920 920 - if err := sb.Enqueue(&ScanJobEvent{ 921 921 - ManifestDigest: manifest.Digest, 922 922 - Repository: manifest.Repository, 923 923 - UserDID: did, 924 924 - UserHandle: userHandle, 925 925 - Tier: "deckhand", 926 926 - Config: configJSON, 927 927 - Layers: layersJSON, 928 928 - }); err != nil { 929 929 - slog.Error("Proactive scan: failed to enqueue", 930 930 - "manifest", manifest.Digest, "error", err) 931 931 - return false 999 999 + // Stale scan — track the oldest 1000 1000 + if oldest == nil || scannedAt.Before(oldest.scannedAt) { 1001 1001 + oldest = &scanCandidate{ 1002 1002 + manifest: manifest, 1003 1003 + userDID: did, 1004 1004 + userHandle: userHandle, 1005 1005 + scannedAt: scannedAt, 1006 1006 + } 932 1007 } 933 933 - return true 934 1008 } 935 1009 936 1010 if nextCursor == "" || len(records) == 0 { ··· 939 1013 cursor = nextCursor 940 1014 } 941 1015 942 942 - return false 1016 1016 + // Prefer never-scanned, then oldest stale scan 1017 1017 + var pick *scanCandidate 1018 1018 + if len(unscanned) > 0 { 1019 1019 + pick = &unscanned[0] 1020 1020 + } else if oldest != nil { 1021 1021 + pick = oldest 1022 1022 + } 1023 1023 + 1024 1024 + if pick == nil { 1025 1025 + return false 1026 1026 + } 1027 1027 + 1028 1028 + configJSON, _ := json.Marshal(pick.manifest.Config) 1029 1029 + layersJSON, _ := json.Marshal(pick.manifest.Layers) 1030 1030 + 1031 1031 + reason := "never scanned" 1032 1032 + if !pick.scannedAt.IsZero() { 1033 1033 + reason = fmt.Sprintf("last scanned %s ago", time.Since(pick.scannedAt).Truncate(time.Minute)) 1034 1034 + } 1035 1035 + 1036 1036 + slog.Info("Enqueuing proactive scan", 1037 1037 + "manifestDigest", pick.manifest.Digest, 1038 1038 + "repository", pick.manifest.Repository, 1039 1039 + "userDID", pick.userDID, 1040 1040 + "reason", reason) 1041 1041 + 1042 1042 + if err := sb.Enqueue(&ScanJobEvent{ 1043 1043 + ManifestDigest: pick.manifest.Digest, 1044 1044 + Repository: pick.manifest.Repository, 1045 1045 + UserDID: pick.userDID, 1046 1046 + UserHandle: pick.userHandle, 1047 1047 + Tier: "deckhand", 1048 1048 + Config: configJSON, 1049 1049 + Layers: layersJSON, 1050 1050 + }); err != nil { 1051 1051 + slog.Error("Proactive scan: failed to enqueue", 1052 1052 + "manifest", pick.manifest.Digest, "error", err) 1053 1053 + return false 1054 1054 + } 1055 1055 + return true 943 1056 } 944 1057 945 1058 // isOurManifest checks if a manifest's holdDID matches this hold directly, ··· 1026 1139 } 1027 1140 1028 1141 return false 1029 1029 - } 1030 1030 - 1031 1031 - // isRecentlyScanned checks if a manifest has been scanned within the rescan interval. 1032 1032 - func (sb *ScanBroadcaster) isRecentlyScanned(ctx context.Context, manifestDigest string) bool { 1033 1033 - _, scanRecord, err := sb.pds.GetScanRecord(ctx, manifestDigest) 1034 1034 - if err != nil { 1035 1035 - return false // Not scanned or error reading → needs scanning 1036 1036 - } 1037 1037 - 1038 1038 - scannedAt, err := time.Parse(time.RFC3339, scanRecord.ScannedAt) 1039 1039 - if err != nil { 1040 1040 - return false // Can't parse timestamp → treat as needing scan 1041 1041 - } 1042 1042 - 1043 1043 - return time.Since(scannedAt) < sb.rescanInterval 1044 1142 } 1045 1143 1046 1144 // hasConnectedScanners returns true if at least one scanner is connected.

+2 -2

scanner/go.mod

··· 56 56 github.com/anchore/go-sync v0.0.0-20260122203928-582959aeb913 // indirect 57 57 github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 // indirect 58 58 github.com/anchore/packageurl-go v0.1.1-0.20250220190351-d62adb6e1115 // indirect 59 59 - github.com/anchore/stereoscope v0.1.20 // indirect 59 59 + github.com/anchore/stereoscope v0.1.20 60 60 github.com/andybalholm/brotli v1.2.0 // indirect 61 61 github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect 62 62 github.com/aquasecurity/go-pep440-version v0.0.1 // indirect ··· 181 181 github.com/json-iterator/go v1.1.12 // indirect 182 182 github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 // indirect 183 183 github.com/kevinburke/ssh_config v1.4.0 // indirect 184 184 - github.com/klauspost/compress v1.18.4 184 184 + github.com/klauspost/compress v1.18.4 // indirect 185 185 github.com/klauspost/cpuid/v2 v2.3.0 // indirect 186 186 github.com/klauspost/pgzip v1.2.6 // indirect 187 187 github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect

+4

scanner/internal/config/config.go

··· 54 54 55 55 // Directory for temporary layer extraction. 56 56 TmpDir string `yaml:"tmp_dir" comment:"Directory for temporary layer extraction."` 57 57 + 58 58 + // Maximum total compressed image size in bytes. Images exceeding this are skipped. 0 = no limit. 59 59 + MaxImageSize int64 `yaml:"max_image_size" comment:"Maximum total compressed image size in bytes. 0 = no limit. Default: 2 GiB."` 57 60 } 58 61 59 62 // setScannerDefaults registers all default values on the given Viper instance. ··· 76 79 v.SetDefault("vuln.enabled", true) 77 80 v.SetDefault("vuln.db_path", "/var/lib/atcr-scanner/vulndb") 78 81 v.SetDefault("vuln.tmp_dir", "/var/lib/atcr-scanner/tmp") 82 82 + v.SetDefault("vuln.max_image_size", 2*1024*1024*1024) // 2 GiB 79 83 80 84 // Log shipper defaults 81 85 v.SetDefault("log_shipper.batch_size", 100)

+128 -155

scanner/internal/scan/extractor.go

··· 1 1 package scan 2 2 3 3 import ( 4 4 - "archive/tar" 5 5 - "compress/gzip" 4 4 + "crypto/sha256" 6 5 "encoding/json" 7 6 "fmt" 8 8 - "io" 9 7 "log/slog" 10 8 "os" 11 9 "path/filepath" ··· 13 11 14 12 scanner "atcr.io/scanner" 15 13 "atcr.io/scanner/internal/client" 16 16 - "github.com/klauspost/compress/zstd" 17 14 ) 18 15 19 19 - // extractLayers downloads and extracts all image layers via presigned URLs 20 20 - // Returns the rootfs directory path and a cleanup function 21 21 - func extractLayers(job *scanner.ScanJob, tmpDir, secret string) (string, func(), error) { 16 16 + // OCI image layout types for constructing the layout on disk. 17 17 + type ociDescriptor struct { 18 18 + MediaType string `json:"mediaType"` 19 19 + Digest string `json:"digest"` 20 20 + Size int64 `json:"size"` 21 21 + } 22 22 + 23 23 + type ociManifest struct { 24 24 + SchemaVersion int `json:"schemaVersion"` 25 25 + MediaType string `json:"mediaType,omitempty"` 26 26 + Config ociDescriptor `json:"config"` 27 27 + Layers []ociDescriptor `json:"layers"` 28 28 + } 29 29 + 30 30 + type ociIndex struct { 31 31 + SchemaVersion int `json:"schemaVersion"` 32 32 + Manifests []ociDescriptor `json:"manifests"` 33 33 + } 34 34 + 35 35 + // buildOCILayout downloads image blobs and constructs an OCI image layout directory. 36 36 + // Instead of extracting layers to a rootfs (which requires decompression and causes 37 37 + // permission/security issues), this writes compressed blobs directly and lets Syft's 38 38 + // stereoscope handle layer processing internally. 39 39 + // 40 40 + // Layout structure: 41 41 + // 42 42 + // scan-*/ 43 43 + // ├── oci-layout 44 44 + // ├── index.json 45 45 + // └── blobs/sha256/ 46 46 + // ├── <manifest-hex> 47 47 + // ├── <config-hex> 48 48 + // └── <layer-hex>... 49 49 + func buildOCILayout(job *scanner.ScanJob, tmpDir, secret string) (string, func(), error) { 22 50 scanDir, err := os.MkdirTemp(tmpDir, "scan-*") 23 51 if err != nil { 24 52 return "", nil, fmt.Errorf("failed to create temp directory: %w", err) ··· 30 58 } 31 59 } 32 60 33 33 - imageDir := filepath.Join(scanDir, "image") 34 34 - rootfsDir := filepath.Join(imageDir, "rootfs") 35 35 - layersDir := filepath.Join(imageDir, "layers") 36 36 - 37 37 - for _, dir := range []string{rootfsDir, layersDir} { 38 38 - if err := os.MkdirAll(dir, 0755); err != nil { 39 39 - cleanup() 40 40 - return "", nil, fmt.Errorf("failed to create directory %s: %w", dir, err) 41 41 - } 61 61 + blobsDir := filepath.Join(scanDir, "blobs", "sha256") 62 62 + if err := os.MkdirAll(blobsDir, 0755); err != nil { 63 63 + cleanup() 64 64 + return "", nil, fmt.Errorf("failed to create blobs directory: %w", err) 42 65 } 43 66 44 44 - // Download and validate config blob 67 67 + // Download config blob 45 68 if job.Config.Digest == "" { 46 69 cleanup() 47 70 return "", nil, fmt.Errorf("config blob has empty digest, cannot download") 48 71 } 49 72 slog.Info("Downloading config blob", "digest", job.Config.Digest) 50 50 - configPath := filepath.Join(imageDir, "config.json") 51 51 - if err := downloadBlobViaPresignedURL(job.HoldEndpoint, job.HoldDID, job.Config.Digest, configPath, secret); err != nil { 73 73 + if err := downloadBlob(job, job.Config.Digest, blobsDir, secret); err != nil { 52 74 cleanup() 53 75 return "", nil, fmt.Errorf("failed to download config blob: %w", err) 54 76 } 55 77 56 56 - configData, err := os.ReadFile(configPath) 57 57 - if err != nil { 58 58 - cleanup() 59 59 - return "", nil, fmt.Errorf("failed to read config: %w", err) 60 60 - } 61 61 - var configObj map[string]interface{} 62 62 - if err := json.Unmarshal(configData, &configObj); err != nil { 63 63 - cleanup() 64 64 - return "", nil, fmt.Errorf("invalid config JSON: %w", err) 65 65 - } 66 66 - 67 67 - // Download and extract each layer 78 78 + // Download layer blobs (no extraction — kept compressed) 68 79 for i, layer := range job.Layers { 69 80 if layer.Digest == "" { 70 81 slog.Warn("Skipping layer with empty digest", "index", i) 71 82 continue 72 83 } 73 73 - // Skip non-tar layers (cosign signatures, attestations, etc.) 84 84 + // Skip non-tar layers (cosign signatures, in-toto attestations, etc.) 74 85 if layer.MediaType != "" && !strings.Contains(layer.MediaType, "tar") { 75 86 slog.Info("Skipping non-tar layer", "index", i, "digest", layer.Digest, "mediaType", layer.MediaType) 76 87 continue 77 88 } 78 78 - slog.Info("Extracting layer", "index", i, "digest", layer.Digest, "size", layer.Size, "mediaType", layer.MediaType) 79 79 - 80 80 - layerPath := filepath.Join(layersDir, fmt.Sprintf("layer-%d", i)) 81 81 - if err := downloadBlobViaPresignedURL(job.HoldEndpoint, job.HoldDID, layer.Digest, layerPath, secret); err != nil { 89 89 + slog.Info("Downloading layer", "index", i, "digest", layer.Digest, "size", layer.Size, "mediaType", layer.MediaType) 90 90 + if err := downloadBlob(job, layer.Digest, blobsDir, secret); err != nil { 82 91 cleanup() 83 92 return "", nil, fmt.Errorf("failed to download layer %d: %w", i, err) 84 93 } 94 94 + } 85 95 86 86 - if err := extractLayer(layerPath, rootfsDir, layer.MediaType); err != nil { 87 87 - cleanup() 88 88 - return "", nil, fmt.Errorf("failed to extract layer %d: %w", i, err) 96 96 + // Build OCI manifest from job descriptors 97 97 + manifest := ociManifest{ 98 98 + SchemaVersion: 2, 99 99 + MediaType: "application/vnd.oci.image.manifest.v1+json", 100 100 + Config: ociDescriptor{ 101 101 + MediaType: defaultMediaType(job.Config.MediaType, "application/vnd.oci.image.config.v1+json"), 102 102 + Digest: job.Config.Digest, 103 103 + Size: job.Config.Size, 104 104 + }, 105 105 + Layers: make([]ociDescriptor, 0, len(job.Layers)), 106 106 + } 107 107 + for _, layer := range job.Layers { 108 108 + if layer.Digest == "" { 109 109 + continue 89 110 } 111 111 + if layer.MediaType != "" && !strings.Contains(layer.MediaType, "tar") { 112 112 + continue 113 113 + } 114 114 + manifest.Layers = append(manifest.Layers, ociDescriptor{ 115 115 + MediaType: defaultMediaType(layer.MediaType, "application/vnd.oci.image.layer.v1.tar+gzip"), 116 116 + Digest: layer.Digest, 117 117 + Size: layer.Size, 118 118 + }) 119 119 + } 90 120 91 91 - // Remove layer file to save space 92 92 - os.Remove(layerPath) 121 121 + // Write manifest blob 122 122 + manifestJSON, err := json.Marshal(manifest) 123 123 + if err != nil { 124 124 + cleanup() 125 125 + return "", nil, fmt.Errorf("failed to marshal manifest: %w", err) 126 126 + } 127 127 + manifestHash := sha256.Sum256(manifestJSON) 128 128 + manifestDigest := fmt.Sprintf("sha256:%x", manifestHash) 129 129 + manifestPath := filepath.Join(blobsDir, fmt.Sprintf("%x", manifestHash)) 130 130 + if err := os.WriteFile(manifestPath, manifestJSON, 0644); err != nil { 131 131 + cleanup() 132 132 + return "", nil, fmt.Errorf("failed to write manifest blob: %w", err) 93 133 } 94 134 95 95 - entries, err := os.ReadDir(rootfsDir) 135 135 + // Write index.json 136 136 + index := ociIndex{ 137 137 + SchemaVersion: 2, 138 138 + Manifests: []ociDescriptor{ 139 139 + { 140 140 + MediaType: "application/vnd.oci.image.manifest.v1+json", 141 141 + Digest: manifestDigest, 142 142 + Size: int64(len(manifestJSON)), 143 143 + }, 144 144 + }, 145 145 + } 146 146 + indexJSON, err := json.Marshal(index) 96 147 if err != nil { 97 97 - slog.Warn("Failed to read rootfs directory", "error", err) 98 98 - } else { 99 99 - slog.Info("Successfully extracted image", 100 100 - "layers", len(job.Layers), 101 101 - "topLevelEntries", len(entries)) 148 148 + cleanup() 149 149 + return "", nil, fmt.Errorf("failed to marshal index: %w", err) 150 150 + } 151 151 + if err := os.WriteFile(filepath.Join(scanDir, "index.json"), indexJSON, 0644); err != nil { 152 152 + cleanup() 153 153 + return "", nil, fmt.Errorf("failed to write index.json: %w", err) 154 154 + } 155 155 + 156 156 + // Write oci-layout file 157 157 + ociLayout := []byte(`{"imageLayoutVersion":"1.0.0"}`) 158 158 + if err := os.WriteFile(filepath.Join(scanDir, "oci-layout"), ociLayout, 0644); err != nil { 159 159 + cleanup() 160 160 + return "", nil, fmt.Errorf("failed to write oci-layout: %w", err) 102 161 } 103 162 104 104 - return rootfsDir, cleanup, nil 163 163 + slog.Info("OCI layout built", 164 164 + "dir", scanDir, 165 165 + "layers", len(manifest.Layers), 166 166 + "manifestDigest", manifestDigest) 167 167 + 168 168 + return scanDir, cleanup, nil 105 169 } 106 170 107 107 - // downloadBlobViaPresignedURL gets a presigned URL from the hold and downloads the blob 108 108 - func downloadBlobViaPresignedURL(holdEndpoint, holdDID, digest, destPath, secret string) error { 109 109 - presignedURL, err := client.GetBlobPresignedURL(holdEndpoint, holdDID, digest, secret) 171 171 + // downloadBlob downloads a blob by digest to the blobs directory using presigned URLs. 172 172 + func downloadBlob(job *scanner.ScanJob, digest, blobsDir, secret string) error { 173 173 + hex := digestHex(digest) 174 174 + destPath := filepath.Join(blobsDir, hex) 175 175 + 176 176 + presignedURL, err := client.GetBlobPresignedURL(job.HoldEndpoint, job.HoldDID, digest, secret) 110 177 if err != nil { 111 178 return fmt.Errorf("failed to get presigned URL for %s: %w", digest, err) 112 179 } 113 180 return client.DownloadBlob(presignedURL, destPath) 114 181 } 115 182 116 116 - // extractLayer extracts a layer tar archive to a destination directory (overlayfs style). 117 117 - // Supports gzip, zstd, and uncompressed tar based on the OCI media type. 118 118 - // Falls back to header sniffing if the media type is unrecognized. 119 119 - func extractLayer(layerPath, destDir, mediaType string) error { 120 120 - file, err := os.Open(layerPath) 121 121 - if err != nil { 122 122 - return fmt.Errorf("failed to open layer: %w", err) 183 183 + // digestHex extracts the hex portion from a digest string (e.g., "sha256:abc123" → "abc123"). 184 184 + func digestHex(digest string) string { 185 185 + if _, hex, ok := strings.Cut(digest, ":"); ok { 186 186 + return hex 123 187 } 124 124 - defer file.Close() 188 188 + return digest 189 189 + } 125 190 126 126 - var tarReader io.Reader 127 127 - 128 128 - switch { 129 129 - case strings.Contains(mediaType, "zstd"): 130 130 - decoder, err := zstd.NewReader(file) 131 131 - if err != nil { 132 132 - return fmt.Errorf("failed to create zstd reader: %w", err) 133 133 - } 134 134 - defer decoder.Close() 135 135 - tarReader = decoder 136 136 - 137 137 - case strings.Contains(mediaType, "gzip") || mediaType == "": 138 138 - // Default to gzip for unspecified media types (most common) 139 139 - gzr, err := gzip.NewReader(file) 140 140 - if err != nil { 141 141 - // If gzip fails, try plain tar (header sniff fallback) 142 142 - if _, seekErr := file.Seek(0, io.SeekStart); seekErr != nil { 143 143 - return fmt.Errorf("failed to create gzip reader: %w", err) 144 144 - } 145 145 - slog.Debug("Gzip header invalid, falling back to plain tar", "mediaType", mediaType) 146 146 - tarReader = file 147 147 - } else { 148 148 - defer gzr.Close() 149 149 - tarReader = gzr 150 150 - } 151 151 - 152 152 - default: 153 153 - // Uncompressed tar or unknown — try plain tar 154 154 - tarReader = file 191 191 + func defaultMediaType(mediaType, fallback string) string { 192 192 + if mediaType == "" { 193 193 + return fallback 155 194 } 156 156 - 157 157 - tr := tar.NewReader(tarReader) 158 158 - 159 159 - for { 160 160 - header, err := tr.Next() 161 161 - if err == io.EOF { 162 162 - break 163 163 - } 164 164 - if err != nil { 165 165 - return fmt.Errorf("failed to read tar header: %w", err) 166 166 - } 167 167 - 168 168 - target := filepath.Join(destDir, filepath.Clean(header.Name)) 169 169 - 170 170 - // Security: ensure target is within destDir 171 171 - if !strings.HasPrefix(target, filepath.Clean(destDir)+string(os.PathSeparator)) { 172 172 - slog.Warn("Skipping path outside destination", "path", header.Name) 173 173 - continue 174 174 - } 175 175 - 176 176 - switch header.Typeflag { 177 177 - case tar.TypeDir: 178 178 - // Always set owner write bit so we can create files inside (e.g. Go module 179 179 - // cache dirs are 0555 in images, which would block subsequent writes) 180 180 - if err := os.MkdirAll(target, os.FileMode(header.Mode)|0200); err != nil { 181 181 - return fmt.Errorf("failed to create directory %s: %w", target, err) 182 182 - } 183 183 - 184 184 - case tar.TypeReg: 185 185 - if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { 186 186 - return fmt.Errorf("failed to create parent directory: %w", err) 187 187 - } 188 188 - outFile, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(header.Mode)) 189 189 - if err != nil { 190 190 - return fmt.Errorf("failed to create file %s: %w", target, err) 191 191 - } 192 192 - if _, err := io.Copy(outFile, tr); err != nil { 193 193 - outFile.Close() 194 194 - return fmt.Errorf("failed to write file %s: %w", target, err) 195 195 - } 196 196 - outFile.Close() 197 197 - 198 198 - case tar.TypeSymlink: 199 199 - if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { 200 200 - return fmt.Errorf("failed to create parent directory for symlink: %w", err) 201 201 - } 202 202 - os.Remove(target) 203 203 - if err := os.Symlink(header.Linkname, target); err != nil { 204 204 - slog.Warn("Failed to create symlink", "target", target, "link", header.Linkname, "error", err) 205 205 - } 206 206 - 207 207 - case tar.TypeLink: 208 208 - linkTarget := filepath.Join(destDir, filepath.Clean(header.Linkname)) 209 209 - if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { 210 210 - return fmt.Errorf("failed to create parent directory for hardlink: %w", err) 211 211 - } 212 212 - os.Remove(target) 213 213 - if err := os.Link(linkTarget, target); err != nil { 214 214 - slog.Warn("Failed to create hardlink", "target", target, "link", linkTarget, "error", err) 215 215 - } 216 216 - 217 217 - default: 218 218 - slog.Debug("Skipping unsupported tar entry type", "type", header.Typeflag, "name", header.Name) 219 219 - } 220 220 - } 221 221 - 222 222 - return nil 195 195 + return mediaType 223 196 }

+22 -12

scanner/internal/scan/syft.go

··· 5 5 "crypto/sha256" 6 6 "fmt" 7 7 "log/slog" 8 8 - "os" 9 8 9 9 + "github.com/anchore/stereoscope/pkg/file" 10 10 + "github.com/anchore/stereoscope/pkg/image/oci" 10 11 "github.com/anchore/syft/syft" 11 12 "github.com/anchore/syft/syft/format" 12 13 "github.com/anchore/syft/syft/format/spdxjson" 13 14 "github.com/anchore/syft/syft/sbom" 14 14 - "github.com/anchore/syft/syft/source/directorysource" 15 15 + "github.com/anchore/syft/syft/source/stereoscopesource" 15 16 ) 16 17 17 17 - // generateSBOM generates an SBOM using Syft from an extracted image directory 18 18 - // Returns the SBOM object, SBOM JSON bytes, and its digest 19 19 - func generateSBOM(ctx context.Context, imageDir string) (*sbom.SBOM, []byte, string, error) { 20 20 - slog.Info("Generating SBOM with Syft", "imageDir", imageDir) 18 18 + // generateSBOM generates an SBOM using Syft from an OCI image layout directory. 19 19 + // Returns the SBOM object, SBOM JSON bytes, and its digest. 20 20 + func generateSBOM(ctx context.Context, ociLayoutDir string) (*sbom.SBOM, []byte, string, error) { 21 21 + slog.Info("Generating SBOM with Syft", "ociLayout", ociLayoutDir) 21 22 22 22 - entries, err := os.ReadDir(imageDir) 23 23 + // Create stereoscope OCI directory provider 24 24 + tmpGen := file.NewTempDirGenerator("syft-scan") 25 25 + defer tmpGen.Cleanup() 26 26 + 27 27 + provider := oci.NewDirectoryProvider(tmpGen, ociLayoutDir) 28 28 + img, err := provider.Provide(ctx) 23 29 if err != nil { 24 24 - return nil, nil, "", fmt.Errorf("failed to read image directory: %w", err) 30 30 + return nil, nil, "", fmt.Errorf("failed to load OCI image: %w", err) 25 31 } 26 26 - slog.Info("Image directory contents", "path", imageDir, "entries", len(entries)) 32 32 + defer img.Cleanup() 27 33 28 28 - src, err := directorysource.NewFromPath(imageDir) 29 29 - if err != nil { 30 30 - return nil, nil, "", fmt.Errorf("failed to create Syft source: %w", err) 34 34 + if err := img.Read(); err != nil { 35 35 + return nil, nil, "", fmt.Errorf("failed to read OCI image: %w", err) 31 36 } 37 37 + 38 38 + // Wrap in Syft source 39 39 + src := stereoscopesource.New(img, stereoscopesource.ImageConfig{ 40 40 + Reference: ociLayoutDir, 41 41 + }) 32 42 defer src.Close() 33 43 34 44 slog.Info("Running Syft cataloging")

+22 -6

scanner/internal/scan/worker.go

··· 45 45 }() 46 46 } 47 47 48 48 + // Point TMPDIR at the configured tmp dir so stereoscope's internal 49 49 + // layer extraction uses the same partition (not /tmp which may be small) 50 50 + os.Setenv("TMPDIR", wp.cfg.Vuln.TmpDir) 51 51 + 48 52 for i := 0; i < wp.cfg.Scanner.Workers; i++ { 49 53 wp.wg.Add(1) 50 54 go wp.worker(ctx, i) ··· 104 108 return nil, fmt.Errorf("failed to create tmp dir: %w", err) 105 109 } 106 110 107 107 - // Step 1: Extract image layers from hold via presigned URLs 108 108 - slog.Info("Extracting image layers", "repository", job.Repository) 109 109 - imageDir, cleanup, err := extractLayers(job, wp.cfg.Vuln.TmpDir, wp.cfg.Hold.Secret) 111 111 + // Check total compressed image size before downloading 112 112 + if wp.cfg.Vuln.MaxImageSize > 0 { 113 113 + var totalSize int64 114 114 + for _, layer := range job.Layers { 115 115 + totalSize += layer.Size 116 116 + } 117 117 + totalSize += job.Config.Size 118 118 + if totalSize > wp.cfg.Vuln.MaxImageSize { 119 119 + return nil, fmt.Errorf("image too large: %d bytes compressed (limit %d bytes)", totalSize, wp.cfg.Vuln.MaxImageSize) 120 120 + } 121 121 + } 122 122 + 123 123 + // Step 1: Build OCI image layout from hold via presigned URLs 124 124 + slog.Info("Building OCI layout", "repository", job.Repository) 125 125 + ociLayoutDir, cleanup, err := buildOCILayout(job, wp.cfg.Vuln.TmpDir, wp.cfg.Hold.Secret) 110 126 if err != nil { 111 111 - return nil, fmt.Errorf("failed to extract layers: %w", err) 127 127 + return nil, fmt.Errorf("failed to build OCI layout: %w", err) 112 128 } 113 129 defer cleanup() 114 130 115 131 // Step 2: Generate SBOM with Syft 116 132 slog.Info("Generating SBOM", "repository", job.Repository) 117 117 - sbomResult, sbomJSON, sbomDigest, err := generateSBOM(ctx, imageDir) 133 133 + sbomResult, sbomJSON, sbomDigest, err := generateSBOM(ctx, ociLayoutDir) 118 134 if err != nil { 119 135 return nil, fmt.Errorf("failed to generate SBOM: %w", err) 120 136 } ··· 127 143 128 144 // Step 3: Scan SBOM with Grype (if enabled) 129 145 if wp.cfg.Vuln.Enabled { 130 130 - slog.Info("Scanning for vulnerabilities", "repository", job.Repository) 146 146 + slog.Info("Scanning for vulnerabilities", "repository", job.Repository, "handle", job.UserHandle) 131 147 vulnJSON, vulnDigest, summary, err := scanVulnerabilities(ctx, sbomResult, wp.cfg.Vuln.DBPath) 132 148 if err != nil { 133 149 return nil, fmt.Errorf("failed to scan vulnerabilities: %w", err)