+703

docs/ANNOTATIONS_REFACTOR.md

··· 1 1 + # Annotations Table Refactoring 2 2 + 3 3 + ## Overview 4 4 + 5 5 + Refactor manifest annotations from individual columns (`title`, `description`, `source_url`, etc.) to a normalized key-value table. This enables flexible annotation storage without schema changes for new OCI annotations. 6 6 + 7 7 + ## Motivation 8 8 + 9 9 + **Current Problems:** 10 10 + - Each new annotation (e.g., `org.opencontainers.image.version`) requires schema change 11 11 + - Many NULL columns in manifests table 12 12 + - Rigid schema doesn't match OCI's flexible annotation model 13 13 + 14 14 + **Benefits:** 15 15 + - ✅ Add any annotation without code/schema changes 16 16 + - ✅ Normalized database design 17 17 + - ✅ Easy to query "all repos with annotation X" 18 18 + - ✅ Simple queries (no joins needed for repository pages) 19 19 + 20 20 + ## Database Schema Changes 21 21 + 22 22 + ### 1. New Table: `repository_annotations` 23 23 + 24 24 + ```sql 25 25 + CREATE TABLE IF NOT EXISTS repository_annotations ( 26 26 + did TEXT NOT NULL, 27 27 + repository TEXT NOT NULL, 28 28 + key TEXT NOT NULL, 29 29 + value TEXT NOT NULL, 30 30 + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 31 31 + PRIMARY KEY(did, repository, key), 32 32 + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE 33 33 + ); 34 34 + CREATE INDEX IF NOT EXISTS idx_repository_annotations_did_repo ON repository_annotations(did, repository); 35 35 + CREATE INDEX IF NOT EXISTS idx_repository_annotations_key ON repository_annotations(key); 36 36 + ``` 37 37 + 38 38 + **Key Design Decisions:** 39 39 + - Primary key: `(did, repository, key)` - one value per annotation per repository 40 40 + - No `manifest_id` foreign key - annotations are repository-level, not manifest-level 41 41 + - `updated_at` - track when annotation was last updated (from most recent manifest) 42 42 + - Stored at repository level because that's where they're displayed 43 43 + 44 44 + ### 2. Drop Columns from `manifests` Table 45 45 + 46 46 + Remove these columns (migration will preserve data by copying to annotations table): 47 47 + - `title` 48 48 + - `description` 49 49 + - `source_url` 50 50 + - `documentation_url` 51 51 + - `licenses` 52 52 + - `icon_url` 53 53 + - `readme_url` 54 54 + - `version` 55 55 + 56 56 + Keep only core manifest metadata: 57 57 + - `id`, `did`, `repository`, `digest` 58 58 + - `hold_endpoint`, `schema_version`, `media_type` 59 59 + - `config_digest`, `config_size` 60 60 + - `created_at` 61 61 + 62 62 + ## Migration Strategy 63 63 + 64 64 + ### Migration File: `0004_refactor_annotations_table.yaml` 65 65 + 66 66 + ```yaml 67 67 + description: Migrate manifest annotations to separate table 68 68 + query: | 69 69 + -- Step 1: Create new annotations table 70 70 + CREATE TABLE IF NOT EXISTS repository_annotations ( 71 71 + did TEXT NOT NULL, 72 72 + repository TEXT NOT NULL, 73 73 + key TEXT NOT NULL, 74 74 + value TEXT NOT NULL, 75 75 + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 76 76 + PRIMARY KEY(did, repository, key), 77 77 + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE 78 78 + ); 79 79 + CREATE INDEX IF NOT EXISTS idx_repository_annotations_did_repo ON repository_annotations(did, repository); 80 80 + CREATE INDEX IF NOT EXISTS idx_repository_annotations_key ON repository_annotations(key); 81 81 + 82 82 + -- Step 2: Migrate existing data from manifests to annotations 83 83 + -- For each repository, use the most recent manifest with non-empty data 84 84 + INSERT OR REPLACE INTO repository_annotations (did, repository, key, value, updated_at) 85 85 + SELECT 86 86 + m.did, 87 87 + m.repository, 88 88 + 'org.opencontainers.image.title' as key, 89 89 + m.title as value, 90 90 + m.created_at as updated_at 91 91 + FROM manifests m 92 92 + WHERE m.title IS NOT NULL AND m.title != '' 93 93 + AND m.created_at = ( 94 94 + SELECT MAX(created_at) FROM manifests m2 95 95 + WHERE m2.did = m.did AND m2.repository = m.repository 96 96 + AND m2.title IS NOT NULL AND m2.title != '' 97 97 + ); 98 98 + 99 99 + INSERT OR REPLACE INTO repository_annotations (did, repository, key, value, updated_at) 100 100 + SELECT m.did, m.repository, 'org.opencontainers.image.description', m.description, m.created_at 101 101 + FROM manifests m 102 102 + WHERE m.description IS NOT NULL AND m.description != '' 103 103 + AND m.created_at = ( 104 104 + SELECT MAX(created_at) FROM manifests m2 105 105 + WHERE m2.did = m.did AND m2.repository = m.repository 106 106 + AND m2.description IS NOT NULL AND m2.description != '' 107 107 + ); 108 108 + 109 109 + INSERT OR REPLACE INTO repository_annotations (did, repository, key, value, updated_at) 110 110 + SELECT m.did, m.repository, 'org.opencontainers.image.source', m.source_url, m.created_at 111 111 + FROM manifests m 112 112 + WHERE m.source_url IS NOT NULL AND m.source_url != '' 113 113 + AND m.created_at = ( 114 114 + SELECT MAX(created_at) FROM manifests m2 115 115 + WHERE m2.did = m.did AND m2.repository = m.repository 116 116 + AND m2.source_url IS NOT NULL AND m2.source_url != '' 117 117 + ); 118 118 + 119 119 + INSERT OR REPLACE INTO repository_annotations (did, repository, key, value, updated_at) 120 120 + SELECT m.did, m.repository, 'org.opencontainers.image.documentation', m.documentation_url, m.created_at 121 121 + FROM manifests m 122 122 + WHERE m.documentation_url IS NOT NULL AND m.documentation_url != '' 123 123 + AND m.created_at = ( 124 124 + SELECT MAX(created_at) FROM manifests m2 125 125 + WHERE m2.did = m.did AND m2.repository = m.repository 126 126 + AND m2.documentation_url IS NOT NULL AND m2.documentation_url != '' 127 127 + ); 128 128 + 129 129 + INSERT OR REPLACE INTO repository_annotations (did, repository, key, value, updated_at) 130 130 + SELECT m.did, m.repository, 'org.opencontainers.image.licenses', m.licenses, m.created_at 131 131 + FROM manifests m 132 132 + WHERE m.licenses IS NOT NULL AND m.licenses != '' 133 133 + AND m.created_at = ( 134 134 + SELECT MAX(created_at) FROM manifests m2 135 135 + WHERE m2.did = m.did AND m2.repository = m.repository 136 136 + AND m2.licenses IS NOT NULL AND m2.licenses != '' 137 137 + ); 138 138 + 139 139 + INSERT OR REPLACE INTO repository_annotations (did, repository, key, value, updated_at) 140 140 + SELECT m.did, m.repository, 'io.atcr.icon', m.icon_url, m.created_at 141 141 + FROM manifests m 142 142 + WHERE m.icon_url IS NOT NULL AND m.icon_url != '' 143 143 + AND m.created_at = ( 144 144 + SELECT MAX(created_at) FROM manifests m2 145 145 + WHERE m2.did = m.did AND m2.repository = m.repository 146 146 + AND m2.icon_url IS NOT NULL AND m2.icon_url != '' 147 147 + ); 148 148 + 149 149 + INSERT OR REPLACE INTO repository_annotations (did, repository, key, value, updated_at) 150 150 + SELECT m.did, m.repository, 'io.atcr.readme', m.readme_url, m.created_at 151 151 + FROM manifests m 152 152 + WHERE m.readme_url IS NOT NULL AND m.readme_url != '' 153 153 + AND m.created_at = ( 154 154 + SELECT MAX(created_at) FROM manifests m2 155 155 + WHERE m2.did = m.did AND m2.repository = m.repository 156 156 + AND m2.readme_url IS NOT NULL AND m2.readme_url != '' 157 157 + ); 158 158 + 159 159 + -- Step 3: Drop old columns from manifests table 160 160 + -- SQLite requires recreating table to drop columns 161 161 + CREATE TABLE manifests_new ( 162 162 + id INTEGER PRIMARY KEY AUTOINCREMENT, 163 163 + did TEXT NOT NULL, 164 164 + repository TEXT NOT NULL, 165 165 + digest TEXT NOT NULL, 166 166 + hold_endpoint TEXT NOT NULL, 167 167 + schema_version INTEGER NOT NULL, 168 168 + media_type TEXT NOT NULL, 169 169 + config_digest TEXT, 170 170 + config_size INTEGER, 171 171 + created_at TIMESTAMP NOT NULL, 172 172 + UNIQUE(did, repository, digest), 173 173 + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE 174 174 + ); 175 175 + 176 176 + -- Copy data to new table 177 177 + INSERT INTO manifests_new 178 178 + SELECT id, did, repository, digest, hold_endpoint, schema_version, media_type, 179 179 + config_digest, config_size, created_at 180 180 + FROM manifests; 181 181 + 182 182 + -- Replace old table 183 183 + DROP TABLE manifests; 184 184 + ALTER TABLE manifests_new RENAME TO manifests; 185 185 + 186 186 + -- Recreate indexes 187 187 + CREATE INDEX IF NOT EXISTS idx_manifests_did_repo ON manifests(did, repository); 188 188 + CREATE INDEX IF NOT EXISTS idx_manifests_created_at ON manifests(created_at DESC); 189 189 + CREATE INDEX IF NOT EXISTS idx_manifests_digest ON manifests(digest); 190 190 + ``` 191 191 + 192 192 + ## Code Changes 193 193 + 194 194 + ### 1. Database Helper Functions 195 195 + 196 196 + **New file: `pkg/appview/db/annotations.go`** 197 197 + 198 198 + ```go 199 199 + package db 200 200 + 201 201 + import ( 202 202 + "database/sql" 203 203 + "time" 204 204 + ) 205 205 + 206 206 + // GetRepositoryAnnotations retrieves all annotations for a repository 207 207 + func GetRepositoryAnnotations(db *sql.DB, did, repository string) (map[string]string, error) { 208 208 + rows, err := db.Query(` 209 209 + SELECT key, value 210 210 + FROM repository_annotations 211 211 + WHERE did = ? AND repository = ? 212 212 + `, did, repository) 213 213 + if err != nil { 214 214 + return nil, err 215 215 + } 216 216 + defer rows.Close() 217 217 + 218 218 + annotations := make(map[string]string) 219 219 + for rows.Next() { 220 220 + var key, value string 221 221 + if err := rows.Scan(&key, &value); err != nil { 222 222 + return nil, err 223 223 + } 224 224 + annotations[key] = value 225 225 + } 226 226 + 227 227 + return annotations, rows.Err() 228 228 + } 229 229 + 230 230 + // UpsertRepositoryAnnotations replaces all annotations for a repository 231 231 + // Only called when manifest has at least one non-empty annotation 232 232 + func UpsertRepositoryAnnotations(db *sql.DB, did, repository string, annotations map[string]string) error { 233 233 + tx, err := db.Begin() 234 234 + if err != nil { 235 235 + return err 236 236 + } 237 237 + defer tx.Rollback() 238 238 + 239 239 + // Delete existing annotations 240 240 + _, err = tx.Exec(` 241 241 + DELETE FROM repository_annotations 242 242 + WHERE did = ? AND repository = ? 243 243 + `, did, repository) 244 244 + if err != nil { 245 245 + return err 246 246 + } 247 247 + 248 248 + // Insert new annotations 249 249 + stmt, err := tx.Prepare(` 250 250 + INSERT INTO repository_annotations (did, repository, key, value, updated_at) 251 251 + VALUES (?, ?, ?, ?, ?) 252 252 + `) 253 253 + if err != nil { 254 254 + return err 255 255 + } 256 256 + defer stmt.Close() 257 257 + 258 258 + now := time.Now() 259 259 + for key, value := range annotations { 260 260 + _, err = stmt.Exec(did, repository, key, value, now) 261 261 + if err != nil { 262 262 + return err 263 263 + } 264 264 + } 265 265 + 266 266 + return tx.Commit() 267 267 + } 268 268 + 269 269 + // DeleteRepositoryAnnotations removes all annotations for a repository 270 270 + func DeleteRepositoryAnnotations(db *sql.DB, did, repository string) error { 271 271 + _, err := db.Exec(` 272 272 + DELETE FROM repository_annotations 273 273 + WHERE did = ? AND repository = ? 274 274 + `, did, repository) 275 275 + return err 276 276 + } 277 277 + ``` 278 278 + 279 279 + ### 2. Update Backfill Worker 280 280 + 281 281 + **File: `pkg/appview/jetstream/backfill.go`** 282 282 + 283 283 + In `processManifestRecord()` function, after extracting annotations: 284 284 + 285 285 + ```go 286 286 + // Extract OCI annotations from manifest 287 287 + var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string 288 288 + if manifestRecord.Annotations != nil { 289 289 + title = manifestRecord.Annotations["org.opencontainers.image.title"] 290 290 + description = manifestRecord.Annotations["org.opencontainers.image.description"] 291 291 + sourceURL = manifestRecord.Annotations["org.opencontainers.image.source"] 292 292 + documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"] 293 293 + licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"] 294 294 + iconURL = manifestRecord.Annotations["io.atcr.icon"] 295 295 + readmeURL = manifestRecord.Annotations["io.atcr.readme"] 296 296 + } 297 297 + 298 298 + // Prepare manifest for insertion (WITHOUT annotation fields) 299 299 + manifest := &db.Manifest{ 300 300 + DID: did, 301 301 + Repository: manifestRecord.Repository, 302 302 + Digest: manifestRecord.Digest, 303 303 + MediaType: manifestRecord.MediaType, 304 304 + SchemaVersion: manifestRecord.SchemaVersion, 305 305 + HoldEndpoint: manifestRecord.HoldEndpoint, 306 306 + CreatedAt: manifestRecord.CreatedAt, 307 307 + // NO annotation fields 308 308 + } 309 309 + 310 310 + // Set config fields only for image manifests (not manifest lists) 311 311 + if !isManifestList && manifestRecord.Config != nil { 312 312 + manifest.ConfigDigest = manifestRecord.Config.Digest 313 313 + manifest.ConfigSize = manifestRecord.Config.Size 314 314 + } 315 315 + 316 316 + // Insert manifest 317 317 + manifestID, err := db.InsertManifest(b.db, manifest) 318 318 + if err != nil { 319 319 + return fmt.Errorf("failed to insert manifest: %w", err) 320 320 + } 321 321 + 322 322 + // Update repository annotations ONLY if manifest has at least one non-empty annotation 323 323 + if manifestRecord.Annotations != nil { 324 324 + hasData := false 325 325 + for _, value := range manifestRecord.Annotations { 326 326 + if value != "" { 327 327 + hasData = true 328 328 + break 329 329 + } 330 330 + } 331 331 + 332 332 + if hasData { 333 333 + // Replace all annotations for this repository 334 334 + err = db.UpsertRepositoryAnnotations(b.db, did, manifestRecord.Repository, manifestRecord.Annotations) 335 335 + if err != nil { 336 336 + return fmt.Errorf("failed to upsert annotations: %w", err) 337 337 + } 338 338 + } 339 339 + } 340 340 + ``` 341 341 + 342 342 + ### 3. Update Jetstream Worker 343 343 + 344 344 + **File: `pkg/appview/jetstream/worker.go`** 345 345 + 346 346 + Same changes as backfill - in `processManifestCommit()` function: 347 347 + 348 348 + ```go 349 349 + // Extract OCI annotations from manifest 350 350 + var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string 351 351 + if manifestRecord.Annotations != nil { 352 352 + title = manifestRecord.Annotations["org.opencontainers.image.title"] 353 353 + description = manifestRecord.Annotations["org.opencontainers.image.description"] 354 354 + sourceURL = manifestRecord.Annotations["org.opencontainers.image.source"] 355 355 + documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"] 356 356 + licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"] 357 357 + iconURL = manifestRecord.Annotations["io.atcr.icon"] 358 358 + readmeURL = manifestRecord.Annotations["io.atcr.readme"] 359 359 + } 360 360 + 361 361 + // Prepare manifest for insertion (WITHOUT annotation fields) 362 362 + manifest := &db.Manifest{ 363 363 + DID: commit.DID, 364 364 + Repository: manifestRecord.Repository, 365 365 + Digest: manifestRecord.Digest, 366 366 + MediaType: manifestRecord.MediaType, 367 367 + SchemaVersion: manifestRecord.SchemaVersion, 368 368 + HoldEndpoint: manifestRecord.HoldEndpoint, 369 369 + CreatedAt: manifestRecord.CreatedAt, 370 370 + // NO annotation fields 371 371 + } 372 372 + 373 373 + // Set config fields only for image manifests (not manifest lists) 374 374 + if !isManifestList && manifestRecord.Config != nil { 375 375 + manifest.ConfigDigest = manifestRecord.Config.Digest 376 376 + manifest.ConfigSize = manifestRecord.Config.Size 377 377 + } 378 378 + 379 379 + // Insert manifest 380 380 + manifestID, err := db.InsertManifest(w.db, manifest) 381 381 + if err != nil { 382 382 + return fmt.Errorf("failed to insert manifest: %w", err) 383 383 + } 384 384 + 385 385 + // Update repository annotations ONLY if manifest has at least one non-empty annotation 386 386 + if manifestRecord.Annotations != nil { 387 387 + hasData := false 388 388 + for _, value := range manifestRecord.Annotations { 389 389 + if value != "" { 390 390 + hasData = true 391 391 + break 392 392 + } 393 393 + } 394 394 + 395 395 + if hasData { 396 396 + // Replace all annotations for this repository 397 397 + err = db.UpsertRepositoryAnnotations(w.db, commit.DID, manifestRecord.Repository, manifestRecord.Annotations) 398 398 + if err != nil { 399 399 + return fmt.Errorf("failed to upsert annotations: %w", err) 400 400 + } 401 401 + } 402 402 + } 403 403 + ``` 404 404 + 405 405 + ### 4. Update Database Queries 406 406 + 407 407 + **File: `pkg/appview/db/queries.go`** 408 408 + 409 409 + Replace `GetRepositoryMetadata()` function: 410 410 + 411 411 + ```go 412 412 + // GetRepositoryMetadata retrieves metadata for a repository from annotations table 413 413 + func GetRepositoryMetadata(db *sql.DB, did string, repository string) (title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, version string, err error) { 414 414 + annotations, err := GetRepositoryAnnotations(db, did, repository) 415 415 + if err != nil { 416 416 + return "", "", "", "", "", "", "", "", err 417 417 + } 418 418 + 419 419 + title = annotations["org.opencontainers.image.title"] 420 420 + description = annotations["org.opencontainers.image.description"] 421 421 + sourceURL = annotations["org.opencontainers.image.source"] 422 422 + documentationURL = annotations["org.opencontainers.image.documentation"] 423 423 + licenses = annotations["org.opencontainers.image.licenses"] 424 424 + iconURL = annotations["io.atcr.icon"] 425 425 + readmeURL = annotations["io.atcr.readme"] 426 426 + version = annotations["org.opencontainers.image.version"] 427 427 + 428 428 + return title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, version, nil 429 429 + } 430 430 + ``` 431 431 + 432 432 + Update `InsertManifest()` to remove annotation columns: 433 433 + 434 434 + ```go 435 435 + func InsertManifest(db *sql.DB, manifest *Manifest) (int64, error) { 436 436 + _, err := db.Exec(` 437 437 + INSERT INTO manifests 438 438 + (did, repository, digest, hold_endpoint, schema_version, media_type, 439 439 + config_digest, config_size, created_at) 440 440 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) 441 441 + ON CONFLICT(did, repository, digest) DO UPDATE SET 442 442 + hold_endpoint = excluded.hold_endpoint, 443 443 + schema_version = excluded.schema_version, 444 444 + media_type = excluded.media_type, 445 445 + config_digest = excluded.config_digest, 446 446 + config_size = excluded.config_size 447 447 + `, manifest.DID, manifest.Repository, manifest.Digest, manifest.HoldEndpoint, 448 448 + manifest.SchemaVersion, manifest.MediaType, manifest.ConfigDigest, 449 449 + manifest.ConfigSize, manifest.CreatedAt) 450 450 + 451 451 + if err != nil { 452 452 + return 0, err 453 453 + } 454 454 + 455 455 + // Query for the ID (works for both insert and update) 456 456 + var id int64 457 457 + err = db.QueryRow(` 458 458 + SELECT id FROM manifests 459 459 + WHERE did = ? AND repository = ? AND digest = ? 460 460 + `, manifest.DID, manifest.Repository, manifest.Digest).Scan(&id) 461 461 + 462 462 + if err != nil { 463 463 + return 0, fmt.Errorf("failed to get manifest ID after upsert: %w", err) 464 464 + } 465 465 + 466 466 + return id, nil 467 467 + } 468 468 + ``` 469 469 + 470 470 + Similar updates needed for: 471 471 + - `GetUserRepositories()` - fetch annotations separately and populate Repository struct 472 472 + - `GetRecentPushes()` - join with annotations or fetch separately 473 473 + - `SearchPushes()` - can now search annotations table directly 474 474 + 475 475 + ### 5. Update Models 476 476 + 477 477 + **File: `pkg/appview/db/models.go`** 478 478 + 479 479 + Remove annotation fields from `Manifest` struct: 480 480 + 481 481 + ```go 482 482 + type Manifest struct { 483 483 + ID int64 484 484 + DID string 485 485 + Repository string 486 486 + Digest string 487 487 + HoldEndpoint string 488 488 + SchemaVersion int 489 489 + MediaType string 490 490 + ConfigDigest string 491 491 + ConfigSize int64 492 492 + CreatedAt time.Time 493 493 + // Removed: Title, Description, SourceURL, DocumentationURL, Licenses, IconURL, ReadmeURL 494 494 + } 495 495 + ``` 496 496 + 497 497 + Keep annotation fields on `Repository` struct (populated from annotations table): 498 498 + 499 499 + ```go 500 500 + type Repository struct { 501 501 + Name string 502 502 + TagCount int 503 503 + ManifestCount int 504 504 + LastPush time.Time 505 505 + Tags []Tag 506 506 + Manifests []Manifest 507 507 + Title string 508 508 + Description string 509 509 + SourceURL string 510 510 + DocumentationURL string 511 511 + Licenses string 512 512 + IconURL string 513 513 + ReadmeURL string 514 514 + Version string // NEW 515 515 + } 516 516 + ``` 517 517 + 518 518 + ### 6. Update Schema.sql 519 519 + 520 520 + **File: `pkg/appview/db/schema.sql`** 521 521 + 522 522 + Add new table: 523 523 + 524 524 + ```sql 525 525 + CREATE TABLE IF NOT EXISTS repository_annotations ( 526 526 + did TEXT NOT NULL, 527 527 + repository TEXT NOT NULL, 528 528 + key TEXT NOT NULL, 529 529 + value TEXT NOT NULL, 530 530 + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 531 531 + PRIMARY KEY(did, repository, key), 532 532 + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE 533 533 + ); 534 534 + CREATE INDEX IF NOT EXISTS idx_repository_annotations_did_repo ON repository_annotations(did, repository); 535 535 + CREATE INDEX IF NOT EXISTS idx_repository_annotations_key ON repository_annotations(key); 536 536 + ``` 537 537 + 538 538 + Update manifests table (remove annotation columns): 539 539 + 540 540 + ```sql 541 541 + CREATE TABLE IF NOT EXISTS manifests ( 542 542 + id INTEGER PRIMARY KEY AUTOINCREMENT, 543 543 + did TEXT NOT NULL, 544 544 + repository TEXT NOT NULL, 545 545 + digest TEXT NOT NULL, 546 546 + hold_endpoint TEXT NOT NULL, 547 547 + schema_version INTEGER NOT NULL, 548 548 + media_type TEXT NOT NULL, 549 549 + config_digest TEXT, 550 550 + config_size INTEGER, 551 551 + created_at TIMESTAMP NOT NULL, 552 552 + UNIQUE(did, repository, digest), 553 553 + FOREIGN KEY(did) REFERENCES users(did) ON DELETE CASCADE 554 554 + ); 555 555 + ``` 556 556 + 557 557 + ## Update Logic Summary 558 558 + 559 559 + **Key Decision: Only update annotations when manifest has data** 560 560 + 561 561 + ``` 562 562 + For each manifest processed (backfill or jetstream): 563 563 + 1. Parse manifest.Annotations map 564 564 + 2. Check if ANY annotation has non-empty value 565 565 + 3. IF hasData: 566 566 + DELETE all annotations for (did, repository) 567 567 + INSERT all annotations from manifest (including empty ones) 568 568 + ELSE: 569 569 + SKIP (don't touch existing annotations) 570 570 + ``` 571 571 + 572 572 + **Why this works:** 573 573 + - Manifest lists have no annotations or all empty → skip, preserve existing 574 574 + - Platform manifests have real data → replace everything 575 575 + - Removing annotation from Dockerfile → it's gone (not in new INSERT) 576 576 + - Can't accidentally clear data (need at least one non-empty value) 577 577 + 578 578 + ## UI/Template Changes 579 579 + 580 580 + ### Handler Updates 581 581 + 582 582 + **File: `pkg/appview/handlers/repository.go`** 583 583 + 584 584 + Update the handler to include version: 585 585 + 586 586 + ```go 587 587 + // Fetch repository metadata from annotations 588 588 + title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL, version, err := db.GetRepositoryMetadata(h.DB, owner.DID, repository) 589 589 + if err != nil { 590 590 + log.Printf("Failed to fetch repository metadata: %v", err) 591 591 + // Continue without metadata on error 592 592 + } else { 593 593 + repo.Title = title 594 594 + repo.Description = description 595 595 + repo.SourceURL = sourceURL 596 596 + repo.DocumentationURL = documentationURL 597 597 + repo.Licenses = licenses 598 598 + repo.IconURL = iconURL 599 599 + repo.ReadmeURL = readmeURL 600 600 + repo.Version = version // NEW 601 601 + } 602 602 + ``` 603 603 + 604 604 + ### Template Updates 605 605 + 606 606 + **File: `pkg/appview/templates/pages/repository.html`** 607 607 + 608 608 + Update the metadata section condition to include version: 609 609 + 610 610 + ```html 611 611 + <!-- Metadata Section --> 612 612 + {{ if or .Repository.Licenses .Repository.SourceURL .Repository.DocumentationURL .Repository.Version }} 613 613 + <div class="repo-metadata"> 614 614 + <!-- Version Badge (if present) --> 615 615 + {{ if .Repository.Version }} 616 616 + <span class="metadata-badge version-badge" title="Version"> 617 617 + {{ .Repository.Version }} 618 618 + </span> 619 619 + {{ end }} 620 620 + 621 621 + <!-- License Badges --> 622 622 + {{ if .Repository.Licenses }} 623 623 + {{ range parseLicenses .Repository.Licenses }} 624 624 + {{ if .IsValid }} 625 625 + <a href="{{ .URL }}" target="_blank" rel="noopener noreferrer" class="metadata-badge license-badge" title="{{ .Name }}"> 626 626 + {{ .SPDXID }} 627 627 + </a> 628 628 + {{ else }} 629 629 + <span class="metadata-badge license-badge" title="Custom license: {{ .Name }}"> 630 630 + {{ .Name }} 631 631 + </span> 632 632 + {{ end }} 633 633 + {{ end }} 634 634 + {{ end }} 635 635 + 636 636 + <!-- Source Link --> 637 637 + {{ if .Repository.SourceURL }} 638 638 + <a href="{{ .Repository.SourceURL }}" target="_blank" class="metadata-link"> 639 639 + Source 640 640 + </a> 641 641 + {{ end }} 642 642 + 643 643 + <!-- Documentation Link --> 644 644 + {{ if .Repository.DocumentationURL }} 645 645 + <a href="{{ .Repository.DocumentationURL }}" target="_blank" class="metadata-link"> 646 646 + Documentation 647 647 + </a> 648 648 + {{ end }} 649 649 + </div> 650 650 + {{ end }} 651 651 + ``` 652 652 + 653 653 + ### CSS Updates 654 654 + 655 655 + **File: `pkg/appview/static/css/style.css`** 656 656 + 657 657 + Add styling for version badge (different color from license badge): 658 658 + 659 659 + ```css 660 660 + .version-badge { 661 661 + background: #0969da; /* GitHub blue */ 662 662 + color: white; 663 663 + padding: 0.25rem 0.5rem; 664 664 + border-radius: 0.25rem; 665 665 + font-size: 0.875rem; 666 666 + font-weight: 500; 667 667 + display: inline-block; 668 668 + } 669 669 + ``` 670 670 + 671 671 + ### Data Flow Summary 672 672 + 673 673 + **Before refactor:** 674 674 + ``` 675 675 + DB columns → GetRepositoryMetadata() → Handler assigns to Repository struct → Template displays 676 676 + ``` 677 677 + 678 678 + **After refactor:** 679 679 + ``` 680 680 + annotations table → GetRepositoryAnnotations() → GetRepositoryMetadata() extracts known fields → 681 681 + Handler assigns to Repository struct → Template displays (same as before) 682 682 + ``` 683 683 + 684 684 + **Key point:** Templates still access `.Repository.Title`, `.Repository.Version`, etc. - the source just changed from DB columns to annotations table. The abstraction layer hides this complexity. 685 685 + 686 686 + ## Benefits Recap 687 687 + 688 688 + 1. **Flexible**: Support any OCI annotation without code changes 689 689 + 2. **Clean**: No NULL columns in manifests table 690 690 + 3. **Simple queries**: `SELECT * FROM repository_annotations WHERE did=? AND repo=?` 691 691 + 4. **Safe updates**: Only update when manifest has data 692 692 + 5. **Natural deletion**: Remove annotation from Dockerfile → it's deleted on next push 693 693 + 6. **Extensible**: Future features (annotation search, filtering) are trivial 694 694 + 695 695 + ## Testing Checklist 696 696 + 697 697 + After migration: 698 698 + - [ ] Verify existing repositories show annotations correctly 699 699 + - [ ] Push new manifest with annotations → updates correctly 700 700 + - [ ] Push manifest list → doesn't clear annotations 701 701 + - [ ] Remove annotation from Dockerfile and push → annotation deleted 702 702 + - [ ] Backfill re-run → annotations repopulated correctly 703 703 + - [ ] Search still works (if implemented)

+8 -8

pkg/appview/db/hold_store.go

··· 8 8 9 9 // HoldCaptainRecord represents a cached captain record from a hold's PDS 10 10 type HoldCaptainRecord struct { 11 11 - HoldDID string 12 12 - OwnerDID string 13 13 - Public bool 14 14 - AllowAllCrew bool 15 15 - DeployedAt string 16 16 - Region string 17 17 - Provider string 18 18 - UpdatedAt time.Time 11 11 + HoldDID string `json:"-"` // Set manually, not from JSON 12 12 + OwnerDID string `json:"owner"` 13 13 + Public bool `json:"public"` 14 14 + AllowAllCrew bool `json:"allowAllCrew"` 15 15 + DeployedAt string `json:"deployedAt"` 16 16 + Region string `json:"region"` 17 17 + Provider string `json:"provider"` 18 18 + UpdatedAt time.Time `json:"-"` // Set manually, not from JSON 19 19 } 20 20 21 21 // GetCaptainRecord retrieves a captain record from the cache

+27 -335

pkg/appview/jetstream/backfill.go

··· 8 8 "strings" 9 9 "time" 10 10 11 11 - "github.com/bluesky-social/indigo/atproto/identity" 12 11 "github.com/bluesky-social/indigo/atproto/syntax" 13 12 13 13 + "atcr.io/pkg/appview" 14 14 "atcr.io/pkg/appview/db" 15 15 "atcr.io/pkg/atproto" 16 16 ) ··· 19 19 type BackfillWorker struct { 20 20 db *sql.DB 21 21 client *atproto.Client 22 22 - directory identity.Directory 23 23 - defaultHoldDID string // Default hold DID from AppView config (e.g., "did:web:hold01.atcr.io") 24 24 - testMode bool // If true, suppress warnings for external holds 22 22 + processor *Processor // Shared processor for DB operations 23 23 + defaultHoldDID string // Default hold DID from AppView config (e.g., "did:web:hold01.atcr.io") 24 24 + testMode bool // If true, suppress warnings for external holds 25 25 } 26 26 27 27 // BackfillState tracks backfill progress ··· 44 44 45 45 return &BackfillWorker{ 46 46 db: database, 47 47 - client: client, // This points to the relay 48 48 - directory: identity.DefaultDirectory(), 47 47 + client: client, // This points to the relay 48 48 + processor: NewProcessor(database, false), // No cache for batch processing 49 49 defaultHoldDID: defaultHoldDID, 50 50 testMode: testMode, 51 51 }, nil ··· 132 132 // backfillRepo backfills all records for a single repo/DID 133 133 func (b *BackfillWorker) backfillRepo(ctx context.Context, did, collection string) (int, error) { 134 134 // Ensure user exists in database and get their PDS endpoint 135 135 - if err := b.ensureUser(ctx, did); err != nil { 135 135 + if err := b.processor.EnsureUser(ctx, did); err != nil { 136 136 return 0, fmt.Errorf("failed to ensure user: %w", err) 137 137 } 138 138 ··· 142 142 return 0, fmt.Errorf("invalid DID %s: %w", did, err) 143 143 } 144 144 145 145 - ident, err := b.directory.LookupDID(ctx, didParsed) 145 145 + ident, err := b.processor.directory.LookupDID(ctx, didParsed) 146 146 if err != nil { 147 147 return 0, fmt.Errorf("failed to resolve DID to PDS: %w", err) 148 148 } ··· 173 173 // Process each record 174 174 for _, record := range records { 175 175 // Track what we found for deletion reconciliation 176 176 - if collection == atproto.ManifestCollection { 176 176 + switch collection { 177 177 + case atproto.ManifestCollection: 177 178 var manifestRecord atproto.ManifestRecord 178 179 if err := json.Unmarshal(record.Value, &manifestRecord); err == nil { 179 180 foundManifestDigests = append(foundManifestDigests, manifestRecord.Digest) 180 181 } 181 181 - } else if collection == atproto.TagCollection { 182 182 + case atproto.TagCollection: 182 183 var tagRecord atproto.TagRecord 183 184 if err := json.Unmarshal(record.Value, &tagRecord); err == nil { 184 185 foundTags = append(foundTags, struct{ Repository, Tag string }{ ··· 186 187 Tag: tagRecord.Tag, 187 188 }) 188 189 } 189 189 - } else if collection == atproto.StarCollection { 190 190 + case atproto.StarCollection: 190 191 var starRecord atproto.StarRecord 191 192 if err := json.Unmarshal(record.Value, &starRecord); err == nil { 192 193 key := fmt.Sprintf("%s/%s", starRecord.Subject.DID, starRecord.Subject.Repository) ··· 278 279 func (b *BackfillWorker) processRecord(ctx context.Context, did, collection string, record *atproto.Record) error { 279 280 switch collection { 280 281 case atproto.ManifestCollection: 281 281 - return b.processManifestRecord(did, record) 282 282 + _, err := b.processor.ProcessManifest(context.Background(), did, record.Value) 283 283 + return err 282 284 case atproto.TagCollection: 283 283 - return b.processTagRecord(did, record) 285 285 + return b.processor.ProcessTag(context.Background(), did, record.Value) 284 286 case atproto.StarCollection: 285 285 - return b.processStarRecord(did, record) 287 287 + return b.processor.ProcessStar(context.Background(), did, record.Value) 286 288 case atproto.SailorProfileCollection: 287 287 - return b.processSailorProfileRecord(ctx, did, record) 289 289 + return b.processor.ProcessSailorProfile(ctx, did, record.Value, b.queryCaptainRecordWrapper) 288 290 default: 289 291 return fmt.Errorf("unsupported collection: %s", collection) 290 292 } 291 293 } 292 294 293 293 - // processManifestRecord processes a manifest record 294 294 - func (b *BackfillWorker) processManifestRecord(did string, record *atproto.Record) error { 295 295 - var manifestRecord atproto.ManifestRecord 296 296 - if err := json.Unmarshal(record.Value, &manifestRecord); err != nil { 297 297 - return fmt.Errorf("failed to unmarshal manifest: %w", err) 298 298 - } 299 299 - 300 300 - // Extract OCI annotations from manifest 301 301 - var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string 302 302 - if manifestRecord.Annotations != nil { 303 303 - title = manifestRecord.Annotations["org.opencontainers.image.title"] 304 304 - description = manifestRecord.Annotations["org.opencontainers.image.description"] 305 305 - sourceURL = manifestRecord.Annotations["org.opencontainers.image.source"] 306 306 - documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"] 307 307 - licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"] 308 308 - iconURL = manifestRecord.Annotations["io.atcr.icon"] 309 309 - readmeURL = manifestRecord.Annotations["io.atcr.readme"] 310 310 - } 311 311 - 312 312 - // Detect manifest type 313 313 - isManifestList := len(manifestRecord.Manifests) > 0 314 314 - 315 315 - // Prepare manifest for insertion 316 316 - manifest := &db.Manifest{ 317 317 - DID: did, 318 318 - Repository: manifestRecord.Repository, 319 319 - Digest: manifestRecord.Digest, 320 320 - MediaType: manifestRecord.MediaType, 321 321 - SchemaVersion: manifestRecord.SchemaVersion, 322 322 - HoldEndpoint: manifestRecord.HoldEndpoint, 323 323 - CreatedAt: manifestRecord.CreatedAt, 324 324 - Title: title, 325 325 - Description: description, 326 326 - SourceURL: sourceURL, 327 327 - DocumentationURL: documentationURL, 328 328 - Licenses: licenses, 329 329 - IconURL: iconURL, 330 330 - ReadmeURL: readmeURL, 331 331 - } 332 332 - 333 333 - // Set config fields only for image manifests (not manifest lists) 334 334 - if !isManifestList && manifestRecord.Config != nil { 335 335 - manifest.ConfigDigest = manifestRecord.Config.Digest 336 336 - manifest.ConfigSize = manifestRecord.Config.Size 337 337 - } 338 338 - 339 339 - // Platform info is only stored for multi-arch images in manifest_references table 340 340 - // Single-arch images don't need platform display (it's obvious) 341 341 - 342 342 - // Insert manifest (or get existing ID if already exists) 343 343 - manifestID, err := db.InsertManifest(b.db, manifest) 344 344 - if err != nil { 345 345 - // If manifest already exists, get its ID so we can still insert references/layers 346 346 - if strings.Contains(err.Error(), "UNIQUE constraint failed") { 347 347 - // Query for existing manifest ID 348 348 - var existingID int64 349 349 - err := b.db.QueryRow(` 350 350 - SELECT id FROM manifests 351 351 - WHERE did = ? AND repository = ? AND digest = ? 352 352 - `, manifest.DID, manifest.Repository, manifest.Digest).Scan(&existingID) 353 353 - 354 354 - if err != nil { 355 355 - return fmt.Errorf("failed to get existing manifest ID: %w", err) 356 356 - } 357 357 - manifestID = existingID 358 358 - } else { 359 359 - return fmt.Errorf("failed to insert manifest: %w", err) 360 360 - } 361 361 - } 362 362 - 363 363 - if isManifestList { 364 364 - // Insert manifest references (for manifest lists/indexes) 365 365 - for i, ref := range manifestRecord.Manifests { 366 366 - platformArch := "" 367 367 - platformOS := "" 368 368 - platformVariant := "" 369 369 - platformOSVersion := "" 370 370 - 371 371 - if ref.Platform != nil { 372 372 - platformArch = ref.Platform.Architecture 373 373 - platformOS = ref.Platform.OS 374 374 - platformVariant = ref.Platform.Variant 375 375 - platformOSVersion = ref.Platform.OSVersion 376 376 - } 377 377 - 378 378 - if err := db.InsertManifestReference(b.db, &db.ManifestReference{ 379 379 - ManifestID: manifestID, 380 380 - Digest: ref.Digest, 381 381 - MediaType: ref.MediaType, 382 382 - Size: ref.Size, 383 383 - PlatformArchitecture: platformArch, 384 384 - PlatformOS: platformOS, 385 385 - PlatformVariant: platformVariant, 386 386 - PlatformOSVersion: platformOSVersion, 387 387 - ReferenceIndex: i, 388 388 - }); err != nil { 389 389 - // Continue on error - reference might already exist 390 390 - continue 391 391 - } 392 392 - } 393 393 - } else { 394 394 - // Insert layers (for image manifests) 395 395 - for i, layer := range manifestRecord.Layers { 396 396 - if err := db.InsertLayer(b.db, &db.Layer{ 397 397 - ManifestID: manifestID, 398 398 - Digest: layer.Digest, 399 399 - MediaType: layer.MediaType, 400 400 - Size: layer.Size, 401 401 - LayerIndex: i, 402 402 - }); err != nil { 403 403 - // Continue on error - layer might already exist 404 404 - continue 405 405 - } 406 406 - } 407 407 - } 408 408 - 409 409 - return nil 410 410 - } 411 411 - 412 412 - // processTagRecord processes a tag record 413 413 - func (b *BackfillWorker) processTagRecord(did string, record *atproto.Record) error { 414 414 - var tagRecord atproto.TagRecord 415 415 - if err := json.Unmarshal(record.Value, &tagRecord); err != nil { 416 416 - return fmt.Errorf("failed to unmarshal tag: %w", err) 417 417 - } 418 418 - 419 419 - // Extract digest from tag record (tries manifest field first, falls back to manifestDigest) 420 420 - manifestDigest, err := tagRecord.GetManifestDigest() 421 421 - if err != nil { 422 422 - return fmt.Errorf("failed to get manifest digest from tag record: %w", err) 423 423 - } 424 424 - 425 425 - // Insert or update tag 426 426 - return db.UpsertTag(b.db, &db.Tag{ 427 427 - DID: did, 428 428 - Repository: tagRecord.Repository, 429 429 - Tag: tagRecord.Tag, 430 430 - Digest: manifestDigest, 431 431 - CreatedAt: tagRecord.UpdatedAt, 432 432 - }) 433 433 - } 434 434 - 435 435 - // processStarRecord processes a star record 436 436 - func (b *BackfillWorker) processStarRecord(did string, record *atproto.Record) error { 437 437 - var starRecord atproto.StarRecord 438 438 - if err := json.Unmarshal(record.Value, &starRecord); err != nil { 439 439 - return fmt.Errorf("failed to unmarshal star: %w", err) 440 440 - } 441 441 - 442 442 - // Upsert the star record (idempotent - won't duplicate) 443 443 - // The DID here is the starrer (user who starred) 444 444 - // The subject contains the owner DID and repository 445 445 - // Star count will be calculated on demand from the stars table 446 446 - return db.UpsertStar(b.db, did, starRecord.Subject.DID, starRecord.Subject.Repository, starRecord.CreatedAt) 447 447 - } 448 448 - 449 449 - // processSailorProfileRecord processes a sailor profile record 450 450 - // Extracts defaultHold and queries the hold's captain record to cache it 451 451 - func (b *BackfillWorker) processSailorProfileRecord(ctx context.Context, did string, record *atproto.Record) error { 452 452 - var profileRecord atproto.SailorProfileRecord 453 453 - if err := json.Unmarshal(record.Value, &profileRecord); err != nil { 454 454 - return fmt.Errorf("failed to unmarshal sailor profile: %w", err) 455 455 - } 456 456 - 457 457 - // Skip if no default hold set 458 458 - if profileRecord.DefaultHold == "" { 459 459 - return nil 460 460 - } 461 461 - 462 462 - // Convert hold URL/DID to canonical DID 463 463 - holdDID := atproto.ResolveHoldDIDFromURL(profileRecord.DefaultHold) 464 464 - if holdDID == "" { 465 465 - fmt.Printf("WARNING [backfill]: Invalid hold reference in profile for %s: %s\n", did, profileRecord.DefaultHold) 466 466 - return nil 467 467 - } 468 468 - 469 469 - // Query and cache the captain record 295 295 + // queryCaptainRecordWrapper wraps queryCaptainRecord with backfill-specific logic 296 296 + func (b *BackfillWorker) queryCaptainRecordWrapper(ctx context.Context, holdDID string) error { 470 297 if err := b.queryCaptainRecord(ctx, holdDID); err != nil { 471 298 // In test mode, only warn about default hold (local hold) 472 299 // External/production holds may not have captain records yet (dev ahead of prod) ··· 478 305 // Don't fail the whole backfill - just skip this hold 479 306 return nil 480 307 } 481 481 - 482 308 return nil 483 309 } 484 310 ··· 494 320 } 495 321 496 322 // Resolve hold DID to URL 497 497 - // For did:web, we need to fetch .well-known/did.json 498 498 - holdURL, err := resolveHoldDIDToURL(ctx, holdDID) 499 499 - if err != nil { 500 500 - return fmt.Errorf("failed to resolve hold DID to URL: %w", err) 501 501 - } 323 323 + holdURL := appview.ResolveHoldURL(holdDID) 502 324 503 325 // Create client for hold's PDS 504 326 holdClient := atproto.NewClient(holdURL, holdDID, "") ··· 522 344 return fmt.Errorf("failed to get captain record: %w", err) 523 345 } 524 346 525 525 - // Parse captain record from the record's Value field 526 526 - var captainRecord struct { 527 527 - Owner string `json:"owner"` 528 528 - Public bool `json:"public"` 529 529 - AllowAllCrew bool `json:"allowAllCrew"` 530 530 - DeployedAt string `json:"deployedAt"` 531 531 - Region string `json:"region"` 532 532 - Provider string `json:"provider"` 533 533 - } 534 534 - 347 347 + // Parse captain record directly into db struct 348 348 + var captainRecord db.HoldCaptainRecord 535 349 if err := json.Unmarshal(record.Value, &captainRecord); err != nil { 536 350 return fmt.Errorf("failed to parse captain record: %w", err) 537 351 } 538 352 539 539 - // Cache in database 540 540 - dbRecord := &db.HoldCaptainRecord{ 541 541 - HoldDID: holdDID, 542 542 - OwnerDID: captainRecord.Owner, 543 543 - Public: captainRecord.Public, 544 544 - AllowAllCrew: captainRecord.AllowAllCrew, 545 545 - DeployedAt: captainRecord.DeployedAt, 546 546 - Region: captainRecord.Region, 547 547 - Provider: captainRecord.Provider, 548 548 - UpdatedAt: time.Now(), 549 549 - } 353 353 + // Set fields not from JSON 354 354 + captainRecord.HoldDID = holdDID 355 355 + captainRecord.UpdatedAt = time.Now() 550 356 551 551 - if err := db.UpsertCaptainRecord(b.db, dbRecord); err != nil { 357 357 + if err := db.UpsertCaptainRecord(b.db, &captainRecord); err != nil { 552 358 return fmt.Errorf("failed to cache captain record: %w", err) 553 359 } 554 360 555 555 - fmt.Printf("Backfill: Cached captain record for hold %s (owner: %s)\n", holdDID, captainRecord.Owner) 361 361 + fmt.Printf("Backfill: Cached captain record for hold %s (owner: %s)\n", holdDID, captainRecord.OwnerDID) 556 362 return nil 557 363 } 558 558 - 559 559 - // resolveHoldDIDToURL resolves a hold DID to its service endpoint URL 560 560 - // Fetches the DID document and returns both the canonical DID and service endpoint 561 561 - func resolveHoldDIDToURL(ctx context.Context, inputDID string) (string, error) { 562 562 - // For did:web, construct the .well-known URL 563 563 - if !strings.HasPrefix(inputDID, "did:web:") { 564 564 - return "", fmt.Errorf("only did:web is supported, got: %s", inputDID) 565 565 - } 566 566 - 567 567 - // Extract hostname from did:web:hostname[:port] 568 568 - hostname := strings.TrimPrefix(inputDID, "did:web:") 569 569 - 570 570 - // Try HTTP first (for local Docker), then HTTPS 571 571 - var serviceEndpoint string 572 572 - for _, scheme := range []string{"http", "https"} { 573 573 - testURL := fmt.Sprintf("%s://%s/.well-known/did.json", scheme, hostname) 574 574 - 575 575 - // Fetch DID document (use NewClient to initialize httpClient) 576 576 - client := atproto.NewClient("", "", "") 577 577 - didDoc, err := client.FetchDIDDocument(ctx, testURL) 578 578 - if err == nil && didDoc != nil { 579 579 - // Extract service endpoint from DID document 580 580 - for _, service := range didDoc.Service { 581 581 - if service.Type == "AtprotoPersonalDataServer" || service.Type == "AtcrHoldService" { 582 582 - serviceEndpoint = service.ServiceEndpoint 583 583 - break 584 584 - } 585 585 - } 586 586 - 587 587 - if serviceEndpoint != "" { 588 588 - fmt.Printf("DEBUG [backfill]: Resolved %s → canonical DID: %s, endpoint: %s\n", 589 589 - inputDID, didDoc.ID, serviceEndpoint) 590 590 - return serviceEndpoint, nil 591 591 - } 592 592 - } 593 593 - } 594 594 - 595 595 - // Fallback: assume the hold service is at the root of the hostname 596 596 - // Try HTTP first for local development 597 597 - url := fmt.Sprintf("http://%s", hostname) 598 598 - fmt.Printf("WARNING [backfill]: Failed to fetch DID document for %s, using fallback URL: %s\n", inputDID, url) 599 599 - return url, nil 600 600 - } 601 601 - 602 602 - // ensureUser resolves and upserts a user by DID 603 603 - func (b *BackfillWorker) ensureUser(ctx context.Context, did string) error { 604 604 - // Check if user already exists 605 605 - existingUser, err := db.GetUserByDID(b.db, did) 606 606 - if err == nil && existingUser != nil { 607 607 - // Update last seen 608 608 - existingUser.LastSeen = time.Now() 609 609 - return db.UpsertUser(b.db, existingUser) 610 610 - } 611 611 - 612 612 - // Resolve DID to get handle and PDS endpoint 613 613 - didParsed, err := syntax.ParseDID(did) 614 614 - if err != nil { 615 615 - // Fallback: use DID as handle 616 616 - user := &db.User{ 617 617 - DID: did, 618 618 - Handle: did, 619 619 - PDSEndpoint: "https://bsky.social", 620 620 - LastSeen: time.Now(), 621 621 - } 622 622 - return db.UpsertUser(b.db, user) 623 623 - } 624 624 - 625 625 - ident, err := b.directory.LookupDID(ctx, didParsed) 626 626 - if err != nil { 627 627 - // Fallback: use DID as handle 628 628 - user := &db.User{ 629 629 - DID: did, 630 630 - Handle: did, 631 631 - PDSEndpoint: "https://bsky.social", 632 632 - LastSeen: time.Now(), 633 633 - } 634 634 - return db.UpsertUser(b.db, user) 635 635 - } 636 636 - 637 637 - resolvedDID := ident.DID.String() 638 638 - handle := ident.Handle.String() 639 639 - pdsEndpoint := ident.PDSEndpoint() 640 640 - 641 641 - // If handle is invalid or PDS is missing, use defaults 642 642 - if handle == "handle.invalid" || handle == "" { 643 643 - handle = resolvedDID 644 644 - } 645 645 - if pdsEndpoint == "" { 646 646 - pdsEndpoint = "https://bsky.social" 647 647 - } 648 648 - 649 649 - // Fetch user's Bluesky profile (including avatar) 650 650 - // Use public Bluesky AppView API (doesn't require auth for public profiles) 651 651 - avatar := "" 652 652 - publicClient := atproto.NewClient("https://public.api.bsky.app", "", "") 653 653 - profile, err := publicClient.GetActorProfile(ctx, resolvedDID) 654 654 - if err != nil { 655 655 - fmt.Printf("WARNING [backfill]: Failed to fetch profile for DID %s: %v\n", resolvedDID, err) 656 656 - // Continue without avatar 657 657 - } else { 658 658 - avatar = profile.Avatar 659 659 - } 660 660 - 661 661 - // Upsert to database 662 662 - user := &db.User{ 663 663 - DID: resolvedDID, 664 664 - Handle: handle, 665 665 - PDSEndpoint: pdsEndpoint, 666 666 - Avatar: avatar, 667 667 - LastSeen: time.Now(), 668 668 - } 669 669 - 670 670 - return db.UpsertUser(b.db, user) 671 671 - }

+320

pkg/appview/jetstream/processor.go

··· 1 1 + package jetstream 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "database/sql" 6 6 + "encoding/json" 7 7 + "fmt" 8 8 + "strings" 9 9 + "time" 10 10 + 11 11 + "github.com/bluesky-social/indigo/atproto/identity" 12 12 + "github.com/bluesky-social/indigo/atproto/syntax" 13 13 + 14 14 + "atcr.io/pkg/appview/db" 15 15 + "atcr.io/pkg/atproto" 16 16 + ) 17 17 + 18 18 + // Processor handles shared database operations for both Worker (live) and Backfill (sync) 19 19 + // This eliminates code duplication between the two data ingestion paths 20 20 + type Processor struct { 21 21 + db *sql.DB 22 22 + directory identity.Directory 23 23 + userCache *UserCache // Optional - enabled for Worker, disabled for Backfill 24 24 + useCache bool 25 25 + } 26 26 + 27 27 + // NewProcessor creates a new shared processor 28 28 + // useCache: true for Worker (live streaming), false for Backfill (batch processing) 29 29 + func NewProcessor(database *sql.DB, useCache bool) *Processor { 30 30 + p := &Processor{ 31 31 + db: database, 32 32 + directory: identity.DefaultDirectory(), 33 33 + useCache: useCache, 34 34 + } 35 35 + 36 36 + if useCache { 37 37 + p.userCache = &UserCache{ 38 38 + cache: make(map[string]*db.User), 39 39 + } 40 40 + } 41 41 + 42 42 + return p 43 43 + } 44 44 + 45 45 + // EnsureUser resolves and upserts a user by DID 46 46 + // Uses cache if enabled (Worker), queries DB if cache disabled (Backfill) 47 47 + func (p *Processor) EnsureUser(ctx context.Context, did string) error { 48 48 + // Check cache first (if enabled) 49 49 + if p.useCache && p.userCache != nil { 50 50 + if user, ok := p.userCache.cache[did]; ok { 51 51 + // Update last seen 52 52 + user.LastSeen = time.Now() 53 53 + return db.UpsertUser(p.db, user) 54 54 + } 55 55 + } else if !p.useCache { 56 56 + // No cache - check if user already exists in DB 57 57 + existingUser, err := db.GetUserByDID(p.db, did) 58 58 + if err == nil && existingUser != nil { 59 59 + // Update last seen 60 60 + existingUser.LastSeen = time.Now() 61 61 + return db.UpsertUser(p.db, existingUser) 62 62 + } 63 63 + } 64 64 + 65 65 + // Resolve DID to get handle and PDS endpoint 66 66 + didParsed, err := syntax.ParseDID(did) 67 67 + if err != nil { 68 68 + // Fallback: use DID as handle 69 69 + user := &db.User{ 70 70 + DID: did, 71 71 + Handle: did, 72 72 + PDSEndpoint: "https://bsky.social", 73 73 + LastSeen: time.Now(), 74 74 + } 75 75 + if p.useCache { 76 76 + p.userCache.cache[did] = user 77 77 + } 78 78 + return db.UpsertUser(p.db, user) 79 79 + } 80 80 + 81 81 + ident, err := p.directory.LookupDID(ctx, didParsed) 82 82 + if err != nil { 83 83 + // Fallback: use DID as handle 84 84 + user := &db.User{ 85 85 + DID: did, 86 86 + Handle: did, 87 87 + PDSEndpoint: "https://bsky.social", 88 88 + LastSeen: time.Now(), 89 89 + } 90 90 + if p.useCache { 91 91 + p.userCache.cache[did] = user 92 92 + } 93 93 + return db.UpsertUser(p.db, user) 94 94 + } 95 95 + 96 96 + resolvedDID := ident.DID.String() 97 97 + handle := ident.Handle.String() 98 98 + pdsEndpoint := ident.PDSEndpoint() 99 99 + 100 100 + // If handle is invalid or PDS is missing, use defaults 101 101 + if handle == "handle.invalid" || handle == "" { 102 102 + handle = resolvedDID 103 103 + } 104 104 + if pdsEndpoint == "" { 105 105 + pdsEndpoint = "https://bsky.social" 106 106 + } 107 107 + 108 108 + // Fetch user's Bluesky profile (including avatar) 109 109 + // Use public Bluesky AppView API (doesn't require auth for public profiles) 110 110 + avatar := "" 111 111 + publicClient := atproto.NewClient("https://public.api.bsky.app", "", "") 112 112 + profile, err := publicClient.GetActorProfile(ctx, resolvedDID) 113 113 + if err != nil { 114 114 + fmt.Printf("WARNING [processor]: Failed to fetch profile for DID %s: %v\n", resolvedDID, err) 115 115 + // Continue without avatar 116 116 + } else { 117 117 + avatar = profile.Avatar 118 118 + } 119 119 + 120 120 + // Create user record 121 121 + user := &db.User{ 122 122 + DID: resolvedDID, 123 123 + Handle: handle, 124 124 + PDSEndpoint: pdsEndpoint, 125 125 + Avatar: avatar, 126 126 + LastSeen: time.Now(), 127 127 + } 128 128 + 129 129 + // Cache if enabled 130 130 + if p.useCache { 131 131 + p.userCache.cache[did] = user 132 132 + } 133 133 + 134 134 + // Upsert to database 135 135 + return db.UpsertUser(p.db, user) 136 136 + } 137 137 + 138 138 + // ProcessManifest processes a manifest record and stores it in the database 139 139 + // Returns the manifest ID for further processing (layers/references) 140 140 + func (p *Processor) ProcessManifest(ctx context.Context, did string, recordData []byte) (int64, error) { 141 141 + // Unmarshal manifest record 142 142 + var manifestRecord atproto.ManifestRecord 143 143 + if err := json.Unmarshal(recordData, &manifestRecord); err != nil { 144 144 + return 0, fmt.Errorf("failed to unmarshal manifest: %w", err) 145 145 + } 146 146 + // Extract OCI annotations from manifest 147 147 + var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string 148 148 + if manifestRecord.Annotations != nil { 149 149 + title = manifestRecord.Annotations["org.opencontainers.image.title"] 150 150 + description = manifestRecord.Annotations["org.opencontainers.image.description"] 151 151 + sourceURL = manifestRecord.Annotations["org.opencontainers.image.source"] 152 152 + documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"] 153 153 + licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"] 154 154 + iconURL = manifestRecord.Annotations["io.atcr.icon"] 155 155 + readmeURL = manifestRecord.Annotations["io.atcr.readme"] 156 156 + } 157 157 + 158 158 + // Detect manifest type 159 159 + isManifestList := len(manifestRecord.Manifests) > 0 160 160 + 161 161 + // Prepare manifest for insertion 162 162 + manifest := &db.Manifest{ 163 163 + DID: did, 164 164 + Repository: manifestRecord.Repository, 165 165 + Digest: manifestRecord.Digest, 166 166 + MediaType: manifestRecord.MediaType, 167 167 + SchemaVersion: manifestRecord.SchemaVersion, 168 168 + HoldEndpoint: manifestRecord.HoldEndpoint, 169 169 + CreatedAt: manifestRecord.CreatedAt, 170 170 + Title: title, 171 171 + Description: description, 172 172 + SourceURL: sourceURL, 173 173 + DocumentationURL: documentationURL, 174 174 + Licenses: licenses, 175 175 + IconURL: iconURL, 176 176 + ReadmeURL: readmeURL, 177 177 + } 178 178 + 179 179 + // Set config fields only for image manifests (not manifest lists) 180 180 + if !isManifestList && manifestRecord.Config != nil { 181 181 + manifest.ConfigDigest = manifestRecord.Config.Digest 182 182 + manifest.ConfigSize = manifestRecord.Config.Size 183 183 + } 184 184 + 185 185 + // Insert manifest 186 186 + manifestID, err := db.InsertManifest(p.db, manifest) 187 187 + if err != nil { 188 188 + // For backfill: if manifest already exists, get its ID 189 189 + if strings.Contains(err.Error(), "UNIQUE constraint failed") { 190 190 + var existingID int64 191 191 + err := p.db.QueryRow(` 192 192 + SELECT id FROM manifests 193 193 + WHERE did = ? AND repository = ? AND digest = ? 194 194 + `, manifest.DID, manifest.Repository, manifest.Digest).Scan(&existingID) 195 195 + 196 196 + if err != nil { 197 197 + return 0, fmt.Errorf("failed to get existing manifest ID: %w", err) 198 198 + } 199 199 + manifestID = existingID 200 200 + } else { 201 201 + return 0, fmt.Errorf("failed to insert manifest: %w", err) 202 202 + } 203 203 + } 204 204 + 205 205 + // Insert manifest references or layers 206 206 + if isManifestList { 207 207 + // Insert manifest references (for manifest lists/indexes) 208 208 + for i, ref := range manifestRecord.Manifests { 209 209 + platformArch := "" 210 210 + platformOS := "" 211 211 + platformVariant := "" 212 212 + platformOSVersion := "" 213 213 + 214 214 + if ref.Platform != nil { 215 215 + platformArch = ref.Platform.Architecture 216 216 + platformOS = ref.Platform.OS 217 217 + platformVariant = ref.Platform.Variant 218 218 + platformOSVersion = ref.Platform.OSVersion 219 219 + } 220 220 + 221 221 + if err := db.InsertManifestReference(p.db, &db.ManifestReference{ 222 222 + ManifestID: manifestID, 223 223 + Digest: ref.Digest, 224 224 + MediaType: ref.MediaType, 225 225 + Size: ref.Size, 226 226 + PlatformArchitecture: platformArch, 227 227 + PlatformOS: platformOS, 228 228 + PlatformVariant: platformVariant, 229 229 + PlatformOSVersion: platformOSVersion, 230 230 + ReferenceIndex: i, 231 231 + }); err != nil { 232 232 + // Continue on error - reference might already exist 233 233 + continue 234 234 + } 235 235 + } 236 236 + } else { 237 237 + // Insert layers (for image manifests) 238 238 + for i, layer := range manifestRecord.Layers { 239 239 + if err := db.InsertLayer(p.db, &db.Layer{ 240 240 + ManifestID: manifestID, 241 241 + Digest: layer.Digest, 242 242 + MediaType: layer.MediaType, 243 243 + Size: layer.Size, 244 244 + LayerIndex: i, 245 245 + }); err != nil { 246 246 + // Continue on error - layer might already exist 247 247 + continue 248 248 + } 249 249 + } 250 250 + } 251 251 + 252 252 + return manifestID, nil 253 253 + } 254 254 + 255 255 + // ProcessTag processes a tag record and stores it in the database 256 256 + func (p *Processor) ProcessTag(ctx context.Context, did string, recordData []byte) error { 257 257 + // Unmarshal tag record 258 258 + var tagRecord atproto.TagRecord 259 259 + if err := json.Unmarshal(recordData, &tagRecord); err != nil { 260 260 + return fmt.Errorf("failed to unmarshal tag: %w", err) 261 261 + } 262 262 + // Extract digest from tag record (tries manifest field first, falls back to manifestDigest) 263 263 + manifestDigest, err := tagRecord.GetManifestDigest() 264 264 + if err != nil { 265 265 + return fmt.Errorf("failed to get manifest digest from tag record: %w", err) 266 266 + } 267 267 + 268 268 + // Insert or update tag 269 269 + return db.UpsertTag(p.db, &db.Tag{ 270 270 + DID: did, 271 271 + Repository: tagRecord.Repository, 272 272 + Tag: tagRecord.Tag, 273 273 + Digest: manifestDigest, 274 274 + CreatedAt: tagRecord.UpdatedAt, 275 275 + }) 276 276 + } 277 277 + 278 278 + // ProcessStar processes a star record and stores it in the database 279 279 + func (p *Processor) ProcessStar(ctx context.Context, did string, recordData []byte) error { 280 280 + // Unmarshal star record 281 281 + var starRecord atproto.StarRecord 282 282 + if err := json.Unmarshal(recordData, &starRecord); err != nil { 283 283 + return fmt.Errorf("failed to unmarshal star: %w", err) 284 284 + } 285 285 + // Upsert the star record (idempotent - won't duplicate) 286 286 + // The DID here is the starrer (user who starred) 287 287 + // The subject contains the owner DID and repository 288 288 + // Star count will be calculated on demand from the stars table 289 289 + return db.UpsertStar(p.db, did, starRecord.Subject.DID, starRecord.Subject.Repository, starRecord.CreatedAt) 290 290 + } 291 291 + 292 292 + // ProcessSailorProfile processes a sailor profile record 293 293 + // This is primarily used by backfill to cache captain records for holds 294 294 + func (p *Processor) ProcessSailorProfile(ctx context.Context, did string, recordData []byte, queryCaptainFn func(context.Context, string) error) error { 295 295 + // Unmarshal sailor profile record 296 296 + var profileRecord atproto.SailorProfileRecord 297 297 + if err := json.Unmarshal(recordData, &profileRecord); err != nil { 298 298 + return fmt.Errorf("failed to unmarshal sailor profile: %w", err) 299 299 + } 300 300 + 301 301 + // Skip if no default hold set 302 302 + if profileRecord.DefaultHold == "" { 303 303 + return nil 304 304 + } 305 305 + 306 306 + // Convert hold URL/DID to canonical DID 307 307 + holdDID := atproto.ResolveHoldDIDFromURL(profileRecord.DefaultHold) 308 308 + if holdDID == "" { 309 309 + fmt.Printf("WARNING [processor]: Invalid hold reference in profile for %s: %s\n", did, profileRecord.DefaultHold) 310 310 + return nil 311 311 + } 312 312 + 313 313 + // Query and cache the captain record using provided function 314 314 + // This allows backfill-specific logic (retries, test mode handling) without duplicating it here 315 315 + if queryCaptainFn != nil { 316 316 + return queryCaptainFn(ctx, holdDID) 317 317 + } 318 318 + 319 319 + return nil 320 320 + }

+540

pkg/appview/jetstream/processor_test.go

··· 1 1 + package jetstream 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "database/sql" 6 6 + "encoding/json" 7 7 + "testing" 8 8 + "time" 9 9 + 10 10 + "atcr.io/pkg/atproto" 11 11 + _ "github.com/mattn/go-sqlite3" 12 12 + ) 13 13 + 14 14 + // setupTestDB creates an in-memory SQLite database for testing 15 15 + func setupTestDB(t *testing.T) *sql.DB { 16 16 + database, err := sql.Open("sqlite3", ":memory:") 17 17 + if err != nil { 18 18 + t.Fatalf("Failed to open test database: %v", err) 19 19 + } 20 20 + 21 21 + // Create schema 22 22 + schema := ` 23 23 + CREATE TABLE users ( 24 24 + did TEXT PRIMARY KEY, 25 25 + handle TEXT NOT NULL, 26 26 + pds_endpoint TEXT NOT NULL, 27 27 + avatar TEXT, 28 28 + last_seen TIMESTAMP NOT NULL 29 29 + ); 30 30 + 31 31 + CREATE TABLE manifests ( 32 32 + id INTEGER PRIMARY KEY AUTOINCREMENT, 33 33 + did TEXT NOT NULL, 34 34 + repository TEXT NOT NULL, 35 35 + digest TEXT NOT NULL, 36 36 + hold_endpoint TEXT NOT NULL, 37 37 + schema_version INTEGER NOT NULL, 38 38 + media_type TEXT NOT NULL, 39 39 + config_digest TEXT, 40 40 + config_size INTEGER, 41 41 + created_at TIMESTAMP NOT NULL, 42 42 + title TEXT, 43 43 + description TEXT, 44 44 + source_url TEXT, 45 45 + documentation_url TEXT, 46 46 + licenses TEXT, 47 47 + icon_url TEXT, 48 48 + readme_url TEXT, 49 49 + UNIQUE(did, repository, digest) 50 50 + ); 51 51 + 52 52 + CREATE TABLE layers ( 53 53 + manifest_id INTEGER NOT NULL, 54 54 + digest TEXT NOT NULL, 55 55 + size INTEGER NOT NULL, 56 56 + media_type TEXT NOT NULL, 57 57 + layer_index INTEGER NOT NULL, 58 58 + PRIMARY KEY(manifest_id, layer_index) 59 59 + ); 60 60 + 61 61 + CREATE TABLE manifest_references ( 62 62 + manifest_id INTEGER NOT NULL, 63 63 + digest TEXT NOT NULL, 64 64 + media_type TEXT NOT NULL, 65 65 + size INTEGER NOT NULL, 66 66 + platform_architecture TEXT, 67 67 + platform_os TEXT, 68 68 + platform_variant TEXT, 69 69 + platform_os_version TEXT, 70 70 + reference_index INTEGER NOT NULL, 71 71 + PRIMARY KEY(manifest_id, reference_index) 72 72 + ); 73 73 + 74 74 + CREATE TABLE tags ( 75 75 + id INTEGER PRIMARY KEY AUTOINCREMENT, 76 76 + did TEXT NOT NULL, 77 77 + repository TEXT NOT NULL, 78 78 + tag TEXT NOT NULL, 79 79 + digest TEXT NOT NULL, 80 80 + created_at TIMESTAMP NOT NULL, 81 81 + UNIQUE(did, repository, tag) 82 82 + ); 83 83 + 84 84 + CREATE TABLE stars ( 85 85 + starrer_did TEXT NOT NULL, 86 86 + owner_did TEXT NOT NULL, 87 87 + repository TEXT NOT NULL, 88 88 + created_at TIMESTAMP NOT NULL, 89 89 + PRIMARY KEY(starrer_did, owner_did, repository) 90 90 + ); 91 91 + ` 92 92 + 93 93 + if _, err := database.Exec(schema); err != nil { 94 94 + t.Fatalf("Failed to create schema: %v", err) 95 95 + } 96 96 + 97 97 + return database 98 98 + } 99 99 + 100 100 + func TestNewProcessor(t *testing.T) { 101 101 + database := setupTestDB(t) 102 102 + defer database.Close() 103 103 + 104 104 + tests := []struct { 105 105 + name string 106 106 + useCache bool 107 107 + }{ 108 108 + {"with cache", true}, 109 109 + {"without cache", false}, 110 110 + } 111 111 + 112 112 + for _, tt := range tests { 113 113 + t.Run(tt.name, func(t *testing.T) { 114 114 + p := NewProcessor(database, tt.useCache) 115 115 + if p == nil { 116 116 + t.Fatal("NewProcessor returned nil") 117 117 + } 118 118 + if p.db != database { 119 119 + t.Error("Processor database not set correctly") 120 120 + } 121 121 + if p.useCache != tt.useCache { 122 122 + t.Errorf("useCache = %v, want %v", p.useCache, tt.useCache) 123 123 + } 124 124 + if tt.useCache && p.userCache == nil { 125 125 + t.Error("Cache enabled but userCache is nil") 126 126 + } 127 127 + if !tt.useCache && p.userCache != nil { 128 128 + t.Error("Cache disabled but userCache is not nil") 129 129 + } 130 130 + }) 131 131 + } 132 132 + } 133 133 + 134 134 + func TestProcessManifest_ImageManifest(t *testing.T) { 135 135 + database := setupTestDB(t) 136 136 + defer database.Close() 137 137 + 138 138 + p := NewProcessor(database, false) 139 139 + ctx := context.Background() 140 140 + 141 141 + // Create test manifest record 142 142 + manifestRecord := &atproto.ManifestRecord{ 143 143 + Repository: "test-app", 144 144 + Digest: "sha256:abc123", 145 145 + MediaType: "application/vnd.oci.image.manifest.v1+json", 146 146 + SchemaVersion: 2, 147 147 + HoldEndpoint: "did:web:hold01.atcr.io", 148 148 + CreatedAt: time.Now(), 149 149 + Config: &atproto.BlobReference{ 150 150 + Digest: "sha256:config123", 151 151 + Size: 1234, 152 152 + }, 153 153 + Layers: []atproto.BlobReference{ 154 154 + {Digest: "sha256:layer1", Size: 5000, MediaType: "application/vnd.oci.image.layer.v1.tar+gzip"}, 155 155 + {Digest: "sha256:layer2", Size: 3000, MediaType: "application/vnd.oci.image.layer.v1.tar+gzip"}, 156 156 + }, 157 157 + Annotations: map[string]string{ 158 158 + "org.opencontainers.image.title": "Test App", 159 159 + "org.opencontainers.image.description": "A test application", 160 160 + "org.opencontainers.image.source": "https://github.com/test/app", 161 161 + "org.opencontainers.image.licenses": "MIT", 162 162 + "io.atcr.icon": "https://example.com/icon.png", 163 163 + }, 164 164 + } 165 165 + 166 166 + // Marshal to bytes for ProcessManifest 167 167 + recordBytes, err := json.Marshal(manifestRecord) 168 168 + if err != nil { 169 169 + t.Fatalf("Failed to marshal manifest: %v", err) 170 170 + } 171 171 + 172 172 + // Process manifest 173 173 + manifestID, err := p.ProcessManifest(ctx, "did:plc:test123", recordBytes) 174 174 + if err != nil { 175 175 + t.Fatalf("ProcessManifest failed: %v", err) 176 176 + } 177 177 + if manifestID == 0 { 178 178 + t.Error("Expected non-zero manifest ID") 179 179 + } 180 180 + 181 181 + // Verify manifest was inserted 182 182 + var count int 183 183 + err = database.QueryRow("SELECT COUNT(*) FROM manifests WHERE did = ? AND repository = ? AND digest = ?", 184 184 + "did:plc:test123", "test-app", "sha256:abc123").Scan(&count) 185 185 + if err != nil { 186 186 + t.Fatalf("Failed to query manifests: %v", err) 187 187 + } 188 188 + if count != 1 { 189 189 + t.Errorf("Expected 1 manifest, got %d", count) 190 190 + } 191 191 + 192 192 + // Verify annotations were stored 193 193 + var title, source string 194 194 + err = database.QueryRow("SELECT title, source_url FROM manifests WHERE id = ?", manifestID).Scan(&title, &source) 195 195 + if err != nil { 196 196 + t.Fatalf("Failed to query manifest fields: %v", err) 197 197 + } 198 198 + if title != "Test App" { 199 199 + t.Errorf("title = %q, want %q", title, "Test App") 200 200 + } 201 201 + if source != "https://github.com/test/app" { 202 202 + t.Errorf("source_url = %q, want %q", source, "https://github.com/test/app") 203 203 + } 204 204 + 205 205 + // Verify layers were inserted 206 206 + var layerCount int 207 207 + err = database.QueryRow("SELECT COUNT(*) FROM layers WHERE manifest_id = ?", manifestID).Scan(&layerCount) 208 208 + if err != nil { 209 209 + t.Fatalf("Failed to query layers: %v", err) 210 210 + } 211 211 + if layerCount != 2 { 212 212 + t.Errorf("Expected 2 layers, got %d", layerCount) 213 213 + } 214 214 + 215 215 + // Verify no manifest references (this is an image, not a list) 216 216 + var refCount int 217 217 + err = database.QueryRow("SELECT COUNT(*) FROM manifest_references WHERE manifest_id = ?", manifestID).Scan(&refCount) 218 218 + if err != nil { 219 219 + t.Fatalf("Failed to query manifest_references: %v", err) 220 220 + } 221 221 + if refCount != 0 { 222 222 + t.Errorf("Expected 0 manifest references, got %d", refCount) 223 223 + } 224 224 + } 225 225 + 226 226 + func TestProcessManifest_ManifestList(t *testing.T) { 227 227 + database := setupTestDB(t) 228 228 + defer database.Close() 229 229 + 230 230 + p := NewProcessor(database, false) 231 231 + ctx := context.Background() 232 232 + 233 233 + // Create test manifest list record 234 234 + manifestRecord := &atproto.ManifestRecord{ 235 235 + Repository: "test-app", 236 236 + Digest: "sha256:list123", 237 237 + MediaType: "application/vnd.oci.image.index.v1+json", 238 238 + SchemaVersion: 2, 239 239 + HoldEndpoint: "did:web:hold01.atcr.io", 240 240 + CreatedAt: time.Now(), 241 241 + Manifests: []atproto.ManifestReference{ 242 242 + { 243 243 + Digest: "sha256:amd64manifest", 244 244 + MediaType: "application/vnd.oci.image.manifest.v1+json", 245 245 + Size: 1000, 246 246 + Platform: &atproto.Platform{ 247 247 + Architecture: "amd64", 248 248 + OS: "linux", 249 249 + }, 250 250 + }, 251 251 + { 252 252 + Digest: "sha256:arm64manifest", 253 253 + MediaType: "application/vnd.oci.image.manifest.v1+json", 254 254 + Size: 1100, 255 255 + Platform: &atproto.Platform{ 256 256 + Architecture: "arm64", 257 257 + OS: "linux", 258 258 + Variant: "v8", 259 259 + }, 260 260 + }, 261 261 + }, 262 262 + } 263 263 + 264 264 + // Marshal to bytes for ProcessManifest 265 265 + recordBytes, err := json.Marshal(manifestRecord) 266 266 + if err != nil { 267 267 + t.Fatalf("Failed to marshal manifest: %v", err) 268 268 + } 269 269 + 270 270 + // Process manifest list 271 271 + manifestID, err := p.ProcessManifest(ctx, "did:plc:test123", recordBytes) 272 272 + if err != nil { 273 273 + t.Fatalf("ProcessManifest failed: %v", err) 274 274 + } 275 275 + 276 276 + // Verify manifest references were inserted 277 277 + var refCount int 278 278 + err = database.QueryRow("SELECT COUNT(*) FROM manifest_references WHERE manifest_id = ?", manifestID).Scan(&refCount) 279 279 + if err != nil { 280 280 + t.Fatalf("Failed to query manifest_references: %v", err) 281 281 + } 282 282 + if refCount != 2 { 283 283 + t.Errorf("Expected 2 manifest references, got %d", refCount) 284 284 + } 285 285 + 286 286 + // Verify platform info was stored 287 287 + var arch, os string 288 288 + err = database.QueryRow("SELECT platform_architecture, platform_os FROM manifest_references WHERE manifest_id = ? AND reference_index = 0", manifestID).Scan(&arch, &os) 289 289 + if err != nil { 290 290 + t.Fatalf("Failed to query platform info: %v", err) 291 291 + } 292 292 + if arch != "amd64" { 293 293 + t.Errorf("platform_architecture = %q, want %q", arch, "amd64") 294 294 + } 295 295 + if os != "linux" { 296 296 + t.Errorf("platform_os = %q, want %q", os, "linux") 297 297 + } 298 298 + 299 299 + // Verify no layers (this is a list, not an image) 300 300 + var layerCount int 301 301 + err = database.QueryRow("SELECT COUNT(*) FROM layers WHERE manifest_id = ?", manifestID).Scan(&layerCount) 302 302 + if err != nil { 303 303 + t.Fatalf("Failed to query layers: %v", err) 304 304 + } 305 305 + if layerCount != 0 { 306 306 + t.Errorf("Expected 0 layers, got %d", layerCount) 307 307 + } 308 308 + } 309 309 + 310 310 + func TestProcessTag(t *testing.T) { 311 311 + database := setupTestDB(t) 312 312 + defer database.Close() 313 313 + 314 314 + p := NewProcessor(database, false) 315 315 + ctx := context.Background() 316 316 + 317 317 + // Create test tag record (using ManifestDigest field for simplicity) 318 318 + tagRecord := &atproto.TagRecord{ 319 319 + Repository: "test-app", 320 320 + Tag: "latest", 321 321 + ManifestDigest: "sha256:abc123", 322 322 + UpdatedAt: time.Now(), 323 323 + } 324 324 + 325 325 + // Marshal to bytes for ProcessTag 326 326 + recordBytes, err := json.Marshal(tagRecord) 327 327 + if err != nil { 328 328 + t.Fatalf("Failed to marshal tag: %v", err) 329 329 + } 330 330 + 331 331 + // Process tag 332 332 + err = p.ProcessTag(ctx, "did:plc:test123", recordBytes) 333 333 + if err != nil { 334 334 + t.Fatalf("ProcessTag failed: %v", err) 335 335 + } 336 336 + 337 337 + // Verify tag was inserted 338 338 + var count int 339 339 + err = database.QueryRow("SELECT COUNT(*) FROM tags WHERE did = ? AND repository = ? AND tag = ?", 340 340 + "did:plc:test123", "test-app", "latest").Scan(&count) 341 341 + if err != nil { 342 342 + t.Fatalf("Failed to query tags: %v", err) 343 343 + } 344 344 + if count != 1 { 345 345 + t.Errorf("Expected 1 tag, got %d", count) 346 346 + } 347 347 + 348 348 + // Verify digest was stored 349 349 + var digest string 350 350 + err = database.QueryRow("SELECT digest FROM tags WHERE did = ? AND repository = ? AND tag = ?", 351 351 + "did:plc:test123", "test-app", "latest").Scan(&digest) 352 352 + if err != nil { 353 353 + t.Fatalf("Failed to query tag digest: %v", err) 354 354 + } 355 355 + if digest != "sha256:abc123" { 356 356 + t.Errorf("digest = %q, want %q", digest, "sha256:abc123") 357 357 + } 358 358 + 359 359 + // Test upserting same tag with new digest 360 360 + tagRecord.ManifestDigest = "sha256:newdigest" 361 361 + recordBytes, err = json.Marshal(tagRecord) 362 362 + if err != nil { 363 363 + t.Fatalf("Failed to marshal tag: %v", err) 364 364 + } 365 365 + err = p.ProcessTag(ctx, "did:plc:test123", recordBytes) 366 366 + if err != nil { 367 367 + t.Fatalf("ProcessTag (upsert) failed: %v", err) 368 368 + } 369 369 + 370 370 + // Verify tag was updated 371 371 + err = database.QueryRow("SELECT digest FROM tags WHERE did = ? AND repository = ? AND tag = ?", 372 372 + "did:plc:test123", "test-app", "latest").Scan(&digest) 373 373 + if err != nil { 374 374 + t.Fatalf("Failed to query updated tag: %v", err) 375 375 + } 376 376 + if digest != "sha256:newdigest" { 377 377 + t.Errorf("digest = %q, want %q", digest, "sha256:newdigest") 378 378 + } 379 379 + 380 380 + // Verify still only one tag (upsert, not insert) 381 381 + err = database.QueryRow("SELECT COUNT(*) FROM tags WHERE did = ? AND repository = ? AND tag = ?", 382 382 + "did:plc:test123", "test-app", "latest").Scan(&count) 383 383 + if err != nil { 384 384 + t.Fatalf("Failed to query tags after upsert: %v", err) 385 385 + } 386 386 + if count != 1 { 387 387 + t.Errorf("Expected 1 tag after upsert, got %d", count) 388 388 + } 389 389 + } 390 390 + 391 391 + func TestProcessStar(t *testing.T) { 392 392 + database := setupTestDB(t) 393 393 + defer database.Close() 394 394 + 395 395 + p := NewProcessor(database, false) 396 396 + ctx := context.Background() 397 397 + 398 398 + // Create test star record 399 399 + starRecord := &atproto.StarRecord{ 400 400 + Subject: atproto.StarSubject{ 401 401 + DID: "did:plc:owner123", 402 402 + Repository: "test-app", 403 403 + }, 404 404 + CreatedAt: time.Now(), 405 405 + } 406 406 + 407 407 + // Marshal to bytes for ProcessStar 408 408 + recordBytes, err := json.Marshal(starRecord) 409 409 + if err != nil { 410 410 + t.Fatalf("Failed to marshal star: %v", err) 411 411 + } 412 412 + 413 413 + // Process star 414 414 + err = p.ProcessStar(ctx, "did:plc:starrer123", recordBytes) 415 415 + if err != nil { 416 416 + t.Fatalf("ProcessStar failed: %v", err) 417 417 + } 418 418 + 419 419 + // Verify star was inserted 420 420 + var count int 421 421 + err = database.QueryRow("SELECT COUNT(*) FROM stars WHERE starrer_did = ? AND owner_did = ? AND repository = ?", 422 422 + "did:plc:starrer123", "did:plc:owner123", "test-app").Scan(&count) 423 423 + if err != nil { 424 424 + t.Fatalf("Failed to query stars: %v", err) 425 425 + } 426 426 + if count != 1 { 427 427 + t.Errorf("Expected 1 star, got %d", count) 428 428 + } 429 429 + 430 430 + // Test upserting same star (should be idempotent) 431 431 + recordBytes, err = json.Marshal(starRecord) 432 432 + if err != nil { 433 433 + t.Fatalf("Failed to marshal star: %v", err) 434 434 + } 435 435 + err = p.ProcessStar(ctx, "did:plc:starrer123", recordBytes) 436 436 + if err != nil { 437 437 + t.Fatalf("ProcessStar (upsert) failed: %v", err) 438 438 + } 439 439 + 440 440 + // Verify still only one star 441 441 + err = database.QueryRow("SELECT COUNT(*) FROM stars WHERE starrer_did = ? AND owner_did = ? AND repository = ?", 442 442 + "did:plc:starrer123", "did:plc:owner123", "test-app").Scan(&count) 443 443 + if err != nil { 444 444 + t.Fatalf("Failed to query stars after upsert: %v", err) 445 445 + } 446 446 + if count != 1 { 447 447 + t.Errorf("Expected 1 star after upsert, got %d", count) 448 448 + } 449 449 + } 450 450 + 451 451 + func TestProcessManifest_Duplicate(t *testing.T) { 452 452 + database := setupTestDB(t) 453 453 + defer database.Close() 454 454 + 455 455 + p := NewProcessor(database, false) 456 456 + ctx := context.Background() 457 457 + 458 458 + manifestRecord := &atproto.ManifestRecord{ 459 459 + Repository: "test-app", 460 460 + Digest: "sha256:abc123", 461 461 + MediaType: "application/vnd.oci.image.manifest.v1+json", 462 462 + SchemaVersion: 2, 463 463 + HoldEndpoint: "did:web:hold01.atcr.io", 464 464 + CreatedAt: time.Now(), 465 465 + } 466 466 + 467 467 + // Marshal to bytes for ProcessManifest 468 468 + recordBytes, err := json.Marshal(manifestRecord) 469 469 + if err != nil { 470 470 + t.Fatalf("Failed to marshal manifest: %v", err) 471 471 + } 472 472 + 473 473 + // Insert first time 474 474 + id1, err := p.ProcessManifest(ctx, "did:plc:test123", recordBytes) 475 475 + if err != nil { 476 476 + t.Fatalf("First ProcessManifest failed: %v", err) 477 477 + } 478 478 + 479 479 + // Insert duplicate 480 480 + id2, err := p.ProcessManifest(ctx, "did:plc:test123", recordBytes) 481 481 + if err != nil { 482 482 + t.Fatalf("Duplicate ProcessManifest failed: %v", err) 483 483 + } 484 484 + 485 485 + // Should return existing ID 486 486 + if id1 != id2 { 487 487 + t.Errorf("Duplicate manifest got different ID: %d vs %d", id1, id2) 488 488 + } 489 489 + 490 490 + // Verify only one manifest exists 491 491 + var count int 492 492 + err = database.QueryRow("SELECT COUNT(*) FROM manifests WHERE did = ? AND digest = ?", 493 493 + "did:plc:test123", "sha256:abc123").Scan(&count) 494 494 + if err != nil { 495 495 + t.Fatalf("Failed to query manifests: %v", err) 496 496 + } 497 497 + if count != 1 { 498 498 + t.Errorf("Expected 1 manifest, got %d", count) 499 499 + } 500 500 + } 501 501 + 502 502 + func TestProcessManifest_EmptyAnnotations(t *testing.T) { 503 503 + database := setupTestDB(t) 504 504 + defer database.Close() 505 505 + 506 506 + p := NewProcessor(database, false) 507 507 + ctx := context.Background() 508 508 + 509 509 + // Manifest with nil annotations 510 510 + manifestRecord := &atproto.ManifestRecord{ 511 511 + Repository: "test-app", 512 512 + Digest: "sha256:abc123", 513 513 + MediaType: "application/vnd.oci.image.manifest.v1+json", 514 514 + SchemaVersion: 2, 515 515 + HoldEndpoint: "did:web:hold01.atcr.io", 516 516 + CreatedAt: time.Now(), 517 517 + Annotations: nil, 518 518 + } 519 519 + 520 520 + // Marshal to bytes for ProcessManifest 521 521 + recordBytes, err := json.Marshal(manifestRecord) 522 522 + if err != nil { 523 523 + t.Fatalf("Failed to marshal manifest: %v", err) 524 524 + } 525 525 + 526 526 + manifestID, err := p.ProcessManifest(ctx, "did:plc:test123", recordBytes) 527 527 + if err != nil { 528 528 + t.Fatalf("ProcessManifest failed: %v", err) 529 529 + } 530 530 + 531 531 + // Verify annotation fields are empty strings (not NULL) 532 532 + var title string 533 533 + err = database.QueryRow("SELECT title FROM manifests WHERE id = ?", manifestID).Scan(&title) 534 534 + if err != nil { 535 535 + t.Fatalf("Failed to query title: %v", err) 536 536 + } 537 537 + if title != "" { 538 538 + t.Errorf("Expected empty title, got %q", title) 539 539 + } 540 540 + }

+28 -222

pkg/appview/jetstream/worker.go

··· 9 9 "sync" 10 10 "time" 11 11 12 12 - "github.com/bluesky-social/indigo/atproto/identity" 13 13 - "github.com/bluesky-social/indigo/atproto/syntax" 14 14 - 15 12 "atcr.io/pkg/appview/db" 16 13 "atcr.io/pkg/atproto" 17 14 "github.com/gorilla/websocket" ··· 33 30 startCursor int64 34 31 wantedCollections []string 35 32 debugCollectionCount int 36 36 - userCache *UserCache 37 37 - directory identity.Directory 33 33 + processor *Processor // Shared processor for DB operations 38 34 eventCallback EventCallback 39 35 connStartTime time.Time // Track when connection started for debugging 40 36 ··· 65 61 atproto.TagCollection, // io.atcr.tag 66 62 atproto.StarCollection, // io.atcr.sailor.star 67 63 }, 68 68 - userCache: &UserCache{ 69 69 - cache: make(map[string]*db.User), 70 70 - }, 71 71 - directory: identity.DefaultDirectory(), 64 64 + processor: NewProcessor(database, true), // Use cache for live streaming 72 65 } 73 66 } 74 67 ··· 333 326 } 334 327 } 335 328 336 336 - // ensureUser resolves and upserts a user by DID 337 337 - func (w *Worker) ensureUser(ctx context.Context, did string) error { 338 338 - // Check cache first 339 339 - if user, ok := w.userCache.cache[did]; ok { 340 340 - // Update last seen 341 341 - user.LastSeen = time.Now() 342 342 - return db.UpsertUser(w.db, user) 343 343 - } 344 344 - 345 345 - // Resolve DID to get handle and PDS endpoint 346 346 - didParsed, err := syntax.ParseDID(did) 347 347 - if err != nil { 348 348 - fmt.Printf("WARNING: Invalid DID %s: %v (using DID as handle)\n", did, err) 349 349 - // Fallback: use DID as handle 350 350 - user := &db.User{ 351 351 - DID: did, 352 352 - Handle: did, 353 353 - PDSEndpoint: "https://bsky.social", // Default PDS endpoint as fallback 354 354 - LastSeen: time.Now(), 355 355 - } 356 356 - w.userCache.cache[did] = user 357 357 - return db.UpsertUser(w.db, user) 358 358 - } 359 359 - 360 360 - ident, err := w.directory.LookupDID(ctx, didParsed) 361 361 - if err != nil { 362 362 - fmt.Printf("WARNING: Failed to resolve DID %s: %v (using DID as handle)\n", did, err) 363 363 - // Fallback: use DID as handle 364 364 - user := &db.User{ 365 365 - DID: did, 366 366 - Handle: did, 367 367 - PDSEndpoint: "https://bsky.social", // Default PDS endpoint as fallback 368 368 - LastSeen: time.Now(), 369 369 - } 370 370 - w.userCache.cache[did] = user 371 371 - return db.UpsertUser(w.db, user) 372 372 - } 373 373 - 374 374 - resolvedDID := ident.DID.String() 375 375 - handle := ident.Handle.String() 376 376 - pdsEndpoint := ident.PDSEndpoint() 377 377 - 378 378 - // If handle is invalid or PDS is missing, use defaults 379 379 - if handle == "handle.invalid" || handle == "" { 380 380 - handle = resolvedDID 381 381 - } 382 382 - if pdsEndpoint == "" { 383 383 - pdsEndpoint = "https://bsky.social" 384 384 - } 385 385 - 386 386 - // Fetch user's Bluesky profile (including avatar) 387 387 - // Use public Bluesky AppView API (doesn't require auth for public profiles) 388 388 - avatar := "" 389 389 - publicClient := atproto.NewClient("https://public.api.bsky.app", "", "") 390 390 - profile, err := publicClient.GetActorProfile(ctx, resolvedDID) 391 391 - if err != nil { 392 392 - fmt.Printf("WARNING [worker]: Failed to fetch profile for DID %s: %v\n", resolvedDID, err) 393 393 - // Continue without avatar 394 394 - } else { 395 395 - avatar = profile.Avatar 396 396 - } 397 397 - 398 398 - // Cache the user 399 399 - user := &db.User{ 400 400 - DID: resolvedDID, 401 401 - Handle: handle, 402 402 - PDSEndpoint: pdsEndpoint, 403 403 - Avatar: avatar, 404 404 - LastSeen: time.Now(), 405 405 - } 406 406 - w.userCache.cache[did] = user 407 407 - 408 408 - // Upsert to database 409 409 - return db.UpsertUser(w.db, user) 410 410 - } 411 411 - 412 329 // processManifest processes a manifest commit event 413 330 func (w *Worker) processManifest(commit *CommitEvent) error { 414 331 // Resolve and upsert user with handle/PDS endpoint 415 415 - if err := w.ensureUser(context.Background(), commit.DID); err != nil { 332 332 + if err := w.processor.EnsureUser(context.Background(), commit.DID); err != nil { 416 333 return fmt.Errorf("failed to ensure user: %w", err) 417 334 } 418 335 ··· 427 344 } 428 345 429 346 // Parse manifest record 430 430 - var manifestRecord atproto.ManifestRecord 431 431 - if commit.Record != nil { 432 432 - recordBytes, err := json.Marshal(commit.Record) 433 433 - if err != nil { 434 434 - return fmt.Errorf("failed to marshal record: %w", err) 435 435 - } 436 436 - if err := json.Unmarshal(recordBytes, &manifestRecord); err != nil { 437 437 - return fmt.Errorf("failed to unmarshal manifest: %w", err) 438 438 - } 439 439 - } else { 440 440 - // No record data, can't process 441 441 - return nil 442 442 - } 443 443 - 444 444 - // Extract OCI annotations from manifest 445 445 - var title, description, sourceURL, documentationURL, licenses, iconURL, readmeURL string 446 446 - if manifestRecord.Annotations != nil { 447 447 - title = manifestRecord.Annotations["org.opencontainers.image.title"] 448 448 - description = manifestRecord.Annotations["org.opencontainers.image.description"] 449 449 - sourceURL = manifestRecord.Annotations["org.opencontainers.image.source"] 450 450 - documentationURL = manifestRecord.Annotations["org.opencontainers.image.documentation"] 451 451 - licenses = manifestRecord.Annotations["org.opencontainers.image.licenses"] 452 452 - iconURL = manifestRecord.Annotations["io.atcr.icon"] 453 453 - readmeURL = manifestRecord.Annotations["io.atcr.readme"] 454 454 - } 455 455 - 456 456 - // Detect manifest type 457 457 - isManifestList := len(manifestRecord.Manifests) > 0 458 458 - 459 459 - // Prepare manifest for insertion 460 460 - manifest := &db.Manifest{ 461 461 - DID: commit.DID, 462 462 - Repository: manifestRecord.Repository, 463 463 - Digest: manifestRecord.Digest, 464 464 - MediaType: manifestRecord.MediaType, 465 465 - SchemaVersion: manifestRecord.SchemaVersion, 466 466 - HoldEndpoint: manifestRecord.HoldEndpoint, 467 467 - CreatedAt: manifestRecord.CreatedAt, 468 468 - Title: title, 469 469 - Description: description, 470 470 - SourceURL: sourceURL, 471 471 - DocumentationURL: documentationURL, 472 472 - Licenses: licenses, 473 473 - IconURL: iconURL, 474 474 - ReadmeURL: readmeURL, 475 475 - } 476 476 - 477 477 - // Set config fields only for image manifests (not manifest lists) 478 478 - if !isManifestList && manifestRecord.Config != nil { 479 479 - manifest.ConfigDigest = manifestRecord.Config.Digest 480 480 - manifest.ConfigSize = manifestRecord.Config.Size 347 347 + if commit.Record == nil { 348 348 + return nil // No record data, can't process 481 349 } 482 350 483 483 - // Insert manifest 484 484 - manifestID, err := db.InsertManifest(w.db, manifest) 351 351 + // Marshal map to bytes for processing 352 352 + recordBytes, err := json.Marshal(commit.Record) 485 353 if err != nil { 486 486 - return fmt.Errorf("failed to insert manifest: %w", err) 354 354 + return fmt.Errorf("failed to marshal record: %w", err) 487 355 } 488 356 489 489 - if isManifestList { 490 490 - // Insert manifest references (for manifest lists/indexes) 491 491 - for i, ref := range manifestRecord.Manifests { 492 492 - platformArch := "" 493 493 - platformOS := "" 494 494 - platformVariant := "" 495 495 - platformOSVersion := "" 496 496 - 497 497 - if ref.Platform != nil { 498 498 - platformArch = ref.Platform.Architecture 499 499 - platformOS = ref.Platform.OS 500 500 - platformVariant = ref.Platform.Variant 501 501 - platformOSVersion = ref.Platform.OSVersion 502 502 - } 503 503 - 504 504 - if err := db.InsertManifestReference(w.db, &db.ManifestReference{ 505 505 - ManifestID: manifestID, 506 506 - Digest: ref.Digest, 507 507 - MediaType: ref.MediaType, 508 508 - Size: ref.Size, 509 509 - PlatformArchitecture: platformArch, 510 510 - PlatformOS: platformOS, 511 511 - PlatformVariant: platformVariant, 512 512 - PlatformOSVersion: platformOSVersion, 513 513 - ReferenceIndex: i, 514 514 - }); err != nil { 515 515 - // Continue on error - reference might already exist 516 516 - continue 517 517 - } 518 518 - } 519 519 - } else { 520 520 - // Insert layers (for image manifests) 521 521 - for i, layer := range manifestRecord.Layers { 522 522 - if err := db.InsertLayer(w.db, &db.Layer{ 523 523 - ManifestID: manifestID, 524 524 - Digest: layer.Digest, 525 525 - MediaType: layer.MediaType, 526 526 - Size: layer.Size, 527 527 - LayerIndex: i, 528 528 - }); err != nil { 529 529 - // Continue on error - layer might already exist 530 530 - continue 531 531 - } 532 532 - } 533 533 - } 534 534 - 535 535 - return nil 357 357 + // Use shared processor for DB operations 358 358 + _, err = w.processor.ProcessManifest(context.Background(), commit.DID, recordBytes) 359 359 + return err 536 360 } 537 361 538 362 // processTag processes a tag commit event 539 363 func (w *Worker) processTag(commit *CommitEvent) error { 540 364 // Resolve and upsert user with handle/PDS endpoint 541 541 - if err := w.ensureUser(context.Background(), commit.DID); err != nil { 365 365 + if err := w.processor.EnsureUser(context.Background(), commit.DID); err != nil { 542 366 return fmt.Errorf("failed to ensure user: %w", err) 543 367 } 544 368 ··· 557 381 } 558 382 559 383 // Parse tag record 560 560 - var tagRecord atproto.TagRecord 561 561 - if commit.Record != nil { 562 562 - recordBytes, err := json.Marshal(commit.Record) 563 563 - if err != nil { 564 564 - return fmt.Errorf("failed to marshal record: %w", err) 565 565 - } 566 566 - if err := json.Unmarshal(recordBytes, &tagRecord); err != nil { 567 567 - return fmt.Errorf("failed to unmarshal tag: %w", err) 568 568 - } 569 569 - } else { 384 384 + if commit.Record == nil { 570 385 return nil 571 386 } 572 387 573 573 - // Extract digest from tag record (tries manifest field first, falls back to manifestDigest) 574 574 - manifestDigest, err := tagRecord.GetManifestDigest() 388 388 + // Marshal map to bytes for processing 389 389 + recordBytes, err := json.Marshal(commit.Record) 575 390 if err != nil { 576 576 - return fmt.Errorf("failed to get manifest digest from tag record: %w", err) 391 391 + return fmt.Errorf("failed to marshal record: %w", err) 577 392 } 578 393 579 579 - // Insert or update tag 580 580 - return db.UpsertTag(w.db, &db.Tag{ 581 581 - DID: commit.DID, 582 582 - Repository: tagRecord.Repository, 583 583 - Tag: tagRecord.Tag, 584 584 - Digest: manifestDigest, 585 585 - CreatedAt: tagRecord.UpdatedAt, 586 586 - }) 394 394 + // Use shared processor for DB operations 395 395 + return w.processor.ProcessTag(context.Background(), commit.DID, recordBytes) 587 396 } 588 397 589 398 // processStar processes a star commit event 590 399 func (w *Worker) processStar(commit *CommitEvent) error { 591 400 // Resolve and upsert the user who starred (starrer) 592 592 - if err := w.ensureUser(context.Background(), commit.DID); err != nil { 401 401 + if err := w.processor.EnsureUser(context.Background(), commit.DID); err != nil { 593 402 return fmt.Errorf("failed to ensure user: %w", err) 594 403 } 595 404 ··· 606 415 } 607 416 608 417 // Parse star record 609 609 - var starRecord atproto.StarRecord 610 610 - if commit.Record != nil { 611 611 - recordBytes, err := json.Marshal(commit.Record) 612 612 - if err != nil { 613 613 - return fmt.Errorf("failed to marshal record: %w", err) 614 614 - } 615 615 - if err := json.Unmarshal(recordBytes, &starRecord); err != nil { 616 616 - return fmt.Errorf("failed to unmarshal star: %w", err) 617 617 - } 618 618 - } else { 418 418 + if commit.Record == nil { 619 419 return nil 620 420 } 621 421 622 622 - // Upsert the star record (idempotent - star count will be calculated on demand) 623 623 - return db.UpsertStar(w.db, commit.DID, starRecord.Subject.DID, starRecord.Subject.Repository, starRecord.CreatedAt) 422 422 + // Marshal map to bytes for processing 423 423 + recordBytes, err := json.Marshal(commit.Record) 424 424 + if err != nil { 425 425 + return fmt.Errorf("failed to marshal record: %w", err) 426 426 + } 427 427 + 428 428 + // Use shared processor for DB operations 429 429 + return w.processor.ProcessStar(context.Background(), commit.DID, recordBytes) 624 430 } 625 431 626 432 // JetstreamEvent represents a Jetstream event

+1 -7

pkg/appview/storage/proxy_blob_store.go

··· 40 40 // NewProxyBlobStore creates a new proxy blob store 41 41 func NewProxyBlobStore(ctx *RegistryContext) *ProxyBlobStore { 42 42 // Resolve DID to URL once at construction time 43 43 - holdURL := resolveHoldURL(ctx.HoldDID) 43 43 + holdURL := appview.ResolveHoldURL(ctx.HoldDID) 44 44 45 45 fmt.Printf("DEBUG [proxy_blob_store]: NewProxyBlobStore created with holdDID=%s, holdURL=%s, userDID=%s, repo=%s\n", 46 46 ctx.HoldDID, holdURL, ctx.DID, ctx.Repository) ··· 106 106 return fmt.Errorf("write access denied to hold %s", p.ctx.HoldDID) 107 107 } 108 108 return nil 109 109 - } 110 110 - 111 111 - // resolveHoldURL converts a hold identifier (DID or URL) to an HTTP URL 112 112 - // Deprecated: Use appview.ResolveHoldURL instead 113 113 - func resolveHoldURL(holdDID string) string { 114 114 - return appview.ResolveHoldURL(holdDID) 115 109 } 116 110 117 111 // Stat returns the descriptor for a blob

+2 -1

pkg/appview/storage/proxy_blob_store_test.go

··· 11 11 "testing" 12 12 "time" 13 13 14 14 + "atcr.io/pkg/appview" 14 15 "atcr.io/pkg/atproto" 15 16 "atcr.io/pkg/auth/token" 16 17 "github.com/opencontainers/go-digest" ··· 218 219 219 220 for _, tt := range tests { 220 221 t.Run(tt.name, func(t *testing.T) { 221 221 - result := resolveHoldURL(tt.holdDID) 222 222 + result := appview.ResolveHoldURL(tt.holdDID) 222 223 if result != tt.expected { 223 224 t.Errorf("Expected %s, got %s", tt.expected, result) 224 225 }

+54

scripts/migrate-image.sh

··· 1 1 + #!/bin/bash 2 2 + set -e 3 3 + 4 4 + # Configuration 5 5 + SOURCE_REGISTRY="ghcr.io/evanjarrett/hsm-secrets-operator" 6 6 + TARGET_REGISTRY="atcr.io/evan.jarrett.net/hsm-secrets-operator" 7 7 + TAG="latest" 8 8 + 9 9 + # Image digests 10 10 + AMD64_DIGEST="sha256:274284a623810cf07c5b4735628832751926b7d192863681d5af1b4137f44254" 11 11 + ARM64_DIGEST="sha256:b57929fd100033092766aad1c7e747deef9b1e3206756c11d0d7a7af74daedff" 12 12 + 13 13 + echo "=== Migrating multi-arch image from GHCR to ATCR ===" 14 14 + echo "Source: ${SOURCE_REGISTRY}" 15 15 + echo "Target: ${TARGET_REGISTRY}:${TAG}" 16 16 + echo "" 17 17 + 18 18 + # Tag and push amd64 image 19 19 + echo ">>> Tagging and pushing amd64 image..." 20 20 + docker tag "${SOURCE_REGISTRY}@${AMD64_DIGEST}" "${TARGET_REGISTRY}:${TAG}-amd64" 21 21 + docker push "${TARGET_REGISTRY}:${TAG}-amd64" 22 22 + echo "" 23 23 + 24 24 + # Tag and push arm64 image 25 25 + echo ">>> Tagging and pushing arm64 image..." 26 26 + docker tag "${SOURCE_REGISTRY}@${ARM64_DIGEST}" "${TARGET_REGISTRY}:${TAG}-arm64" 27 27 + docker push "${TARGET_REGISTRY}:${TAG}-arm64" 28 28 + echo "" 29 29 + 30 30 + # Create multi-arch manifest using the pushed tags 31 31 + echo ">>> Creating multi-arch manifest..." 32 32 + docker manifest create "${TARGET_REGISTRY}:${TAG}" \ 33 33 + --amend "${TARGET_REGISTRY}:${TAG}-amd64" \ 34 34 + --amend "${TARGET_REGISTRY}:${TAG}-arm64" 35 35 + echo "" 36 36 + 37 37 + # Annotate the manifest with platform information 38 38 + echo ">>> Annotating manifest with platform information..." 39 39 + docker manifest annotate "${TARGET_REGISTRY}:${TAG}" \ 40 40 + "${TARGET_REGISTRY}:${TAG}-amd64" \ 41 41 + --os linux --arch amd64 42 42 + 43 43 + docker manifest annotate "${TARGET_REGISTRY}:${TAG}" \ 44 44 + "${TARGET_REGISTRY}:${TAG}-arm64" \ 45 45 + --os linux --arch arm64 46 46 + echo "" 47 47 + 48 48 + # Push the manifest list 49 49 + echo ">>> Pushing multi-arch manifest..." 50 50 + docker manifest push "${TARGET_REGISTRY}:${TAG}" 51 51 + echo "" 52 52 + 53 53 + echo "=== Migration complete! ===" 54 54 + echo "You can now pull: docker pull ${TARGET_REGISTRY}:${TAG}"