+5 -2

src/backend.go

··· 46 46 } 47 47 48 48 type ManifestMetadata struct { 49 49 + Name string 50 50 + Size int64 49 51 LastModified time.Time 50 52 ETag string 51 53 } ··· 122 124 // Delete a manifest. 123 125 DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error 124 126 125 125 - // List all manifests. 126 126 - ListManifests(ctx context.Context) (manifests []string, err error) 127 127 + // Iterate through all manifests. Whether manifests that are newly added during iteration 128 128 + // will appear in the results is unspecified. 129 129 + EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] 127 130 128 131 // Check whether a domain has any deployments. 129 132 CheckDomain(ctx context.Context, domain string) (found bool, err error)

+31 -16

src/backend_fs.go

··· 207 207 } 208 208 } 209 209 210 210 - func (fs *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) { 211 211 - err = iofs.WalkDir(fs.siteRoot.FS(), ".", 212 212 - func(path string, entry iofs.DirEntry, err error) error { 213 213 - if strings.Count(path, "/") > 1 { 214 214 - return iofs.SkipDir 215 215 - } 216 216 - _, project, _ := strings.Cut(path, "/") 217 217 - if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 218 218 - return nil 219 219 - } 220 220 - manifests = append(manifests, path) 221 221 - return nil 222 222 - }) 223 223 - return 224 224 - } 225 225 - 226 210 func (fs *FSBackend) GetManifest( 227 211 ctx context.Context, name string, opts GetManifestOptions, 228 212 ) ( ··· 409 393 return nil 410 394 } else { 411 395 return err 396 396 + } 397 397 + } 398 398 + 399 399 + func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { 400 400 + return func(yield func(ManifestMetadata, error) bool) { 401 401 + iofs.WalkDir(fs.siteRoot.FS(), ".", 402 402 + func(path string, entry iofs.DirEntry, err error) error { 403 403 + _, project, _ := strings.Cut(path, "/") 404 404 + var metadata ManifestMetadata 405 405 + if err != nil { 406 406 + // report error 407 407 + } else if entry.IsDir() { 408 408 + // skip directory 409 409 + return nil 410 410 + } else if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 411 411 + // skip internal 412 412 + return nil 413 413 + } else if info, err := entry.Info(); err != nil { 414 414 + // report error 415 415 + } else { 416 416 + // report blob 417 417 + metadata.Name = path 418 418 + metadata.Size = info.Size() 419 419 + metadata.LastModified = info.ModTime() 420 420 + // not setting metadata.ETag since it is too costly 421 421 + } 422 422 + if !yield(metadata, err) { 423 423 + return iofs.SkipAll 424 424 + } 425 425 + return nil 426 426 + }) 412 427 } 413 428 } 414 429

+35 -28

src/backend_s3.go

··· 397 397 return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData)) 398 398 } 399 399 400 400 - func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) { 401 401 - logc.Print(ctx, "s3: list manifests") 402 402 - 403 403 - ctx, cancel := context.WithCancel(ctx) 404 404 - defer cancel() 405 405 - 406 406 - prefix := manifestObjectName("") 407 407 - for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 408 408 - Prefix: prefix, 409 409 - Recursive: true, 410 410 - }) { 411 411 - if object.Err != nil { 412 412 - return nil, object.Err 413 413 - } 414 414 - key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/") 415 415 - if strings.Count(key, "/") > 1 { 416 416 - continue 417 417 - } 418 418 - _, project, _ := strings.Cut(key, "/") 419 419 - if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 420 420 - continue 421 421 - } 422 422 - manifests = append(manifests, key) 423 423 - } 424 424 - 425 425 - return 426 426 - } 427 427 - 428 400 type s3ManifestLoader struct { 429 401 s3 *S3Backend 430 402 } ··· 666 638 minio.RemoveObjectOptions{}) 667 639 s3.siteCache.Cache.Invalidate(name) 668 640 return err 641 641 + } 642 642 + 643 643 + func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { 644 644 + return func(yield func(ManifestMetadata, error) bool) { 645 645 + logc.Print(ctx, "s3: enumerate manifests") 646 646 + 647 647 + ctx, cancel := context.WithCancel(ctx) 648 648 + defer cancel() 649 649 + 650 650 + prefix := "site/" 651 651 + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 652 652 + Prefix: prefix, 653 653 + Recursive: true, 654 654 + }) { 655 655 + var metadata ManifestMetadata 656 656 + var err error 657 657 + if err = object.Err; err == nil { 658 658 + key := strings.TrimPrefix(object.Key, prefix) 659 659 + _, project, _ := strings.Cut(key, "/") 660 660 + if strings.HasSuffix(key, "/") { 661 661 + continue // directory; skip 662 662 + } else if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 663 663 + continue // internal; skip 664 664 + } else { 665 665 + metadata.Name = key 666 666 + metadata.Size = object.Size 667 667 + metadata.LastModified = object.LastModified 668 668 + metadata.ETag = object.ETag 669 669 + } 670 670 + } 671 671 + if !yield(metadata, err) { 672 672 + break 673 673 + } 674 674 + } 675 675 + } 669 676 } 670 677 671 678 func domainCheckObjectName(domain string) string {

+16 -1

src/main.go

··· 171 171 fmt.Fprintf(os.Stderr, "(server) "+ 172 172 "git-pages [-config <file>|-no-config]\n") 173 173 fmt.Fprintf(os.Stderr, "(debug) "+ 174 174 - "git-pages {-list-blobs}\n") 174 174 + "git-pages {-list-blobs|-list-manifests}\n") 175 175 fmt.Fprintf(os.Stderr, "(debug) "+ 176 176 "git-pages {-get-blob|-get-manifest|-get-archive|-update-site} <ref> [file]\n") 177 177 fmt.Fprintf(os.Stderr, "(admin) "+ ··· 203 203 "enumerate every blob with its metadata") 204 204 getManifest := flag.String("get-manifest", "", 205 205 "write manifest for `site` (either 'domain.tld' or 'domain.tld/dir') as ProtoJSON") 206 206 + listManifests := flag.Bool("list-manifests", false, 207 207 + "enumerate every manifest with its metadata") 206 208 getArchive := flag.String("get-archive", "", 207 209 "write archive for `site` (either 'domain.tld' or 'domain.tld/dir') in tar format") 208 210 updateSite := flag.String("update-site", "", ··· 225 227 *getBlob != "", 226 228 *listBlobs, 227 229 *getManifest != "", 230 230 + *listManifests, 228 231 *getArchive != "", 229 232 *updateSite != "", 230 233 *freezeDomain != "", ··· 316 319 logc.Fatalln(ctx, err) 317 320 } 318 321 fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest))) 322 322 + 323 323 + case *listManifests: 324 324 + for metadata, err := range backend.EnumerateManifests(ctx) { 325 325 + if err != nil { 326 326 + logc.Fatalln(ctx, err) 327 327 + } 328 328 + fmt.Fprintf(color.Output, "%s %s %s\n", 329 329 + metadata.Name, 330 330 + color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)), 331 331 + color.HiGreenString(fmt.Sprint(metadata.Size)), 332 332 + ) 333 333 + } 319 334 320 335 case *getArchive != "": 321 336 webRoot := webRootArg(*getArchive)

+7 -4

src/migrate.go

··· 22 22 return nil 23 23 } 24 24 25 25 - var manifests, domains []string 26 26 - manifests, err := backend.ListManifests(ctx) 27 27 - if err != nil { 28 28 - return fmt.Errorf("list manifests: %w", err) 25 25 + var manifests []string 26 26 + for metadata, err := range backend.EnumerateManifests(ctx) { 27 27 + if err != nil { 28 28 + return fmt.Errorf("enum manifests: %w", err) 29 29 + } 30 30 + manifests = append(manifests, metadata.Name) 29 31 } 30 32 slices.Sort(manifests) 33 33 + var domains []string 31 34 for _, manifest := range manifests { 32 35 domain, _, _ := strings.Cut(manifest, "/") 33 36 if len(domains) == 0 || domains[len(domains)-1] != domain {

+12 -7

src/observe.go

··· 385 385 } 386 386 } 387 387 388 388 - func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) { 389 389 - span, ctx := ObserveFunction(ctx, "ListManifests") 390 390 - manifests, err = backend.inner.ListManifests(ctx) 391 391 - span.Finish() 392 392 - return 393 393 - } 394 394 - 395 388 func (backend *observedBackend) GetManifest( 396 389 ctx context.Context, name string, opts GetManifestOptions, 397 390 ) ( ··· 431 424 err = backend.inner.DeleteManifest(ctx, name, opts) 432 425 span.Finish() 433 426 return 427 427 + } 428 428 + 429 429 + func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { 430 430 + return func(yield func(ManifestMetadata, error) bool) { 431 431 + span, ctx := ObserveFunction(ctx, "EnumerateManifests") 432 432 + for metadata, err := range backend.inner.EnumerateManifests(ctx) { 433 433 + if !yield(metadata, err) { 434 434 + break 435 435 + } 436 436 + } 437 437 + span.Finish() 438 438 + } 434 439 } 435 440 436 441 func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) {