whitequark.org
[mirror] Scalable static site server for Git forges (like GitHub Pages)
Collect unresolved blob references in a dedicated error structure.
This will be used for incremental archive uploads.
+27
-6
src/extract.go
+27
-6
src/extract.go
···
19
19
20
20
var ErrArchiveTooLarge = errors.New("archive too large")
21
21
22
-
const BlobReferencePrefix = "/git/blobs/"
23
-
24
22
func boundArchiveStream(reader io.Reader) io.Reader {
25
23
return ReadAtMost(reader, int64(config.Limits.MaxSiteSize.Bytes()),
26
24
fmt.Errorf("%w: %s limit exceeded", ErrArchiveTooLarge, config.Limits.MaxSiteSize.HR()))
···
52
50
return next(ctx, boundArchiveStream(stream))
53
51
}
54
52
53
+
const BlobReferencePrefix = "/git/blobs/"
54
+
55
+
type UnresolvedRefError struct {
56
+
missing []string
57
+
}
58
+
59
+
func (err UnresolvedRefError) Error() string {
60
+
return fmt.Sprintf("%d unresolved blob references", len(err.missing))
61
+
}
62
+
55
63
// Returns a map of git hash to entry. If `manifest` is nil, returns an empty map.
56
64
func indexManifestByGitHash(manifest *Manifest) map[string]*Entry {
57
65
index := map[string]*Entry{}
···
68
76
}
69
77
70
78
func addSymlinkOrBlobReference(
71
-
manifest *Manifest, fileName string, target string, index map[string]*Entry,
79
+
manifest *Manifest, fileName string, target string,
80
+
index map[string]*Entry, missing *[]string,
72
81
) *Entry {
73
82
if hash, found := strings.CutPrefix(target, BlobReferencePrefix); found {
74
83
if entry, found := index[hash]; found {
75
84
manifest.Contents[fileName] = entry
76
85
return entry
77
86
} else {
78
-
AddProblem(manifest, fileName, "unresolved reference: %s", target)
87
+
*missing = append(*missing, hash)
79
88
return nil
80
89
}
81
90
} else {
···
90
99
var dataBytesTransferred int64
91
100
92
101
index := indexManifestByGitHash(oldManifest)
102
+
missing := []string{}
93
103
manifest := NewManifest()
94
104
for {
95
105
header, err := archive.Next()
···
119
129
AddFile(manifest, fileName, fileData)
120
130
dataBytesTransferred += int64(len(fileData))
121
131
case tar.TypeSymlink:
122
-
entry := addSymlinkOrBlobReference(manifest, fileName, header.Linkname, index)
132
+
entry := addSymlinkOrBlobReference(
133
+
manifest, fileName, header.Linkname, index, &missing)
123
134
dataBytesRecycled += entry.GetOriginalSize()
124
135
case tar.TypeDir:
125
136
AddDirectory(manifest, fileName)
···
127
138
AddProblem(manifest, fileName, "tar: unsupported type '%c'", header.Typeflag)
128
139
continue
129
140
}
141
+
}
142
+
143
+
if len(missing) > 0 {
144
+
return nil, UnresolvedRefError{missing}
130
145
}
131
146
132
147
logc.Printf(ctx,
···
166
181
var dataBytesTransferred int64
167
182
168
183
index := indexManifestByGitHash(oldManifest)
184
+
missing := []string{}
169
185
manifest := NewManifest()
170
186
for _, file := range archive.File {
171
187
if strings.HasSuffix(file.Name, "/") {
···
183
199
}
184
200
185
201
if file.Mode()&os.ModeSymlink != 0 {
186
-
entry := addSymlinkOrBlobReference(manifest, file.Name, string(fileData), index)
202
+
entry := addSymlinkOrBlobReference(
203
+
manifest, file.Name, string(fileData), index, &missing)
187
204
dataBytesRecycled += entry.GetOriginalSize()
188
205
} else {
189
206
AddFile(manifest, file.Name, fileData)
190
207
dataBytesTransferred += int64(len(fileData))
191
208
}
192
209
}
210
+
}
211
+
212
+
if len(missing) > 0 {
213
+
return nil, UnresolvedRefError{missing}
193
214
}
194
215
195
216
logc.Printf(ctx,
+3
src/pages.go
+3
src/pages.go
···
572
572
func reportUpdateResult(w http.ResponseWriter, result UpdateResult) error {
573
573
switch result.outcome {
574
574
case UpdateError:
575
+
var unresolvedRefErr UnresolvedRefError
575
576
if errors.Is(result.err, ErrManifestTooLarge) {
576
577
w.WriteHeader(http.StatusRequestEntityTooLarge)
577
578
} else if errors.Is(result.err, errArchiveFormat) {
···
586
587
w.WriteHeader(http.StatusConflict)
587
588
} else if errors.Is(result.err, ErrDomainFrozen) {
588
589
w.WriteHeader(http.StatusForbidden)
590
+
} else if errors.As(result.err, &unresolvedRefErr) {
591
+
w.WriteHeader(http.StatusUnprocessableEntity)
589
592
} else {
590
593
w.WriteHeader(http.StatusServiceUnavailable)
591
594
}