whitequark.org
[mirror] Command-line application for uploading a site to a git-pages server
1package main
2
3import (
4 "archive/tar"
5 "bufio"
6 "bytes"
7 "crypto"
8 "crypto/sha256"
9 "encoding/hex"
10 "errors"
11 "fmt"
12 "io"
13 "io/fs"
14 "net/http"
15 "net/url"
16 "os"
17 "regexp"
18 "runtime/debug"
19 "strconv"
20 "strings"
21
22 "github.com/google/uuid"
23 "github.com/klauspost/compress/zstd"
24 "github.com/spf13/pflag"
25)
26
27// By default the version information is retrieved from VCS. If not available during build,
28// override this variable using linker flags to change the displayed version.
29// Example: `-ldflags "-X main.versionOverride=v1.2.3"`
30var versionOverride = ""
31
32func versionInfo() string {
33 version := "(unknown)"
34 if versionOverride != "" {
35 version = versionOverride
36 } else if buildInfo, ok := debug.ReadBuildInfo(); ok {
37 version = buildInfo.Main.Version
38 }
39 return fmt.Sprintf("git-pages-cli %s", version)
40}
41
42var passwordFlag = pflag.String("password", "", "password for DNS challenge authorization")
43var tokenFlag = pflag.String("token", "", "token for forge authorization")
44var challengeFlag = pflag.Bool("challenge", false, "compute DNS challenge entry from password (output zone file record)")
45var challengeBareFlag = pflag.Bool("challenge-bare", false, "compute DNS challenge entry from password (output bare TXT value)")
46var uploadGitFlag = pflag.String("upload-git", "", "replace site with contents of specified git repository")
47var uploadDirFlag = pflag.String("upload-dir", "", "replace whole site or a path with contents of specified directory")
48var deleteFlag = pflag.Bool("delete", false, "delete whole site or a path")
49var debugManifestFlag = pflag.Bool("debug-manifest", false, "retrieve site manifest as ProtoJSON, for debugging")
50var serverFlag = pflag.String("server", "", "hostname of server to connect to")
51var pathFlag = pflag.String("path", "", "partially update site at specified path")
52var parentsFlag = pflag.Bool("parents", false, "create parent directories of --path")
53var atomicFlag = pflag.Bool("atomic", false, "require partial updates to be atomic")
54var incrementalFlag = pflag.Bool("incremental", false, "make --upload-dir only upload changed files")
55var verboseFlag = pflag.BoolP("verbose", "v", false, "display more information for debugging")
56var versionFlag = pflag.BoolP("version", "V", false, "display version information")
57
58func singleOperation() bool {
59 operations := 0
60 if *challengeFlag {
61 operations++
62 }
63 if *challengeBareFlag {
64 operations++
65 }
66 if *uploadDirFlag != "" {
67 operations++
68 }
69 if *uploadGitFlag != "" {
70 operations++
71 }
72 if *deleteFlag {
73 operations++
74 }
75 if *debugManifestFlag {
76 operations++
77 }
78 if *versionFlag {
79 operations++
80 }
81 return operations == 1
82}
83
84func gitBlobSHA256(data []byte) string {
85 h := crypto.SHA256.New()
86 h.Write([]byte("blob "))
87 h.Write([]byte(strconv.FormatInt(int64(len(data)), 10)))
88 h.Write([]byte{0})
89 h.Write(data)
90 return hex.EncodeToString(h.Sum(nil))
91}
92
93func displayFS(root fs.FS, prefix string) error {
94 return fs.WalkDir(root, ".", func(name string, entry fs.DirEntry, err error) error {
95 if err != nil {
96 return err
97 }
98 switch {
99 case entry.Type().IsDir():
100 fmt.Fprintf(os.Stderr, "dir %s%s\n", prefix, name)
101 case entry.Type().IsRegular():
102 fmt.Fprintf(os.Stderr, "file %s%s\n", prefix, name)
103 case entry.Type() == fs.ModeSymlink:
104 fmt.Fprintf(os.Stderr, "symlink %s%s\n", prefix, name)
105 default:
106 fmt.Fprintf(os.Stderr, "other %s%s\n", prefix, name)
107 }
108 return nil
109 })
110}
111
112// It doesn't make sense to use incremental updates for very small files since the cost of
113// repeating a request to fill in a missing blob is likely to be higher than any savings gained.
114const incrementalSizeThreshold = 256
115
116func archiveFS(writer io.Writer, root fs.FS, prefix string, needBlobs []string) (err error) {
117 requestedSet := make(map[string]struct{})
118 for _, hash := range needBlobs {
119 requestedSet[hash] = struct{}{}
120 }
121 zstdWriter, _ := zstd.NewWriter(writer)
122 tarWriter := tar.NewWriter(zstdWriter)
123 if err = fs.WalkDir(root, ".", func(name string, entry fs.DirEntry, err error) error {
124 if err != nil {
125 return err
126 }
127 header := &tar.Header{}
128 data := []byte{}
129 if prefix == "" && name == "." {
130 return nil
131 } else if name == "." {
132 header.Name = prefix
133 } else {
134 header.Name = prefix + name
135 }
136 switch {
137 case entry.Type().IsDir():
138 header.Typeflag = tar.TypeDir
139 header.Name += "/"
140 case entry.Type().IsRegular():
141 header.Typeflag = tar.TypeReg
142 if data, err = fs.ReadFile(root, name); err != nil {
143 return err
144 }
145 if *incrementalFlag && len(data) > incrementalSizeThreshold {
146 hash := gitBlobSHA256(data)
147 if _, requested := requestedSet[hash]; !requested {
148 header.Typeflag = tar.TypeSymlink
149 header.Linkname = "/git/blobs/" + hash
150 data = nil
151 }
152 }
153 case entry.Type() == fs.ModeSymlink:
154 header.Typeflag = tar.TypeSymlink
155 if header.Linkname, err = fs.ReadLink(root, name); err != nil {
156 return err
157 }
158 default:
159 return errors.New("tar: cannot add non-regular file")
160 }
161 header.Size = int64(len(data))
162 if err = tarWriter.WriteHeader(header); err != nil {
163 return err
164 }
165 if _, err = tarWriter.Write(data); err != nil {
166 return err
167 }
168 return err
169 }); err != nil {
170 return
171 }
172 if err = tarWriter.Close(); err != nil {
173 return
174 }
175 if err = zstdWriter.Close(); err != nil {
176 return
177 }
178 return
179}
180
181// Stream archive data without ever loading the entire working set into RAM.
182func streamArchiveFS(root fs.FS, prefix string, needBlobs []string) io.ReadCloser {
183 reader, writer := io.Pipe()
184 go func() {
185 err := archiveFS(writer, root, prefix, needBlobs)
186 if err != nil {
187 writer.CloseWithError(err)
188 } else {
189 writer.Close()
190 }
191 }()
192 return reader
193}
194
195func makeWhiteout(path string) (reader io.Reader) {
196 buffer := &bytes.Buffer{}
197 tarWriter := tar.NewWriter(buffer)
198 tarWriter.WriteHeader(&tar.Header{
199 Typeflag: tar.TypeChar,
200 Name: path,
201 })
202 tarWriter.Flush()
203 return buffer
204}
205
206const usageExitCode = 125
207
208func usage() {
209 fmt.Fprintf(os.Stderr,
210 "Usage: %s <site-url> {--challenge|--upload-git url|--upload-dir path|--delete} [options...]\n",
211 os.Args[0],
212 )
213 pflag.PrintDefaults()
214}
215
216func main() {
217 pflag.Usage = usage
218 pflag.Parse()
219 if !singleOperation() || (!*versionFlag && len(pflag.Args()) != 1) {
220 pflag.Usage()
221 os.Exit(usageExitCode)
222 }
223
224 if *versionFlag {
225 fmt.Fprintln(os.Stdout, versionInfo())
226 os.Exit(0)
227 }
228
229 if *passwordFlag != "" && *tokenFlag != "" {
230 fmt.Fprintf(os.Stderr, "--password and --token are mutually exclusive")
231 os.Exit(usageExitCode)
232 }
233
234 var pathPrefix string
235 if *pathFlag != "" {
236 if *uploadDirFlag == "" && !*deleteFlag {
237 fmt.Fprintf(os.Stderr, "--path requires --upload-dir or --delete")
238 os.Exit(usageExitCode)
239 } else {
240 pathPrefix = strings.Trim(*pathFlag, "/") + "/"
241 }
242 }
243
244 var err error
245 siteURL, err := url.Parse(pflag.Args()[0])
246 if err != nil {
247 fmt.Fprintf(os.Stderr, "error: invalid site URL: %s\n", err)
248 os.Exit(1)
249 }
250
251 var request *http.Request
252 var uploadDir *os.Root
253 switch {
254 case *challengeFlag || *challengeBareFlag:
255 if *passwordFlag == "" {
256 *passwordFlag = uuid.NewString()
257 fmt.Fprintf(os.Stderr, "password: %s\n", *passwordFlag)
258 }
259
260 challenge := sha256.Sum256(fmt.Appendf(nil, "%s %s", siteURL.Hostname(), *passwordFlag))
261 if *challengeBareFlag {
262 fmt.Fprintf(os.Stdout, "%x\n", challenge)
263 } else {
264 fmt.Fprintf(os.Stdout, "_git-pages-challenge.%s. 3600 IN TXT \"%x\"\n", siteURL.Hostname(), challenge)
265 }
266 os.Exit(0)
267
268 case *uploadGitFlag != "":
269 uploadGitUrl, err := url.Parse(*uploadGitFlag)
270 if err != nil {
271 fmt.Fprintf(os.Stderr, "error: invalid repository URL: %s\n", err)
272 os.Exit(1)
273 }
274
275 requestBody := []byte(uploadGitUrl.String())
276 request, err = http.NewRequest("PUT", siteURL.String(), bytes.NewReader(requestBody))
277 if err != nil {
278 fmt.Fprintf(os.Stderr, "error: %s\n", err)
279 os.Exit(1)
280 }
281 request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
282
283 case *uploadDirFlag != "":
284 uploadDir, err = os.OpenRoot(*uploadDirFlag)
285 if err != nil {
286 fmt.Fprintf(os.Stderr, "error: invalid directory: %s\n", err)
287 os.Exit(1)
288 }
289
290 if *verboseFlag {
291 err := displayFS(uploadDir.FS(), pathPrefix)
292 if err != nil {
293 fmt.Fprintf(os.Stderr, "error: %s\n", err)
294 os.Exit(1)
295 }
296 }
297
298 if *pathFlag == "" {
299 request, err = http.NewRequest("PUT", siteURL.String(), nil)
300 } else {
301 request, err = http.NewRequest("PATCH", siteURL.String(), nil)
302 }
303 if err != nil {
304 fmt.Fprintf(os.Stderr, "error: %s\n", err)
305 os.Exit(1)
306 }
307 request.Body = streamArchiveFS(uploadDir.FS(), pathPrefix, []string{})
308 request.ContentLength = -1
309 request.Header.Add("Content-Type", "application/x-tar+zstd")
310 request.Header.Add("Accept", "application/vnd.git-pages.unresolved;q=1.0, text/plain;q=0.9")
311 if *parentsFlag {
312 request.Header.Add("Create-Parents", "yes")
313 } else {
314 request.Header.Add("Create-Parents", "no")
315 }
316
317 case *deleteFlag:
318 if *pathFlag == "" {
319 request, err = http.NewRequest("DELETE", siteURL.String(), nil)
320 if err != nil {
321 fmt.Fprintf(os.Stderr, "error: %s\n", err)
322 os.Exit(1)
323 }
324 } else {
325 request, err = http.NewRequest("PATCH", siteURL.String(), makeWhiteout(pathPrefix))
326 if err != nil {
327 fmt.Fprintf(os.Stderr, "error: %s\n", err)
328 os.Exit(1)
329 }
330 request.Header.Add("Content-Type", "application/x-tar")
331 }
332
333 case *debugManifestFlag:
334 manifestURL := siteURL.JoinPath(".git-pages/manifest.json")
335 request, err = http.NewRequest("GET", manifestURL.String(), nil)
336 if err != nil {
337 fmt.Fprintf(os.Stderr, "error: %s\n", err)
338 os.Exit(1)
339 }
340
341 default:
342 panic("no operation chosen")
343 }
344 request.Header.Add("User-Agent", versionInfo())
345 if request.Method == "PATCH" {
346 if *atomicFlag {
347 request.Header.Add("Atomic", "yes")
348 request.Header.Add("Race-Free", "yes") // deprecated name, to be removed soon
349 } else {
350 request.Header.Add("Atomic", "no")
351 request.Header.Add("Race-Free", "no") // deprecated name, to be removed soon
352 }
353 }
354 switch {
355 case *passwordFlag != "":
356 request.Header.Add("Authorization", fmt.Sprintf("Pages %s", *passwordFlag))
357 case *tokenFlag != "":
358 request.Header.Add("Forge-Authorization", fmt.Sprintf("token %s", *tokenFlag))
359 }
360 if *serverFlag != "" {
361 // Send the request to `--server` host, but set the `Host:` header to the site host.
362 // This allows first-time publishing to proceed without the git-pages server yet having
363 // a TLS certificate for the site host (which has a circular dependency on completion of
364 // first-time publishing).
365 newURL := *request.URL
366 newURL.Host = *serverFlag
367 request.URL = &newURL
368 request.Header.Set("Host", siteURL.Host)
369 }
370
371 displayServer := *verboseFlag
372 for {
373 response, err := http.DefaultClient.Do(request)
374 if err != nil {
375 fmt.Fprintf(os.Stderr, "error: %s\n", err)
376 os.Exit(1)
377 }
378 serverIdent := strings.Join(response.Header.Values("Server"), ", ")
379 //lint:ignore SA6000 This isn't a hot loop.
380 if matched, err := regexp.MatchString(`\bgit-pages\b`, serverIdent); err != nil {
381 panic(err)
382 } else if !matched {
383 fmt.Fprintf(os.Stderr,
384 "error: the tool only works with git-pages, but the URL points to a %q server\n",
385 serverIdent)
386 os.Exit(1)
387 }
388 if displayServer {
389 fmt.Fprintf(os.Stderr, "server: %s\n", response.Header.Get("Server"))
390 displayServer = false
391 }
392 if *debugManifestFlag {
393 if response.StatusCode == http.StatusOK {
394 io.Copy(os.Stdout, response.Body)
395 fmt.Fprintf(os.Stdout, "\n")
396 } else {
397 io.Copy(os.Stderr, response.Body)
398 os.Exit(1)
399 }
400 } else { // an update operation
401 if *verboseFlag {
402 fmt.Fprintf(os.Stderr, "response: %d %s\n",
403 response.StatusCode, response.Header.Get("Content-Type"))
404 }
405 if response.StatusCode == http.StatusUnprocessableEntity &&
406 response.Header.Get("Content-Type") == "application/vnd.git-pages.unresolved" {
407 needBlobs := []string{}
408 scanner := bufio.NewScanner(response.Body)
409 for scanner.Scan() {
410 needBlobs = append(needBlobs, scanner.Text())
411 }
412 response.Body.Close()
413 if *verboseFlag {
414 fmt.Fprintf(os.Stderr, "incremental: need %d blobs\n", len(needBlobs))
415 }
416 request.Body = streamArchiveFS(uploadDir.FS(), pathPrefix, needBlobs)
417 continue // resubmit
418 } else if response.StatusCode == http.StatusOK {
419 fmt.Fprintf(os.Stdout, "result: %s\n", response.Header.Get("Update-Result"))
420 io.Copy(os.Stdout, response.Body)
421 } else {
422 fmt.Fprintf(os.Stderr, "result: error\n")
423 io.Copy(os.Stderr, response.Body)
424 os.Exit(1)
425 }
426 }
427 break
428 }
429}