whitequark.org
[mirror] Command-line application for uploading a site to a git-pages server
1package main
2
3import (
4 "archive/tar"
5 "bufio"
6 "bytes"
7 "crypto"
8 "crypto/sha256"
9 "encoding/hex"
10 "errors"
11 "fmt"
12 "io"
13 "io/fs"
14 "net/http"
15 "net/url"
16 "os"
17 "regexp"
18 "runtime/debug"
19 "strconv"
20 "strings"
21
22 "github.com/google/uuid"
23 "github.com/klauspost/compress/zstd"
24 "github.com/spf13/pflag"
25)
26
27// By default the version information is retrieved from VCS. If not available during build,
28// override this variable using linker flags to change the displayed version.
29// Example: `-ldflags "-X main.versionOverride=v1.2.3"`
30var versionOverride = ""
31
32func versionInfo() string {
33 version := "(unknown)"
34 if versionOverride != "" {
35 version = versionOverride
36 } else if buildInfo, ok := debug.ReadBuildInfo(); ok {
37 version = buildInfo.Main.Version
38 }
39 return fmt.Sprintf("git-pages-cli %s", version)
40}
41
42var passwordFlag = pflag.String("password", "", "password for DNS challenge authorization")
43var passwordFileFlag = pflag.String("password-file", "", "file with password for DNS challenge authorization")
44var tokenFlag = pflag.String("token", "", "token for forge authorization")
45var challengeFlag = pflag.Bool("challenge", false, "compute DNS challenge entry from password (output zone file record)")
46var challengeBareFlag = pflag.Bool("challenge-bare", false, "compute DNS challenge entry from password (output bare TXT value)")
47var uploadGitFlag = pflag.String("upload-git", "", "replace site with contents of specified git repository")
48var uploadDirFlag = pflag.String("upload-dir", "", "replace whole site or a path with contents of specified directory")
49var deleteFlag = pflag.Bool("delete", false, "delete whole site or a path")
50var debugManifestFlag = pflag.Bool("debug-manifest", false, "retrieve site manifest as ProtoJSON, for debugging")
51var serverFlag = pflag.String("server", "", "hostname of server to connect to")
52var pathFlag = pflag.String("path", "", "partially update site at specified path")
53var parentsFlag = pflag.Bool("parents", false, "create parent directories of --path")
54var atomicFlag = pflag.Bool("atomic", false, "require partial updates to be atomic")
55var incrementalFlag = pflag.Bool("incremental", false, "make --upload-dir only upload changed files")
56var verboseFlag = pflag.BoolP("verbose", "v", false, "display more information for debugging")
57var versionFlag = pflag.BoolP("version", "V", false, "display version information")
58
59func singleOperation() bool {
60 operations := 0
61 if *challengeFlag {
62 operations++
63 }
64 if *challengeBareFlag {
65 operations++
66 }
67 if *uploadDirFlag != "" {
68 operations++
69 }
70 if *uploadGitFlag != "" {
71 operations++
72 }
73 if *deleteFlag {
74 operations++
75 }
76 if *debugManifestFlag {
77 operations++
78 }
79 if *versionFlag {
80 operations++
81 }
82 return operations == 1
83}
84
85func gitBlobSHA256(data []byte) string {
86 h := crypto.SHA256.New()
87 h.Write([]byte("blob "))
88 h.Write([]byte(strconv.FormatInt(int64(len(data)), 10)))
89 h.Write([]byte{0})
90 h.Write(data)
91 return hex.EncodeToString(h.Sum(nil))
92}
93
94func displayFS(root fs.FS, prefix string) error {
95 return fs.WalkDir(root, ".", func(name string, entry fs.DirEntry, err error) error {
96 if err != nil {
97 return err
98 }
99 switch {
100 case entry.Type().IsDir():
101 fmt.Fprintf(os.Stderr, "dir %s%s\n", prefix, name)
102 case entry.Type().IsRegular():
103 fmt.Fprintf(os.Stderr, "file %s%s\n", prefix, name)
104 case entry.Type() == fs.ModeSymlink:
105 fmt.Fprintf(os.Stderr, "symlink %s%s\n", prefix, name)
106 default:
107 fmt.Fprintf(os.Stderr, "other %s%s\n", prefix, name)
108 }
109 return nil
110 })
111}
112
113// It doesn't make sense to use incremental updates for very small files since the cost of
114// repeating a request to fill in a missing blob is likely to be higher than any savings gained.
115const incrementalSizeThreshold = 256
116
117func archiveFS(writer io.Writer, root fs.FS, prefix string, needBlobs []string) (err error) {
118 requestedSet := make(map[string]struct{})
119 for _, hash := range needBlobs {
120 requestedSet[hash] = struct{}{}
121 }
122 zstdWriter, _ := zstd.NewWriter(writer)
123 tarWriter := tar.NewWriter(zstdWriter)
124 if err = fs.WalkDir(root, ".", func(name string, entry fs.DirEntry, err error) error {
125 if err != nil {
126 return err
127 }
128 header := &tar.Header{}
129 data := []byte{}
130 if prefix == "" && name == "." {
131 return nil
132 } else if name == "." {
133 header.Name = prefix
134 } else {
135 header.Name = prefix + name
136 }
137 switch {
138 case entry.Type().IsDir():
139 header.Typeflag = tar.TypeDir
140 header.Name += "/"
141 case entry.Type().IsRegular():
142 header.Typeflag = tar.TypeReg
143 if data, err = fs.ReadFile(root, name); err != nil {
144 return err
145 }
146 if *incrementalFlag && len(data) > incrementalSizeThreshold {
147 hash := gitBlobSHA256(data)
148 if _, requested := requestedSet[hash]; !requested {
149 header.Typeflag = tar.TypeSymlink
150 header.Linkname = "/git/blobs/" + hash
151 data = nil
152 }
153 }
154 case entry.Type() == fs.ModeSymlink:
155 header.Typeflag = tar.TypeSymlink
156 if header.Linkname, err = fs.ReadLink(root, name); err != nil {
157 return err
158 }
159 default:
160 return errors.New("tar: cannot add non-regular file")
161 }
162 header.Size = int64(len(data))
163 if err = tarWriter.WriteHeader(header); err != nil {
164 return err
165 }
166 if _, err = tarWriter.Write(data); err != nil {
167 return err
168 }
169 return err
170 }); err != nil {
171 return
172 }
173 if err = tarWriter.Close(); err != nil {
174 return
175 }
176 if err = zstdWriter.Close(); err != nil {
177 return
178 }
179 return
180}
181
182// Stream archive data without ever loading the entire working set into RAM.
183func streamArchiveFS(root fs.FS, prefix string, needBlobs []string) io.ReadCloser {
184 reader, writer := io.Pipe()
185 go func() {
186 err := archiveFS(writer, root, prefix, needBlobs)
187 if err != nil {
188 writer.CloseWithError(err)
189 } else {
190 writer.Close()
191 }
192 }()
193 return reader
194}
195
196func makeWhiteout(path string) (reader io.Reader) {
197 buffer := &bytes.Buffer{}
198 tarWriter := tar.NewWriter(buffer)
199 tarWriter.WriteHeader(&tar.Header{
200 Typeflag: tar.TypeChar,
201 Name: path,
202 })
203 tarWriter.Flush()
204 return buffer
205}
206
207const usageExitCode = 125
208
209func usage() {
210 fmt.Fprintf(os.Stderr,
211 "Usage: %s <site-url> {--challenge|--upload-git url|--upload-dir path|--delete} [options...]\n",
212 os.Args[0],
213 )
214 pflag.PrintDefaults()
215}
216
217func main() {
218 pflag.Usage = usage
219 pflag.Parse()
220 if !singleOperation() || (!*versionFlag && len(pflag.Args()) != 1) {
221 pflag.Usage()
222 os.Exit(usageExitCode)
223 }
224
225 if *versionFlag {
226 fmt.Fprintln(os.Stdout, versionInfo())
227 os.Exit(0)
228 }
229
230 if *passwordFlag != "" && *passwordFileFlag != "" {
231 fmt.Fprintf(os.Stderr, "--password and --password-file are mutually exclusive")
232 os.Exit(usageExitCode)
233 }
234
235 if *passwordFlag != "" && *tokenFlag != "" {
236 fmt.Fprintf(os.Stderr, "--password and --token are mutually exclusive")
237 os.Exit(usageExitCode)
238 }
239
240 if *passwordFileFlag != "" {
241 contents, err := os.ReadFile(*passwordFileFlag)
242 if err != nil {
243 fmt.Fprintf(os.Stderr, "error: invalid password file: %s\n", err)
244 os.Exit(1)
245 }
246 // Trim all trailing newlines; there's no legitimate reason to have one in a password.
247 *passwordFlag = strings.TrimRight(string(contents), "\n")
248 }
249
250 var pathPrefix string
251 if *pathFlag != "" {
252 if *uploadDirFlag == "" && !*deleteFlag {
253 fmt.Fprintf(os.Stderr, "--path requires --upload-dir or --delete")
254 os.Exit(usageExitCode)
255 } else {
256 pathPrefix = strings.Trim(*pathFlag, "/") + "/"
257 }
258 }
259
260 var err error
261 siteURL, err := url.Parse(pflag.Args()[0])
262 if err != nil {
263 fmt.Fprintf(os.Stderr, "error: invalid site URL: %s\n", err)
264 os.Exit(1)
265 }
266
267 var request *http.Request
268 var uploadDir *os.Root
269 switch {
270 case *challengeFlag || *challengeBareFlag:
271 if *passwordFlag == "" {
272 *passwordFlag = uuid.NewString()
273 fmt.Fprintf(os.Stderr, "password: %s\n", *passwordFlag)
274 }
275
276 challenge := sha256.Sum256(fmt.Appendf(nil, "%s %s", siteURL.Hostname(), *passwordFlag))
277 if *challengeBareFlag {
278 fmt.Fprintf(os.Stdout, "%x\n", challenge)
279 } else {
280 fmt.Fprintf(os.Stdout, "_git-pages-challenge.%s. 3600 IN TXT \"%x\"\n", siteURL.Hostname(), challenge)
281 }
282 os.Exit(0)
283
284 case *uploadGitFlag != "":
285 uploadGitUrl, err := url.Parse(*uploadGitFlag)
286 if err != nil {
287 fmt.Fprintf(os.Stderr, "error: invalid repository URL: %s\n", err)
288 os.Exit(1)
289 }
290
291 requestBody := []byte(uploadGitUrl.String())
292 request, err = http.NewRequest("PUT", siteURL.String(), bytes.NewReader(requestBody))
293 if err != nil {
294 fmt.Fprintf(os.Stderr, "error: %s\n", err)
295 os.Exit(1)
296 }
297 request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
298
299 case *uploadDirFlag != "":
300 uploadDir, err = os.OpenRoot(*uploadDirFlag)
301 if err != nil {
302 fmt.Fprintf(os.Stderr, "error: invalid directory: %s\n", err)
303 os.Exit(1)
304 }
305
306 if *verboseFlag {
307 err := displayFS(uploadDir.FS(), pathPrefix)
308 if err != nil {
309 fmt.Fprintf(os.Stderr, "error: %s\n", err)
310 os.Exit(1)
311 }
312 }
313
314 requestBody := streamArchiveFS(uploadDir.FS(), pathPrefix, []string{})
315 if *pathFlag == "" {
316 request, err = http.NewRequest("PUT", siteURL.String(), requestBody)
317 } else {
318 request, err = http.NewRequest("PATCH", siteURL.String(), requestBody)
319 }
320 if err != nil {
321 fmt.Fprintf(os.Stderr, "error: %s\n", err)
322 os.Exit(1)
323 }
324 request.ContentLength = -1
325 request.Header.Add("Content-Type", "application/x-tar+zstd")
326 request.Header.Add("Accept", "application/vnd.git-pages.unresolved;q=1.0, text/plain;q=0.9")
327 if *parentsFlag {
328 request.Header.Add("Create-Parents", "yes")
329 } else {
330 request.Header.Add("Create-Parents", "no")
331 }
332
333 case *deleteFlag:
334 if *pathFlag == "" {
335 request, err = http.NewRequest("DELETE", siteURL.String(), nil)
336 if err != nil {
337 fmt.Fprintf(os.Stderr, "error: %s\n", err)
338 os.Exit(1)
339 }
340 } else {
341 request, err = http.NewRequest("PATCH", siteURL.String(), makeWhiteout(pathPrefix))
342 if err != nil {
343 fmt.Fprintf(os.Stderr, "error: %s\n", err)
344 os.Exit(1)
345 }
346 request.Header.Add("Content-Type", "application/x-tar")
347 }
348
349 case *debugManifestFlag:
350 manifestURL := siteURL.JoinPath(".git-pages/manifest.json")
351 request, err = http.NewRequest("GET", manifestURL.String(), nil)
352 if err != nil {
353 fmt.Fprintf(os.Stderr, "error: %s\n", err)
354 os.Exit(1)
355 }
356
357 default:
358 panic("no operation chosen")
359 }
360 request.Header.Add("User-Agent", versionInfo())
361 if request.Method == "PATCH" {
362 if *atomicFlag {
363 request.Header.Add("Atomic", "yes")
364 request.Header.Add("Race-Free", "yes") // deprecated name, to be removed soon
365 } else {
366 request.Header.Add("Atomic", "no")
367 request.Header.Add("Race-Free", "no") // deprecated name, to be removed soon
368 }
369 }
370 makeAuthorization := func(headerName string, kind string, value string) {
371 if strings.ContainsAny(value, "\r\n") {
372 fmt.Fprintf(os.Stderr, "error: invalid characters in %s header value: %q\n",
373 headerName, value)
374 os.Exit(1)
375 }
376 request.Header.Add(headerName, fmt.Sprintf("%s %s", kind, value))
377 }
378 switch {
379 case *passwordFlag != "":
380 makeAuthorization("Authorization", "Pages", *passwordFlag)
381 case *tokenFlag != "":
382 makeAuthorization("Forge-Authorization", "token", *tokenFlag)
383 }
384 if *serverFlag != "" {
385 // Send the request to `--server` host, but set the `Host:` header to the site host.
386 // This allows first-time publishing to proceed without the git-pages server yet having
387 // a TLS certificate for the site host (which has a circular dependency on completion of
388 // first-time publishing).
389 newURL := *request.URL
390 newURL.Host = *serverFlag
391 request.URL = &newURL
392 request.Header.Set("Host", siteURL.Host)
393 }
394
395 // HTTP-to-HTTPS redirects are often implemented with 301 or 302 response codes which instruct
396 // the Go HTTP client to disregard the original request method and use GET instead. To prevent
397 // confusion, detect such redirects and preserve the original request method.
398 http.DefaultClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
399 last := via[len(via)-1]
400 newURL := req.URL.JoinPath()
401 lastURL := last.URL.JoinPath()
402 if lastURL.Scheme == "http" && newURL.Scheme == "https" {
403 lastURL.Scheme = newURL.Scheme
404 if lastURL.String() == newURL.String() {
405 req.Method = last.Method
406 return nil
407 }
408 }
409 if req.Method != last.Method {
410 return fmt.Errorf("unexpected redirect")
411 }
412 return nil
413 }
414
415 displayServer := *verboseFlag
416 for {
417 response, err := http.DefaultClient.Do(request)
418 if err != nil {
419 if response.StatusCode == 301 || response.StatusCode == 302 {
420 if innerErr := errors.Unwrap(err); innerErr != nil {
421 // If the error is due to a redirect, that means it wasn't followed, but the
422 // original url.Error confusingly reports the new URL anyway.
423 err = innerErr
424 }
425 }
426 fmt.Fprintf(os.Stderr, "error: %s\n", err)
427 os.Exit(1)
428 }
429 serverIdent := strings.Join(response.Header.Values("Server"), ", ")
430 //lint:ignore SA6000 This isn't a hot loop.
431 if matched, err := regexp.MatchString(`\bgit-pages\b`, serverIdent); err != nil {
432 panic(err)
433 } else if !matched {
434 fmt.Fprintf(os.Stderr,
435 "error: the tool only works with git-pages, but the URL points to a %q server\n",
436 serverIdent)
437 os.Exit(1)
438 }
439 if displayServer {
440 fmt.Fprintf(os.Stderr, "server: %s\n", response.Header.Get("Server"))
441 displayServer = false
442 }
443 if *debugManifestFlag {
444 if response.StatusCode == http.StatusOK {
445 io.Copy(os.Stdout, response.Body)
446 fmt.Fprintf(os.Stdout, "\n")
447 } else {
448 io.Copy(os.Stderr, response.Body)
449 os.Exit(1)
450 }
451 } else { // an update operation
452 if *verboseFlag {
453 fmt.Fprintf(os.Stderr, "response: %d %s\n",
454 response.StatusCode, response.Header.Get("Content-Type"))
455 }
456 if response.StatusCode == http.StatusUnprocessableEntity &&
457 response.Header.Get("Content-Type") == "application/vnd.git-pages.unresolved" {
458 needBlobs := []string{}
459 scanner := bufio.NewScanner(response.Body)
460 for scanner.Scan() {
461 needBlobs = append(needBlobs, scanner.Text())
462 }
463 response.Body.Close()
464 if *verboseFlag {
465 fmt.Fprintf(os.Stderr, "incremental: need %d blobs\n", len(needBlobs))
466 }
467 request.Body = streamArchiveFS(uploadDir.FS(), pathPrefix, needBlobs)
468 continue // resubmit
469 } else if response.StatusCode == http.StatusOK {
470 fmt.Fprintf(os.Stdout, "result: %s\n", response.Header.Get("Update-Result"))
471 io.Copy(os.Stdout, response.Body)
472 } else {
473 fmt.Fprintf(os.Stderr, "result: error\n")
474 io.Copy(os.Stderr, response.Body)
475 os.Exit(1)
476 }
477 }
478 break
479 }
480}