[mirror] Command-line application for uploading a site to a git-pages server

Implement partial update support.

This is currently feature-gated in git-pages, and so may fail when
used against servers not configured with this feature.

+110 -21
+110 -21
main.go
··· 4 4 "archive/tar" 5 5 "bytes" 6 6 "crypto/sha256" 7 + "errors" 7 8 "fmt" 8 9 "io" 9 10 "io/fs" ··· 11 12 "net/url" 12 13 "os" 13 14 "runtime/debug" 15 + "strings" 14 16 15 17 "github.com/google/uuid" 16 18 "github.com/klauspost/compress/zstd" ··· 41 43 var deleteFlag = pflag.Bool("delete", false, "delete site") 42 44 var debugManifestFlag = pflag.Bool("debug-manifest", false, "retrieve site manifest as ProtoJSON, for debugging") 43 45 var serverFlag = pflag.String("server", "", "hostname of server to connect to") 46 + var pathFlag = pflag.String("path", "", "partially update site at specified path") 47 + var raceFreeFlag = pflag.Bool("race-free", false, "require partial updates to be atomic") 44 48 var verboseFlag = pflag.Bool("verbose", false, "display more information for debugging") 45 49 var versionFlag = pflag.Bool("version", false, "display version information") 46 50 ··· 70 74 return operations == 1 71 75 } 72 76 73 - func displayFS(root fs.FS) error { 77 + func displayFS(root fs.FS, prefix string) error { 74 78 return fs.WalkDir(root, ".", func(name string, entry fs.DirEntry, err error) error { 75 79 if err != nil { 76 80 return err 77 81 } 78 82 switch { 79 - case entry.Type() == 0: 80 - fmt.Fprintln(os.Stderr, "file", name) 81 - case entry.Type() == fs.ModeDir: 82 - fmt.Fprintln(os.Stderr, "dir", name) 83 + case entry.Type().IsDir(): 84 + fmt.Fprintf(os.Stderr, "dir %s%s\n", prefix, name) 85 + case entry.Type().IsRegular(): 86 + fmt.Fprintf(os.Stderr, "file %s%s\n", prefix, name) 83 87 case entry.Type() == fs.ModeSymlink: 84 - fmt.Fprintln(os.Stderr, "symlink", name) 88 + fmt.Fprintf(os.Stderr, "symlink %s%s\n", prefix, name) 85 89 default: 86 - fmt.Fprintln(os.Stderr, "other", name) 90 + fmt.Fprintf(os.Stderr, "other %s%s\n", prefix, name) 87 91 } 88 92 return nil 89 93 }) 90 94 } 91 95 92 - func archiveFS(writer io.Writer, root fs.FS) (err error) { 96 + func archiveFS(writer io.Writer, root fs.FS, prefix string) (err error) { 93 97 zstdWriter, _ := zstd.NewWriter(writer) 94 98 tarWriter := tar.NewWriter(zstdWriter) 95 - err = tarWriter.AddFS(root) 96 - if err != nil { 99 + if err = fs.WalkDir(root, ".", func(name string, entry fs.DirEntry, err error) error { 100 + if err != nil { 101 + return err 102 + } 103 + fileInfo, err := entry.Info() 104 + if err != nil { 105 + return err 106 + } 107 + var tarName string 108 + if prefix == "" && name == "." { 109 + return nil 110 + } else if prefix == "" { 111 + tarName = name 112 + } else if name == "." { 113 + tarName = prefix 114 + } else { 115 + tarName = fmt.Sprintf("%s/%s", prefix, name) 116 + } 117 + var file io.ReadCloser 118 + var linkTarget string 119 + switch { 120 + case entry.Type().IsDir(): 121 + name += "/" 122 + case entry.Type().IsRegular(): 123 + if file, err = root.Open(name); err != nil { 124 + return err 125 + } 126 + defer file.Close() 127 + case entry.Type() == fs.ModeSymlink: 128 + if linkTarget, err = fs.ReadLink(root, name); err != nil { 129 + return err 130 + } 131 + default: 132 + return errors.New("tar: cannot add non-regular file") 133 + } 134 + header, err := tar.FileInfoHeader(fileInfo, linkTarget) 135 + if err != nil { 136 + return err 137 + } 138 + header.Name = tarName 139 + if err = tarWriter.WriteHeader(header); err != nil { 140 + return err 141 + } 142 + if file != nil { 143 + _, err = io.Copy(tarWriter, file) 144 + } 145 + return err 146 + }); err != nil { 97 147 return 98 148 } 99 - err = tarWriter.Close() 100 - if err != nil { 149 + if err = tarWriter.Close(); err != nil { 101 150 return 102 151 } 103 - err = zstdWriter.Close() 104 - if err != nil { 152 + if err = zstdWriter.Close(); err != nil { 105 153 return 106 154 } 107 155 return 108 156 } 109 157 158 + func makeWhiteout(path string) (reader io.Reader) { 159 + buffer := &bytes.Buffer{} 160 + tarWriter := tar.NewWriter(buffer) 161 + tarWriter.WriteHeader(&tar.Header{ 162 + Typeflag: tar.TypeChar, 163 + Name: path, 164 + }) 165 + tarWriter.Flush() 166 + return buffer 167 + } 168 + 110 169 const usageExitCode = 125 111 170 112 171 func usage() { ··· 133 192 if *passwordFlag != "" && *tokenFlag != "" { 134 193 fmt.Fprintf(os.Stderr, "--password and --token are mutually exclusive") 135 194 os.Exit(usageExitCode) 195 + } 196 + 197 + var pathPrefix string 198 + if *pathFlag != "" { 199 + if *uploadDirFlag == "" && !*deleteFlag { 200 + fmt.Fprintf(os.Stderr, "--path requires --upload-dir or --delete") 201 + os.Exit(usageExitCode) 202 + } else { 203 + pathPrefix = strings.Trim(*pathFlag, "/") 204 + } 136 205 } 137 206 138 207 var err error ··· 181 250 } 182 251 183 252 if *verboseFlag { 184 - err := displayFS(uploadDirFS.FS()) 253 + err := displayFS(uploadDirFS.FS(), pathPrefix) 185 254 if err != nil { 186 255 fmt.Fprintf(os.Stderr, "error: %s\n", err) 187 256 os.Exit(1) ··· 191 260 // Stream archive data without ever loading the entire working set into RAM. 192 261 reader, writer := io.Pipe() 193 262 go func() { 194 - err = archiveFS(writer, uploadDirFS.FS()) 263 + err = archiveFS(writer, uploadDirFS.FS(), pathPrefix) 195 264 if err != nil { 196 265 fmt.Fprintf(os.Stderr, "error: %s\n", err) 197 266 os.Exit(1) ··· 199 268 writer.Close() 200 269 }() 201 270 202 - request, err = http.NewRequest("PUT", siteURL.String(), reader) 271 + if *pathFlag == "" { 272 + request, err = http.NewRequest("PUT", siteURL.String(), reader) 273 + } else { 274 + request, err = http.NewRequest("PATCH", siteURL.String(), reader) 275 + } 203 276 if err != nil { 204 277 fmt.Fprintf(os.Stderr, "error: %s\n", err) 205 278 os.Exit(1) ··· 208 281 request.Header.Add("Content-Type", "application/x-tar+zstd") 209 282 210 283 case *deleteFlag: 211 - request, err = http.NewRequest("DELETE", siteURL.String(), nil) 212 - if err != nil { 213 - fmt.Fprintf(os.Stderr, "error: %s\n", err) 214 - os.Exit(1) 284 + if *pathFlag == "" { 285 + request, err = http.NewRequest("DELETE", siteURL.String(), nil) 286 + if err != nil { 287 + fmt.Fprintf(os.Stderr, "error: %s\n", err) 288 + os.Exit(1) 289 + } 290 + } else { 291 + request, err = http.NewRequest("PATCH", siteURL.String(), makeWhiteout(pathPrefix)) 292 + if err != nil { 293 + fmt.Fprintf(os.Stderr, "error: %s\n", err) 294 + os.Exit(1) 295 + } 296 + request.Header.Add("Content-Type", "application/x-tar") 215 297 } 216 298 217 299 case *debugManifestFlag: ··· 226 308 panic("no operation chosen") 227 309 } 228 310 request.Header.Add("User-Agent", versionInfo()) 311 + if request.Method == "PATCH" { 312 + if *raceFreeFlag { 313 + request.Header.Add("Race-Free", "yes") 314 + } else { 315 + request.Header.Add("Race-Free", "no") 316 + } 317 + } 229 318 switch { 230 319 case *passwordFlag != "": 231 320 request.Header.Add("Authorization", fmt.Sprintf("Pages %s", *passwordFlag))