willdot.net
A tool for backing up ATProto related data to S3
1package main
2
3import (
4 "archive/tar"
5 "compress/gzip"
6 "context"
7 "fmt"
8 "io"
9 "log/slog"
10 "os"
11 "path"
12 "path/filepath"
13 "time"
14
15 "github.com/bugsnag/bugsnag-go/v2"
16 "github.com/minio/minio-go/v7"
17)
18
19func (s *service) backupTangledKnot(ctx context.Context) {
20 err := s.backupKnotDB(ctx)
21 if err != nil {
22 bugsnag.Notify(fmt.Errorf("backup knot db: %w", err))
23 slog.Error("failed to backup knot db", "error", err)
24 }
25 err = s.backupKnotRepos(ctx)
26 if err != nil {
27 bugsnag.Notify(fmt.Errorf("backup knot db: %w", err))
28 slog.Error("failed to backup knot db", "error", err)
29 }
30
31 slog.Info("finished tangled knot backup")
32}
33
34func (s *service) backupKnotDB(ctx context.Context) error {
35 dir := os.Getenv("TANGLED_KNOT_DATABASE_DIRECTORY")
36 if dir == "" {
37 slog.Info("TANGLED_KNOT_DATABASE_DIRECTORY env not set - skipping knot DB backup")
38 }
39
40 filename := path.Join(s.blobDir, fmt.Sprintf("%d-knot.zip", time.Now().UnixMilli()))
41 f, err := os.Create(filename)
42 if err != nil {
43 return fmt.Errorf("creating temp file: %w", err)
44 }
45 defer func() {
46 f.Close()
47
48 err = os.Remove(filename)
49 if err != nil {
50 slog.Error("failed to delete knot db zip file after uploading", "error", err, "filename", f.Name())
51 metadata := bugsnag.MetaData{
52 "file": {
53 "filename": f.Name(),
54 },
55 }
56 bugsnag.Notify(fmt.Errorf("delete knot db zip file after uploading: %w", err), metadata)
57 }
58 }()
59
60 compress(dir, f)
61
62 // reset the reader back to the start so that the minio upload can read the data that's been written.
63 _, err = f.Seek(0, 0)
64 if err != nil {
65 return fmt.Errorf("setting seek on written file: %w", err)
66 }
67
68 fi, err := f.Stat()
69 if err != nil {
70 return fmt.Errorf("stat written file: %w", err)
71 }
72
73 _, err = s.minioClient.PutObject(ctx, s.bucketName, "knot-db.zip", f, fi.Size(), minio.PutObjectOptions{})
74 if err != nil {
75 return fmt.Errorf("put knot db zip file to bucket: %w", err)
76 }
77 return nil
78}
79
80func (s *service) backupKnotRepos(ctx context.Context) error {
81 dir := os.Getenv("TANGLED_KNOT_REPOSITORY_DIRECTORY")
82 if dir == "" {
83 slog.Info("TANGLED_KNOT_REPOSITORY_DIRECTORY env not set - skipping knot repo backup")
84 }
85
86 filename := path.Join(s.blobDir, fmt.Sprintf("%d-knot-repos.zip", time.Now().UnixMilli()))
87 f, err := os.Create(filename)
88 if err != nil {
89 return fmt.Errorf("creating temp file: %w", err)
90 }
91 defer func() {
92 f.Close()
93
94 err = os.Remove(filename)
95 if err != nil {
96 slog.Error("failed to delete knot repos zip file after uploading", "error", err, "filename", f.Name())
97 metadata := bugsnag.MetaData{
98 "file": {
99 "filename": f.Name(),
100 },
101 }
102 bugsnag.Notify(fmt.Errorf("delete knot repos zip file after uploading: %w", err), metadata)
103 }
104 }()
105
106 compress(dir, f)
107
108 // reset the reader back to the start so that the minio upload can read the data that's been written.
109 _, err = f.Seek(0, 0)
110 if err != nil {
111 return fmt.Errorf("setting seek on written file: %w", err)
112 }
113
114 fi, err := f.Stat()
115 if err != nil {
116 return fmt.Errorf("stat written file: %w", err)
117 }
118
119 _, err = s.minioClient.PutObject(ctx, s.bucketName, "knot-repos.zip", f, fi.Size(), minio.PutObjectOptions{})
120 if err != nil {
121 return fmt.Errorf("put knot repo file to bucket: %w", err)
122 }
123 return nil
124}
125
126func compress(src string, writer io.Writer) error {
127 zipWriter := gzip.NewWriter(writer)
128 tarWriter := tar.NewWriter(zipWriter)
129
130 filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {
131 header, err := tar.FileInfoHeader(fi, file)
132 if err != nil {
133 return err
134 }
135
136 // must provide real name
137 // (see https://golang.org/src/archive/tar/common.go?#L626)
138 header.Name = filepath.ToSlash(file)
139
140 if err := tarWriter.WriteHeader(header); err != nil {
141 return err
142 }
143 // if not a dir, write file content
144 if !fi.IsDir() {
145 data, err := os.Open(file)
146 if err != nil {
147 return err
148 }
149 if _, err := io.Copy(tarWriter, data); err != nil {
150 return err
151 }
152 }
153
154 return nil
155 })
156
157 // produce tar
158 if err := tarWriter.Close(); err != nil {
159 return err
160 }
161 // produce gzip
162 if err := zipWriter.Close(); err != nil {
163 return err
164 }
165
166 return nil
167}