helpimnotdrowning.net
forked from
tangled.org/core
Monorepo for Tangled
appview,knotserver: support immutable nix flakeref link header
This will allow users to directly request archive from the knot without
using xrpc.
Xrpc doesn't fit here because it strips out the http headers which might
include valuable metadata like download filename or immutable link.
- implement archive on knot as `/{owner}/{repo}/archive/{ref}`
endpoint
- appview proxies the request to knot on `/archive` like it is doing for
git http endpoints.
if knot version isn't compatible, it will fallback to legacy xrpc
endpoint.
- rename the `git_http.go` file to generalized `proxy_knot.go` filaname
xrpc method `sh.tangled.repo.archive` will be deprecated in future
added `go-version` depenedency to make version constraints
Close: <https://tangled.org/tangled.org/core/issues/231>
Signed-off-by: Seongmin Lee <git@boltless.me>
boltless.me
-49
appview/repo/archive.go
-49
appview/repo/archive.go
···
1
-
package repo
2
-
3
-
import (
4
-
"fmt"
5
-
"net/http"
6
-
"net/url"
7
-
"strings"
8
-
9
-
"tangled.org/core/api/tangled"
10
-
xrpcclient "tangled.org/core/appview/xrpcclient"
11
-
12
-
indigoxrpc "github.com/bluesky-social/indigo/xrpc"
13
-
"github.com/go-chi/chi/v5"
14
-
"github.com/go-git/go-git/v5/plumbing"
15
-
)
16
-
17
-
func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) {
18
-
l := rp.logger.With("handler", "DownloadArchive")
19
-
ref := chi.URLParam(r, "ref")
20
-
ref, _ = url.PathUnescape(ref)
21
-
f, err := rp.repoResolver.Resolve(r)
22
-
if err != nil {
23
-
l.Error("failed to get repo and knot", "err", err)
24
-
return
25
-
}
26
-
scheme := "http"
27
-
if !rp.config.Core.Dev {
28
-
scheme = "https"
29
-
}
30
-
host := fmt.Sprintf("%s://%s", scheme, f.Knot)
31
-
xrpcc := &indigoxrpc.Client{
32
-
Host: host,
33
-
}
34
-
didSlashRepo := f.DidSlashRepo()
35
-
archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, didSlashRepo)
36
-
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
37
-
l.Error("failed to call XRPC repo.archive", "err", xrpcerr)
38
-
rp.pages.Error503(w)
39
-
return
40
-
}
41
-
// Set headers for file download, just pass along whatever the knot specifies
42
-
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
43
-
filename := fmt.Sprintf("%s-%s.tar.gz", f.Name, safeRefFilename)
44
-
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
45
-
w.Header().Set("Content-Type", "application/gzip")
46
-
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes)))
47
-
// Write the archive data directly
48
-
w.Write(archiveBytes)
49
-
}
-4
appview/repo/router.go
-4
appview/repo/router.go
···
40
40
r.Get("/blob/{ref}/*", rp.Blob)
41
41
r.Get("/raw/{ref}/*", rp.RepoBlobRaw)
42
42
43
-
// intentionally doesn't use /* as this isn't
44
-
// a file path
45
-
r.Get("/archive/{ref}", rp.DownloadArchive)
46
-
47
43
r.Route("/fork", func(r chi.Router) {
48
44
r.Use(middleware.AuthMiddleware(rp.oauth))
49
45
r.Get("/", rp.ForkRepo)
-97
appview/state/git_http.go
-97
appview/state/git_http.go
···
1
-
package state
2
-
3
-
import (
4
-
"fmt"
5
-
"io"
6
-
"maps"
7
-
"net/http"
8
-
9
-
"github.com/bluesky-social/indigo/atproto/identity"
10
-
"github.com/go-chi/chi/v5"
11
-
"tangled.org/core/appview/models"
12
-
)
13
-
14
-
func (s *State) InfoRefs(w http.ResponseWriter, r *http.Request) {
15
-
user := r.Context().Value("resolvedId").(identity.Identity)
16
-
repo := r.Context().Value("repo").(*models.Repo)
17
-
18
-
scheme := "https"
19
-
if s.config.Core.Dev {
20
-
scheme = "http"
21
-
}
22
-
23
-
targetURL := fmt.Sprintf("%s://%s/%s/%s/info/refs?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
24
-
s.proxyRequest(w, r, targetURL)
25
-
26
-
}
27
-
28
-
func (s *State) UploadPack(w http.ResponseWriter, r *http.Request) {
29
-
user, ok := r.Context().Value("resolvedId").(identity.Identity)
30
-
if !ok {
31
-
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
32
-
return
33
-
}
34
-
repo := r.Context().Value("repo").(*models.Repo)
35
-
36
-
scheme := "https"
37
-
if s.config.Core.Dev {
38
-
scheme = "http"
39
-
}
40
-
41
-
targetURL := fmt.Sprintf("%s://%s/%s/%s/git-upload-pack?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
42
-
s.proxyRequest(w, r, targetURL)
43
-
}
44
-
45
-
func (s *State) ReceivePack(w http.ResponseWriter, r *http.Request) {
46
-
user, ok := r.Context().Value("resolvedId").(identity.Identity)
47
-
if !ok {
48
-
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
49
-
return
50
-
}
51
-
repo := r.Context().Value("repo").(*models.Repo)
52
-
53
-
scheme := "https"
54
-
if s.config.Core.Dev {
55
-
scheme = "http"
56
-
}
57
-
58
-
targetURL := fmt.Sprintf("%s://%s/%s/%s/git-receive-pack?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
59
-
s.proxyRequest(w, r, targetURL)
60
-
}
61
-
62
-
func (s *State) proxyRequest(w http.ResponseWriter, r *http.Request, targetURL string) {
63
-
client := &http.Client{}
64
-
65
-
// Create new request
66
-
proxyReq, err := http.NewRequest(r.Method, targetURL, r.Body)
67
-
if err != nil {
68
-
http.Error(w, err.Error(), http.StatusInternalServerError)
69
-
return
70
-
}
71
-
72
-
// Copy original headers
73
-
proxyReq.Header = r.Header
74
-
75
-
repoOwnerHandle := chi.URLParam(r, "user")
76
-
proxyReq.Header.Add("x-tangled-repo-owner-handle", repoOwnerHandle)
77
-
78
-
// Execute request
79
-
resp, err := client.Do(proxyReq)
80
-
if err != nil {
81
-
http.Error(w, err.Error(), http.StatusInternalServerError)
82
-
return
83
-
}
84
-
defer resp.Body.Close()
85
-
86
-
// Copy response headers
87
-
maps.Copy(w.Header(), resp.Header)
88
-
89
-
// Set response status code
90
-
w.WriteHeader(resp.StatusCode)
91
-
92
-
// Copy response body
93
-
if _, err := io.Copy(w, resp.Body); err != nil {
94
-
http.Error(w, err.Error(), http.StatusInternalServerError)
95
-
return
96
-
}
97
-
}
+168
appview/state/proxy_knot.go
+168
appview/state/proxy_knot.go
···
1
+
package state
2
+
3
+
import (
4
+
"fmt"
5
+
"io"
6
+
"maps"
7
+
"net/http"
8
+
"strings"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/identity"
11
+
indigoxrpc "github.com/bluesky-social/indigo/xrpc"
12
+
"github.com/go-chi/chi/v5"
13
+
"github.com/go-git/go-git/v5/plumbing"
14
+
"github.com/hashicorp/go-version"
15
+
"tangled.org/core/api/tangled"
16
+
"tangled.org/core/appview/models"
17
+
xrpcclient "tangled.org/core/appview/xrpcclient"
18
+
)
19
+
20
+
func (s *State) InfoRefs(w http.ResponseWriter, r *http.Request) {
21
+
user := r.Context().Value("resolvedId").(identity.Identity)
22
+
repo := r.Context().Value("repo").(*models.Repo)
23
+
24
+
scheme := "https"
25
+
if s.config.Core.Dev {
26
+
scheme = "http"
27
+
}
28
+
29
+
targetURL := fmt.Sprintf("%s://%s/%s/%s/info/refs?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
30
+
s.proxyRequest(w, r, targetURL)
31
+
32
+
}
33
+
34
+
func (s *State) UploadPack(w http.ResponseWriter, r *http.Request) {
35
+
user, ok := r.Context().Value("resolvedId").(identity.Identity)
36
+
if !ok {
37
+
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
38
+
return
39
+
}
40
+
repo := r.Context().Value("repo").(*models.Repo)
41
+
42
+
scheme := "https"
43
+
if s.config.Core.Dev {
44
+
scheme = "http"
45
+
}
46
+
47
+
targetURL := fmt.Sprintf("%s://%s/%s/%s/git-upload-pack?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
48
+
s.proxyRequest(w, r, targetURL)
49
+
}
50
+
51
+
func (s *State) ReceivePack(w http.ResponseWriter, r *http.Request) {
52
+
user, ok := r.Context().Value("resolvedId").(identity.Identity)
53
+
if !ok {
54
+
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
55
+
return
56
+
}
57
+
repo := r.Context().Value("repo").(*models.Repo)
58
+
59
+
scheme := "https"
60
+
if s.config.Core.Dev {
61
+
scheme = "http"
62
+
}
63
+
64
+
targetURL := fmt.Sprintf("%s://%s/%s/%s/git-receive-pack?%s", scheme, repo.Knot, user.DID, repo.Name, r.URL.RawQuery)
65
+
s.proxyRequest(w, r, targetURL)
66
+
}
67
+
68
+
var knotVersionDownloadArchiveConstraint = version.MustConstraints(version.NewConstraint(">= 1.12.0-alpha"))
69
+
70
+
func (s *State) DownloadArchive(w http.ResponseWriter, r *http.Request) {
71
+
l := s.logger.With("handler", "DownloadArchive")
72
+
ref := chi.URLParam(r, "ref")
73
+
74
+
user, ok := r.Context().Value("resolvedId").(identity.Identity)
75
+
if !ok {
76
+
l.Error("failed to resolve user")
77
+
http.Error(w, "failed to resolve user", http.StatusInternalServerError)
78
+
return
79
+
}
80
+
repo := r.Context().Value("repo").(*models.Repo)
81
+
82
+
scheme := "https"
83
+
if s.config.Core.Dev {
84
+
scheme = "http"
85
+
}
86
+
87
+
host := fmt.Sprintf("%s://%s", scheme, repo.Knot)
88
+
xrpcc := &indigoxrpc.Client{
89
+
Host: host,
90
+
}
91
+
l = l.With("knot", repo.Knot)
92
+
93
+
isCompatible := func() bool {
94
+
out, err := tangled.KnotVersion(r.Context(), xrpcc)
95
+
if err != nil {
96
+
l.Warn("failed to get knot version", "err", err)
97
+
return false
98
+
}
99
+
100
+
v, err := version.NewVersion(out.Version)
101
+
if err != nil {
102
+
l.Warn("failed to parse knot version", "version", out.Version, "err", err)
103
+
return false
104
+
}
105
+
106
+
if !knotVersionDownloadArchiveConstraint.Check(v) {
107
+
l.Warn("knot version incompatible.", "version", v)
108
+
return false
109
+
}
110
+
return true
111
+
}()
112
+
l.Debug("knot compatibility check", "isCompatible", isCompatible)
113
+
if isCompatible {
114
+
targetURL := fmt.Sprintf("%s://%s/%s/%s/archive/%s", scheme, repo.Knot, user.DID, repo.Name, ref)
115
+
s.proxyRequest(w, r, targetURL)
116
+
} else {
117
+
l.Debug("requesting xrpc/sh.tangled.repo.archive")
118
+
archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, repo.DidSlashRepo())
119
+
if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil {
120
+
l.Error("failed to call XRPC repo.archive", "err", xrpcerr)
121
+
s.pages.Error503(w)
122
+
return
123
+
}
124
+
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
125
+
filename := fmt.Sprintf("%s-%s.tar.gz", repo.Name, safeRefFilename)
126
+
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
127
+
w.Header().Set("Content-Type", "application/gzip")
128
+
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes)))
129
+
w.Write(archiveBytes)
130
+
}
131
+
}
132
+
133
+
func (s *State) proxyRequest(w http.ResponseWriter, r *http.Request, targetURL string) {
134
+
client := &http.Client{}
135
+
136
+
// Create new request
137
+
proxyReq, err := http.NewRequest(r.Method, targetURL, r.Body)
138
+
if err != nil {
139
+
http.Error(w, err.Error(), http.StatusInternalServerError)
140
+
return
141
+
}
142
+
143
+
// Copy original headers
144
+
proxyReq.Header = r.Header
145
+
146
+
repoOwnerHandle := chi.URLParam(r, "user")
147
+
proxyReq.Header.Add("x-tangled-repo-owner-handle", repoOwnerHandle)
148
+
149
+
// Execute request
150
+
resp, err := client.Do(proxyReq)
151
+
if err != nil {
152
+
http.Error(w, err.Error(), http.StatusInternalServerError)
153
+
return
154
+
}
155
+
defer resp.Body.Close()
156
+
157
+
// Copy response headers
158
+
maps.Copy(w.Header(), resp.Header)
159
+
160
+
// Set response status code
161
+
w.WriteHeader(resp.StatusCode)
162
+
163
+
// Copy response body
164
+
if _, err := io.Copy(w, resp.Body); err != nil {
165
+
http.Error(w, err.Error(), http.StatusInternalServerError)
166
+
return
167
+
}
168
+
}
+3
-1
appview/state/router.go
+3
-1
appview/state/router.go
···
103
103
r.Get("/info/refs", s.InfoRefs)
104
104
r.Post("/git-upload-pack", s.UploadPack)
105
105
r.Post("/git-receive-pack", s.ReceivePack)
106
-
106
+
// intentionally doesn't use /* as this isn't
107
+
// a file path
108
+
r.Get("/archive/{ref}", s.DownloadArchive)
107
109
})
108
110
})
109
111
+1
go.mod
+1
go.mod
···
131
131
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
132
132
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
133
133
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
134
+
github.com/hashicorp/go-version v1.8.0 // indirect
134
135
github.com/hashicorp/golang-lru v1.0.2 // indirect
135
136
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
136
137
github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
+2
go.sum
+2
go.sum
···
264
264
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
265
265
github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
266
266
github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
267
+
github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
268
+
github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
267
269
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
268
270
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
269
271
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+69
knotserver/archive.go
+69
knotserver/archive.go
···
1
+
package knotserver
2
+
3
+
import (
4
+
"compress/gzip"
5
+
"fmt"
6
+
"net/http"
7
+
"strings"
8
+
9
+
securejoin "github.com/cyphar/filepath-securejoin"
10
+
"github.com/go-chi/chi/v5"
11
+
"github.com/go-git/go-git/v5/plumbing"
12
+
"tangled.org/core/knotserver/git"
13
+
)
14
+
15
+
func (h *Knot) Archive(w http.ResponseWriter, r *http.Request) {
16
+
var (
17
+
did = chi.URLParam(r, "did")
18
+
name = chi.URLParam(r, "name")
19
+
ref = chi.URLParam(r, "ref")
20
+
)
21
+
repo, err := securejoin.SecureJoin(did, name)
22
+
if err != nil {
23
+
gitError(w, "repository not found", http.StatusNotFound)
24
+
h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err)
25
+
return
26
+
}
27
+
28
+
repoPath, err := securejoin.SecureJoin(h.c.Repo.ScanPath, repo)
29
+
if err != nil {
30
+
gitError(w, "repository not found", http.StatusNotFound)
31
+
h.l.Error("git: failed to secure join repo path", "handler", "InfoRefs", "error", err)
32
+
return
33
+
}
34
+
35
+
gr, err := git.Open(repoPath, ref)
36
+
37
+
immutableLink := fmt.Sprintf(
38
+
"https://%s/%s/%s/archive/%s",
39
+
h.c.Server.Hostname,
40
+
did,
41
+
name,
42
+
gr.Hash(),
43
+
)
44
+
45
+
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
46
+
filename := fmt.Sprintf("%s-%s.tar.gz", name, safeRefFilename)
47
+
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
48
+
w.Header().Set("Content-Type", "application/gzip")
49
+
w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink))
50
+
51
+
gw := gzip.NewWriter(w)
52
+
defer gw.Close()
53
+
54
+
err = gr.WriteTar(gw, "")
55
+
if err != nil {
56
+
// once we start writing to the body we can't report error anymore
57
+
// so we are only left with logging the error
58
+
h.l.Error("writing tar file", "error", err)
59
+
return
60
+
}
61
+
62
+
err = gw.Flush()
63
+
if err != nil {
64
+
// once we start writing to the body we can't report error anymore
65
+
// so we are only left with logging the error
66
+
h.l.Error("flushing", "error", err.Error())
67
+
return
68
+
}
69
+
}
+4
knotserver/git/git.go
+4
knotserver/git/git.go
+2
knotserver/router.go
+2
knotserver/router.go
+3
nix/gomod2nix.toml
+3
nix/gomod2nix.toml
···
304
304
[mod."github.com/hashicorp/go-sockaddr"]
305
305
version = "v1.0.7"
306
306
hash = "sha256-p6eDOrGzN1jMmT/F/f/VJMq0cKNFhUcEuVVwTE6vSrs="
307
+
[mod."github.com/hashicorp/go-version"]
308
+
version = "v1.8.0"
309
+
hash = "sha256-KXtqERmYrWdpqPCViWcHbe6jnuH7k16bvBIcuJuevj8="
307
310
[mod."github.com/hashicorp/golang-lru"]
308
311
version = "v1.0.2"
309
312
hash = "sha256-yy+5botc6T5wXgOe2mfNXJP3wr+MkVlUZ2JBkmmrA48="