whitequark.org
[mirror] Scalable static site server for Git forges (like GitHub Pages)
1package git_pages
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "io"
8 "iter"
9 "log"
10 "log/slog"
11 "math/rand/v2"
12 "net/http"
13 "os"
14 "runtime/debug"
15 "strconv"
16 "sync"
17 "time"
18
19 slogmulti "github.com/samber/slog-multi"
20
21 syslog "codeberg.org/git-pages/go-slog-syslog"
22
23 "github.com/prometheus/client_golang/prometheus"
24 "github.com/prometheus/client_golang/prometheus/promauto"
25
26 "github.com/getsentry/sentry-go"
27 sentryhttp "github.com/getsentry/sentry-go/http"
28 sentryslog "github.com/getsentry/sentry-go/slog"
29)
30
31var (
32 httpRequestCount = promauto.NewCounterVec(prometheus.CounterOpts{
33 Name: "git_pages_http_request_count",
34 Help: "Count of HTTP requests by method and response status code",
35 }, []string{"method", "code"})
36
37 httpRequestDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{
38 Name: "git_pages_http_request_duration_seconds",
39 Help: "Time to respond to incoming HTTP requests",
40 Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
41
42 NativeHistogramBucketFactor: 1.1,
43 NativeHistogramMaxBucketNumber: 100,
44 NativeHistogramMinResetDuration: 10 * time.Minute,
45 }, []string{"method"})
46)
47
48var syslogHandler syslog.Handler
49
50func hasSentry() bool {
51 return os.Getenv("SENTRY_DSN") != ""
52}
53
54func chainSentryMiddleware(
55 middleware ...func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event,
56) func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
57 return func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
58 for idx := 0; idx < len(middleware) && event != nil; idx++ {
59 event = middleware[idx](event, hint)
60 }
61 return event
62 }
63}
64
65// sensitiveHTTPHeaders extends the list of sensitive headers defined in the Sentry Go SDK with our
66// own application-specific header field names.
67var sensitiveHTTPHeaders = map[string]struct{}{
68 "Forge-Authorization": {},
69}
70
71// scrubSentryEvent removes sensitive HTTP header fields from the Sentry event.
72func scrubSentryEvent(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
73 if event.Request != nil && event.Request.Headers != nil {
74 for key := range event.Request.Headers {
75 if _, ok := sensitiveHTTPHeaders[key]; ok {
76 delete(event.Request.Headers, key)
77 }
78 }
79 }
80 return event
81}
82
83// sampleSentryEvent returns a function that discards a Sentry event according to the sample rate,
84// unless the associated HTTP request triggers a mutation or it took too long to produce a response,
85// in which case the event is never discarded.
86func sampleSentryEvent(sampleRate float64) func(*sentry.Event, *sentry.EventHint) *sentry.Event {
87 return func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
88 newSampleRate := sampleRate
89 if event.Request != nil {
90 switch event.Request.Method {
91 case "PUT", "POST", "DELETE":
92 newSampleRate = 1
93 }
94 }
95 duration := event.Timestamp.Sub(event.StartTime)
96 threshold := time.Duration(config.Observability.SlowResponseThreshold)
97 if duration >= threshold {
98 newSampleRate = 1
99 }
100 if rand.Float64() < newSampleRate {
101 return event
102 }
103 return nil
104 }
105}
106
107func InitObservability() {
108 debug.SetPanicOnFault(true)
109
110 environment := "development"
111 if value, ok := os.LookupEnv("ENVIRONMENT"); ok {
112 environment = value
113 }
114
115 logHandlers := []slog.Handler{}
116
117 switch config.LogFormat {
118 case "none":
119 // nothing to do
120 case "text":
121 logHandlers = append(logHandlers,
122 slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{}))
123 case "json":
124 logHandlers = append(logHandlers,
125 slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{}))
126 default:
127 log.Println("unknown log format", config.LogFormat)
128 }
129
130 if syslogAddr := os.Getenv("SYSLOG_ADDR"); syslogAddr != "" {
131 var err error
132 syslogHandler, err = syslog.NewHandler(&syslog.HandlerOptions{
133 Address: syslogAddr,
134 AppName: "git-pages",
135 StructuredDataID: "git-pages",
136 })
137 if err != nil {
138 log.Fatalf("syslog: %v", err)
139 }
140 logHandlers = append(logHandlers, syslogHandler)
141 }
142
143 if hasSentry() {
144 enableLogs := false
145 if value, err := strconv.ParseBool(os.Getenv("SENTRY_LOGS")); err == nil {
146 enableLogs = value
147 }
148
149 enableTracing := false
150 if value, err := strconv.ParseBool(os.Getenv("SENTRY_TRACING")); err == nil {
151 enableTracing = value
152 }
153
154 tracesSampleRate := 1.00
155 switch environment {
156 case "development", "staging":
157 default:
158 tracesSampleRate = 0.05
159 }
160
161 options := sentry.ClientOptions{}
162 options.Environment = environment
163 options.EnableLogs = enableLogs
164 options.EnableTracing = enableTracing
165 options.TracesSampleRate = 1 // use our own custom sampling logic
166 options.BeforeSend = scrubSentryEvent
167 options.BeforeSendTransaction = chainSentryMiddleware(
168 sampleSentryEvent(tracesSampleRate),
169 scrubSentryEvent,
170 )
171 if err := sentry.Init(options); err != nil {
172 log.Fatalf("sentry: %s\n", err)
173 }
174
175 if enableLogs {
176 logHandlers = append(logHandlers, sentryslog.Option{
177 AddSource: true,
178 }.NewSentryHandler(context.Background()))
179 }
180 }
181
182 slog.SetDefault(slog.New(slogmulti.Fanout(logHandlers...)))
183}
184
185func FiniObservability() {
186 var wg sync.WaitGroup
187 timeout := 2 * time.Second
188 if syslogHandler != nil {
189 wg.Go(func() { syslogHandler.Flush(timeout) })
190 }
191 if hasSentry() {
192 wg.Go(func() { sentry.Flush(timeout) })
193 }
194 wg.Wait()
195}
196
197func ObserveError(err error) {
198 if errors.Is(err, context.Canceled) {
199 // Something has explicitly requested cancellation.
200 // Timeout results in a different error.
201 return
202 }
203
204 if hasSentry() {
205 sentry.CaptureException(err)
206 }
207}
208
209type observedResponseWriter struct {
210 inner http.ResponseWriter
211 status int
212}
213
214func newObservedResponseWriter(w http.ResponseWriter) observedResponseWriter {
215 return observedResponseWriter{
216 inner: w,
217 status: 0,
218 }
219}
220
221func (w *observedResponseWriter) Unwrap() http.ResponseWriter {
222 return w.inner
223}
224
225func (w *observedResponseWriter) Header() http.Header {
226 return w.inner.Header()
227}
228
229func (w *observedResponseWriter) Write(data []byte) (int, error) {
230 return w.inner.Write(data)
231}
232
233func (w *observedResponseWriter) WriteHeader(statusCode int) {
234 w.status = statusCode
235 w.inner.WriteHeader(statusCode)
236}
237
238func ObserveHTTPHandler(handler http.Handler) http.Handler {
239 if hasSentry() {
240 handler = func(next http.Handler) http.Handler {
241 next = sentryhttp.New(sentryhttp.Options{
242 Repanic: true,
243 }).Handle(handler)
244
245 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
246 // Prevent the Sentry SDK from continuing traces as we don't use this feature.
247 r.Header.Del(sentry.SentryTraceHeader)
248 r.Header.Del(sentry.SentryBaggageHeader)
249
250 next.ServeHTTP(w, r)
251 })
252 }(handler)
253 }
254
255 handler = func(next http.Handler) http.Handler {
256 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
257 ow := newObservedResponseWriter(w)
258
259 start := time.Now()
260 next.ServeHTTP(&ow, r)
261 duration := time.Since(start)
262
263 httpRequestCount.
264 With(prometheus.Labels{"method": r.Method, "code": fmt.Sprintf("%d", ow.status)}).
265 Inc()
266 httpRequestDurationSeconds.
267 With(prometheus.Labels{"method": r.Method}).
268 Observe(duration.Seconds())
269 })
270 }(handler)
271
272 return handler
273}
274
275type noopSpan struct{}
276
277func (span noopSpan) Finish() {}
278
279func ObserveFunction(
280 ctx context.Context, funcName string, data ...any,
281) (
282 interface{ Finish() }, context.Context,
283) {
284 switch {
285 case hasSentry():
286 span := sentry.StartSpan(ctx, "function")
287 span.Description = funcName
288 ObserveData(span.Context(), data...)
289 return span, span.Context()
290 default:
291 return noopSpan{}, ctx
292 }
293}
294
295func ObserveData(ctx context.Context, data ...any) {
296 if span := sentry.SpanFromContext(ctx); span != nil {
297 for i := 0; i < len(data); i += 2 {
298 name, value := data[i], data[i+1]
299 span.SetData(name.(string), value)
300 }
301 }
302}
303
304var (
305 blobsRetrievedCount = promauto.NewCounter(prometheus.CounterOpts{
306 Name: "git_pages_blobs_retrieved",
307 Help: "Count of blobs retrieved",
308 })
309 blobsRetrievedBytes = promauto.NewCounter(prometheus.CounterOpts{
310 Name: "git_pages_blobs_retrieved_bytes",
311 Help: "Total size in bytes of blobs retrieved",
312 })
313
314 blobsStoredCount = promauto.NewCounter(prometheus.CounterOpts{
315 Name: "git_pages_blobs_stored",
316 Help: "Count of blobs stored",
317 })
318 blobsStoredBytes = promauto.NewCounter(prometheus.CounterOpts{
319 Name: "git_pages_blobs_stored_bytes",
320 Help: "Total size in bytes of blobs stored",
321 })
322
323 manifestsRetrievedCount = promauto.NewCounter(prometheus.CounterOpts{
324 Name: "git_pages_manifests_retrieved",
325 Help: "Count of manifests retrieved",
326 })
327)
328
329type observedBackend struct {
330 inner Backend
331}
332
333var _ Backend = (*observedBackend)(nil)
334
335func NewObservedBackend(backend Backend) Backend {
336 return &observedBackend{inner: backend}
337}
338
339func (backend *observedBackend) HasFeature(ctx context.Context, feature BackendFeature) (isOn bool) {
340 span, ctx := ObserveFunction(ctx, "HasFeature")
341 isOn = backend.inner.HasFeature(ctx, feature)
342 span.Finish()
343 return
344}
345
346func (backend *observedBackend) EnableFeature(ctx context.Context, feature BackendFeature) (err error) {
347 span, ctx := ObserveFunction(ctx, "EnableFeature")
348 err = backend.inner.EnableFeature(ctx, feature)
349 span.Finish()
350 return
351}
352
353func (backend *observedBackend) GetBlob(
354 ctx context.Context, name string,
355) (
356 reader io.ReadSeeker, metadata BlobMetadata, err error,
357) {
358 span, ctx := ObserveFunction(ctx, "GetBlob", "blob.name", name)
359 if reader, metadata, err = backend.inner.GetBlob(ctx, name); err == nil {
360 ObserveData(ctx, "blob.size", metadata.Size)
361 blobsRetrievedCount.Inc()
362 blobsRetrievedBytes.Add(float64(metadata.Size))
363 }
364 span.Finish()
365 return
366}
367
368func (backend *observedBackend) PutBlob(ctx context.Context, name string, data []byte) (err error) {
369 span, ctx := ObserveFunction(ctx, "PutBlob", "blob.name", name, "blob.size", len(data))
370 if err = backend.inner.PutBlob(ctx, name, data); err == nil {
371 blobsStoredCount.Inc()
372 blobsStoredBytes.Add(float64(len(data)))
373 }
374 span.Finish()
375 return
376}
377
378func (backend *observedBackend) DeleteBlob(ctx context.Context, name string) (err error) {
379 span, ctx := ObserveFunction(ctx, "DeleteBlob", "blob.name", name)
380 err = backend.inner.DeleteBlob(ctx, name)
381 span.Finish()
382 return
383}
384
385func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
386 return func(yield func(BlobMetadata, error) bool) {
387 span, ctx := ObserveFunction(ctx, "EnumerateBlobs")
388 for metadata, err := range backend.inner.EnumerateBlobs(ctx) {
389 if !yield(metadata, err) {
390 break
391 }
392 }
393 span.Finish()
394 }
395}
396
397func (backend *observedBackend) GetManifest(
398 ctx context.Context, name string, opts GetManifestOptions,
399) (
400 manifest *Manifest, metadata ManifestMetadata, err error,
401) {
402 span, ctx := ObserveFunction(ctx, "GetManifest",
403 "manifest.name", name,
404 "manifest.bypass_cache", opts.BypassCache,
405 )
406 if manifest, metadata, err = backend.inner.GetManifest(ctx, name, opts); err == nil {
407 manifestsRetrievedCount.Inc()
408 }
409 span.Finish()
410 return
411}
412
413func (backend *observedBackend) StageManifest(ctx context.Context, manifest *Manifest) (err error) {
414 span, ctx := ObserveFunction(ctx, "StageManifest")
415 err = backend.inner.StageManifest(ctx, manifest)
416 span.Finish()
417 return
418}
419
420func (backend *observedBackend) HasAtomicCAS(ctx context.Context) bool {
421 return backend.inner.HasAtomicCAS(ctx)
422}
423
424func (backend *observedBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) (err error) {
425 span, ctx := ObserveFunction(ctx, "CommitManifest", "manifest.name", name)
426 err = backend.inner.CommitManifest(ctx, name, manifest, opts)
427 span.Finish()
428 return
429}
430
431func (backend *observedBackend) DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) (err error) {
432 span, ctx := ObserveFunction(ctx, "DeleteManifest", "manifest.name", name)
433 err = backend.inner.DeleteManifest(ctx, name, opts)
434 span.Finish()
435 return
436}
437
438func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
439 return func(yield func(ManifestMetadata, error) bool) {
440 span, ctx := ObserveFunction(ctx, "EnumerateManifests")
441 for metadata, err := range backend.inner.EnumerateManifests(ctx) {
442 if !yield(metadata, err) {
443 break
444 }
445 }
446 span.Finish()
447 }
448}
449
450func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) {
451 span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain)
452 found, err = backend.inner.CheckDomain(ctx, domain)
453 span.Finish()
454 return
455}
456
457func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) {
458 span, ctx := ObserveFunction(ctx, "CreateDomain", "domain.name", domain)
459 err = backend.inner.CreateDomain(ctx, domain)
460 span.Finish()
461 return
462}
463
464func (backend *observedBackend) FreezeDomain(ctx context.Context, domain string) (err error) {
465 span, ctx := ObserveFunction(ctx, "FreezeDomain", "domain.name", domain)
466 err = backend.inner.FreezeDomain(ctx, domain)
467 span.Finish()
468 return
469}
470
471func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) {
472 span, ctx := ObserveFunction(ctx, "UnfreezeDomain", "domain.name", domain)
473 err = backend.inner.UnfreezeDomain(ctx, domain)
474 span.Finish()
475 return
476}
477
478func (backend *observedBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) (err error) {
479 span, ctx := ObserveFunction(ctx, "AppendAuditLog", "audit.id", id)
480 err = backend.inner.AppendAuditLog(ctx, id, record)
481 span.Finish()
482 return
483}
484
485func (backend *observedBackend) QueryAuditLog(ctx context.Context, id AuditID) (record *AuditRecord, err error) {
486 span, ctx := ObserveFunction(ctx, "QueryAuditLog", "audit.id", id)
487 record, err = backend.inner.QueryAuditLog(ctx, id)
488 span.Finish()
489 return
490}
491
492func (backend *observedBackend) SearchAuditLog(
493 ctx context.Context, opts SearchAuditLogOptions,
494) iter.Seq2[AuditID, error] {
495 return func(yield func(AuditID, error) bool) {
496 span, ctx := ObserveFunction(ctx, "SearchAuditLog",
497 "audit.search.since", opts.Since,
498 "audit.search.until", opts.Until,
499 )
500 for id, err := range backend.inner.SearchAuditLog(ctx, opts) {
501 if !yield(id, err) {
502 break
503 }
504 }
505 span.Finish()
506 }
507}