Weighs the soul of incoming HTTP requests to stop AI crawlers

Apply bits of the cookie settings PR one by one (#140)

Enables uses to change the cookie domain and partitioned flags.

Signed-off-by: Xe Iaso <me@xeiaso.net>

authored by

Xe Iaso and committed by
GitHub
7d4be0dc d1d63d9c

+168 -56
+6 -1
.github/workflows/go.yml
··· 64 64 ~/.cache/ms-playwright 65 65 key: ${{ runner.os }}-playwright-${{ hashFiles('**/go.sum') }} 66 66 67 + - name: install playwright browsers 68 + run: | 69 + npx --yes playwright@1.50.1 install --with-deps 70 + npx --yes playwright@1.50.1 run-server --port 3000 & 71 + 67 72 - name: Build 68 73 run: go build ./... 69 74 70 75 - name: Test 71 - run: go test ./... 76 + run: go test -v ./...
+8 -4
cmd/anubis/main.go
··· 34 34 bind = flag.String("bind", ":8923", "network address to bind HTTP to") 35 35 bindNetwork = flag.String("bind-network", "tcp", "network family to bind HTTP to, e.g. unix, tcp") 36 36 challengeDifficulty = flag.Int("difficulty", anubis.DefaultDifficulty, "difficulty of the challenge") 37 + cookieDomain = flag.String("cookie-domain", "", "if set, the top-level domain that the Anubis cookie will be valid for") 38 + cookiePartitioned = flag.Bool("cookie-partitioned", false, "if true, sets the partitioned flag on Anubis cookies, enabling CHIPS support") 37 39 ed25519PrivateKeyHex = flag.String("ed25519-private-key-hex", "", "private key used to sign JWTs, if not set a random one will be assigned") 38 40 metricsBind = flag.String("metrics-bind", ":9090", "network address to bind metrics to") 39 41 metricsBindNetwork = flag.String("metrics-bind-network", "tcp", "network family for the metrics server to bind to") ··· 189 191 } 190 192 191 193 s, err := libanubis.New(libanubis.Options{ 192 - Next: rp, 193 - Policy: policy, 194 - ServeRobotsTXT: *robotsTxt, 195 - PrivateKey: priv, 194 + Next: rp, 195 + Policy: policy, 196 + ServeRobotsTXT: *robotsTxt, 197 + PrivateKey: priv, 198 + CookieDomain: *cookieDomain, 199 + CookiePartitioned: *cookiePartitioned, 196 200 }) 197 201 if err != nil { 198 202 log.Fatalf("can't construct libanubis.Server: %v", err)
+3
docs/docs/CHANGELOG.md
··· 19 19 - Fix default difficulty setting that was broken in a refactor 20 20 - Linting fixes 21 21 - Make dark mode diff lines readable in the documentation 22 + - Add the ability to set the cookie domain with the envvar `COOKIE_DOMAIN=techaro.lol` for all domains under `techaro.lol` 23 + - Add the ability to set the cookie partitioned flag with the envvar `COOKIE_PARTITIONED=true` 24 + - Fix CI based browser smoke test 22 25 23 26 ## v1.14.2 24 27
+2
docs/docs/admin/installation.mdx
··· 45 45 | :------------------------ | :---------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 46 46 | `BIND` | `:8923` | The network address that Anubis listens on. For `unix`, set this to a path: `/run/anubis/instance.sock` | 47 47 | `BIND_NETWORK` | `tcp` | The address family that Anubis listens on. Accepts `tcp`, `unix` and anything Go's [`net.Listen`](https://pkg.go.dev/net#Listen) supports. | 48 + | `COOKIE_DOMAIN` | unset | The domain the Anubis challenge pass cookie should be set to. This should be set to the domain you bought from your registrar (EG: `techaro.lol` if your webapp is running on `anubis.techaro.lol`). See [here](https://stackoverflow.com/a/1063760) for more information. | 49 + | `COOKIE_PARTITIONED` | `false` | If set to `true`, enables the [partitioned (CHIPS) flag](https://developers.google.com/privacy-sandbox/cookies/chips), meaning that Anubis inside an iframe has a different set of cookies than the domain hosting the iframe. | 48 50 | `DIFFICULTY` | `5` | The difficulty of the challenge, or the number of leading zeroes that must be in successful responses. | 49 51 | `ED25519_PRIVATE_KEY_HEX` | | The hex-encoded ed25519 private key used to sign Anubis responses. If this is not set, Anubis will generate one for you. This should be exactly 64 characters long. See below for details. | 50 52 | `METRICS_BIND` | `:9090` | The network address that Anubis serves Prometheus metrics on. See `BIND` for more information. |
+22 -18
internal/test/playwright_test.go
··· 166 166 } 167 167 168 168 func TestPlaywrightBrowser(t *testing.T) { 169 - if os.Getenv("CI") == "true" { 170 - t.Skip("XXX(Xe): This is broken in CI, will fix later") 171 - } 172 - 173 169 if os.Getenv("DONT_USE_NETWORK") != "" { 174 170 t.Skip("test requires network egress") 175 171 return ··· 225 221 t.Skip("skipping hard challenge with deadline") 226 222 } 227 223 228 - perfomedAction := executeTestCase(t, tc, typ, anubisURL) 229 - 224 + var perfomedAction action 225 + var err error 226 + for i := 0; i < 5; i++ { 227 + perfomedAction, err = executeTestCase(t, tc, typ, anubisURL) 228 + if perfomedAction == tc.action { 229 + break 230 + } 231 + time.Sleep(time.Duration(i+1) * 250 * time.Millisecond) 232 + } 230 233 if perfomedAction != tc.action { 231 234 t.Errorf("unexpected test result, expected %s, got %s", tc.action, perfomedAction) 232 - } else { 233 - t.Logf("test passed") 235 + } 236 + if err != nil { 237 + t.Fatalf("test error: %v", err) 234 238 } 235 239 }) 236 240 } ··· 247 251 return u.String() 248 252 } 249 253 250 - func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType, anubisURL string) action { 254 + func executeTestCase(t *testing.T, tc testCase, typ playwright.BrowserType, anubisURL string) (action, error) { 251 255 deadline, _ := t.Deadline() 252 256 253 257 browser, err := typ.Connect(buildBrowserConnect(typ.Name()), playwright.BrowserTypeConnectOptions{ 254 258 ExposeNetwork: playwright.String("<loopback>"), 255 259 }) 256 260 if err != nil { 257 - t.Fatalf("could not connect to remote browser: %v", err) 261 + return "", fmt.Errorf("could not connect to remote browser: %w", err) 258 262 } 259 263 defer browser.Close() 260 264 ··· 266 270 UserAgent: playwright.String(tc.userAgent), 267 271 }) 268 272 if err != nil { 269 - t.Fatalf("could not create context: %v", err) 273 + return "", fmt.Errorf("could not create context: %w", err) 270 274 } 271 275 defer ctx.Close() 272 276 273 277 page, err := ctx.NewPage() 274 278 if err != nil { 275 - t.Fatalf("could not create page: %v", err) 279 + return "", fmt.Errorf("could not create page: %w", err) 276 280 } 277 281 defer page.Close() 278 282 ··· 283 287 Timeout: pwTimeout(tc, deadline), 284 288 }) 285 289 if err != nil { 286 - pwFail(t, page, "could not navigate to test server: %v", err) 290 + return "", pwFail(t, page, "could not navigate to test server: %v", err) 287 291 } 288 292 289 293 hadChallenge := false ··· 294 298 hadChallenge = true 295 299 case actionDeny: 296 300 checkImage(t, tc, deadline, page, "#image[src*=sad]") 297 - return actionDeny 301 + return actionDeny, nil 298 302 } 299 303 300 304 // Ensure protected resource was provided. ··· 317 321 } 318 322 319 323 if hadChallenge { 320 - return actionChallenge 324 + return actionChallenge, nil 321 325 } else { 322 - return actionAllow 326 + return actionAllow, nil 323 327 } 324 328 } 325 329 ··· 342 346 } 343 347 } 344 348 345 - func pwFail(t *testing.T, page playwright.Page, format string, args ...any) { 349 + func pwFail(t *testing.T, page playwright.Page, format string, args ...any) error { 346 350 t.Helper() 347 351 348 352 saveScreenshot(t, page) 349 - t.Fatalf(format, args...) 353 + return fmt.Errorf(format, args...) 350 354 } 351 355 352 356 func pwTimeout(tc testCase, deadline time.Time) *float64 {
+3
internal/test/var/.gitignore
··· 1 + *.png 2 + *.txt 3 + *.html
+29 -21
lib/anubis.go
··· 67 67 Policy *policy.ParsedConfig 68 68 ServeRobotsTXT bool 69 69 PrivateKey ed25519.PrivateKey 70 + 71 + CookieDomain string 72 + CookieName string 73 + CookiePartitioned bool 70 74 } 71 75 72 76 func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) { ··· 108 112 priv: opts.PrivateKey, 109 113 pub: opts.PrivateKey.Public().(ed25519.PublicKey), 110 114 policy: opts.Policy, 115 + opts: opts, 111 116 DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](), 112 117 } 113 118 ··· 145 150 priv ed25519.PrivateKey 146 151 pub ed25519.PublicKey 147 152 policy *policy.ParsedConfig 153 + opts Options 148 154 DNSBLCache *decaymap.Impl[string, dnsbl.DroneBLResponse] 149 155 ChallengeDifficulty int 150 156 } ··· 217 223 s.next.ServeHTTP(w, r) 218 224 return 219 225 case config.RuleDeny: 220 - ClearCookie(w) 226 + s.ClearCookie(w) 221 227 lg.Info("explicit deny") 222 228 if rule == nil { 223 229 lg.Error("rule is nil, cannot calculate checksum") ··· 236 242 case config.RuleChallenge: 237 243 lg.Debug("challenge requested") 238 244 default: 239 - ClearCookie(w) 245 + s.ClearCookie(w) 240 246 templ.Handler(web.Base("Oh noes!", web.ErrorPage("Other internal server error (contact the admin)")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) 241 247 return 242 248 } ··· 244 250 ckie, err := r.Cookie(anubis.CookieName) 245 251 if err != nil { 246 252 lg.Debug("cookie not found", "path", r.URL.Path) 247 - ClearCookie(w) 253 + s.ClearCookie(w) 248 254 s.RenderIndex(w, r) 249 255 return 250 256 } 251 257 252 258 if err := ckie.Valid(); err != nil { 253 259 lg.Debug("cookie is invalid", "err", err) 254 - ClearCookie(w) 260 + s.ClearCookie(w) 255 261 s.RenderIndex(w, r) 256 262 return 257 263 } 258 264 259 265 if time.Now().After(ckie.Expires) && !ckie.Expires.IsZero() { 260 266 lg.Debug("cookie expired", "path", r.URL.Path) 261 - ClearCookie(w) 267 + s.ClearCookie(w) 262 268 s.RenderIndex(w, r) 263 269 return 264 270 } ··· 269 275 270 276 if err != nil || !token.Valid { 271 277 lg.Debug("invalid token", "path", r.URL.Path, "err", err) 272 - ClearCookie(w) 278 + s.ClearCookie(w) 273 279 s.RenderIndex(w, r) 274 280 return 275 281 } ··· 284 290 claims, ok := token.Claims.(jwt.MapClaims) 285 291 if !ok { 286 292 lg.Debug("invalid token claims type", "path", r.URL.Path) 287 - ClearCookie(w) 293 + s.ClearCookie(w) 288 294 s.RenderIndex(w, r) 289 295 return 290 296 } ··· 292 298 293 299 if claims["challenge"] != challenge { 294 300 lg.Debug("invalid challenge", "path", r.URL.Path) 295 - ClearCookie(w) 301 + s.ClearCookie(w) 296 302 s.RenderIndex(w, r) 297 303 return 298 304 } ··· 309 315 if subtle.ConstantTimeCompare([]byte(claims["response"].(string)), []byte(calculated)) != 1 { 310 316 lg.Debug("invalid response", "path", r.URL.Path) 311 317 failedValidations.Inc() 312 - ClearCookie(w) 318 + s.ClearCookie(w) 313 319 s.RenderIndex(w, r) 314 320 return 315 321 } ··· 372 378 373 379 nonceStr := r.FormValue("nonce") 374 380 if nonceStr == "" { 375 - ClearCookie(w) 381 + s.ClearCookie(w) 376 382 lg.Debug("no nonce") 377 383 templ.Handler(web.Base("Oh noes!", web.ErrorPage("missing nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) 378 384 return ··· 380 386 381 387 elapsedTimeStr := r.FormValue("elapsedTime") 382 388 if elapsedTimeStr == "" { 383 - ClearCookie(w) 389 + s.ClearCookie(w) 384 390 lg.Debug("no elapsedTime") 385 391 templ.Handler(web.Base("Oh noes!", web.ErrorPage("missing elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) 386 392 return ··· 388 394 389 395 elapsedTime, err := strconv.ParseFloat(elapsedTimeStr, 64) 390 396 if err != nil { 391 - ClearCookie(w) 397 + s.ClearCookie(w) 392 398 lg.Debug("elapsedTime doesn't parse", "err", err) 393 399 templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid elapsedTime")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) 394 400 return ··· 404 410 405 411 nonce, err := strconv.Atoi(nonceStr) 406 412 if err != nil { 407 - ClearCookie(w) 413 + s.ClearCookie(w) 408 414 lg.Debug("nonce doesn't parse", "err", err) 409 415 templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid nonce")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) 410 416 return ··· 414 420 calculated := internal.SHA256sum(calcString) 415 421 416 422 if subtle.ConstantTimeCompare([]byte(response), []byte(calculated)) != 1 { 417 - ClearCookie(w) 423 + s.ClearCookie(w) 418 424 lg.Debug("hash does not match", "got", response, "want", calculated) 419 425 templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r) 420 426 failedValidations.Inc() ··· 423 429 424 430 // compare the leading zeroes 425 431 if !strings.HasPrefix(response, strings.Repeat("0", s.ChallengeDifficulty)) { 426 - ClearCookie(w) 432 + s.ClearCookie(w) 427 433 lg.Debug("difficulty check failed", "response", response, "difficulty", s.ChallengeDifficulty) 428 434 templ.Handler(web.Base("Oh noes!", web.ErrorPage("invalid response")), templ.WithStatus(http.StatusForbidden)).ServeHTTP(w, r) 429 435 failedValidations.Inc() ··· 442 448 tokenString, err := token.SignedString(s.priv) 443 449 if err != nil { 444 450 lg.Error("failed to sign JWT", "err", err) 445 - ClearCookie(w) 451 + s.ClearCookie(w) 446 452 templ.Handler(web.Base("Oh noes!", web.ErrorPage("failed to sign JWT")), templ.WithStatus(http.StatusInternalServerError)).ServeHTTP(w, r) 447 453 return 448 454 } 449 455 450 456 http.SetCookie(w, &http.Cookie{ 451 - Name: anubis.CookieName, 452 - Value: tokenString, 453 - Expires: time.Now().Add(24 * 7 * time.Hour), 454 - SameSite: http.SameSiteLaxMode, 455 - Path: "/", 457 + Name: anubis.CookieName, 458 + Value: tokenString, 459 + Expires: time.Now().Add(24 * 7 * time.Hour), 460 + SameSite: http.SameSiteLaxMode, 461 + Domain: s.opts.CookieDomain, 462 + Partitioned: s.opts.CookiePartitioned, 463 + Path: "/", 456 464 }) 457 465 458 466 challengesValidated.Inc()
+93 -11
lib/anubis_test.go
··· 1 1 package lib 2 2 3 3 import ( 4 + "encoding/json" 4 5 "fmt" 5 6 "net/http" 6 7 "net/http/httptest" 7 8 "testing" 8 9 9 10 "github.com/TecharoHQ/anubis" 11 + "github.com/TecharoHQ/anubis/internal" 12 + "github.com/TecharoHQ/anubis/lib/policy" 10 13 ) 11 14 12 - func spawnAnubis(t *testing.T, h http.Handler) string { 15 + func loadPolicies(t *testing.T, fname string) *policy.ParsedConfig { 13 16 t.Helper() 14 17 15 18 policy, err := LoadPoliciesOrDefault("", anubis.DefaultDifficulty) ··· 17 20 t.Fatal(err) 18 21 } 19 22 20 - s, err := New(Options{ 21 - Next: h, 22 - Policy: policy, 23 - ServeRobotsTXT: true, 24 - }) 23 + return policy 24 + } 25 + 26 + func spawnAnubis(t *testing.T, opts Options) *Server { 27 + t.Helper() 28 + 29 + s, err := New(opts) 25 30 if err != nil { 26 31 t.Fatalf("can't construct libanubis.Server: %v", err) 27 32 } 28 33 29 - ts := httptest.NewServer(s) 30 - t.Log(ts.URL) 34 + return s 35 + } 31 36 32 - t.Cleanup(func() { 33 - ts.Close() 37 + func TestCookieSettings(t *testing.T) { 38 + pol := loadPolicies(t, "") 39 + pol.DefaultDifficulty = 0 40 + 41 + srv := spawnAnubis(t, Options{ 42 + Next: http.NewServeMux(), 43 + Policy: pol, 44 + 45 + CookieDomain: "local.cetacean.club", 46 + CookiePartitioned: true, 47 + CookieName: t.Name(), 34 48 }) 35 49 36 - return ts.URL 50 + ts := httptest.NewServer(internal.DefaultXRealIP("127.0.0.1", srv)) 51 + defer ts.Close() 52 + 53 + cli := &http.Client{ 54 + CheckRedirect: func(req *http.Request, via []*http.Request) error { 55 + return http.ErrUseLastResponse 56 + }, 57 + } 58 + 59 + resp, err := cli.Post(ts.URL+"/.within.website/x/cmd/anubis/api/make-challenge", "", nil) 60 + if err != nil { 61 + t.Fatalf("can't request challenge: %v", err) 62 + } 63 + defer resp.Body.Close() 64 + 65 + var chall = struct { 66 + Challenge string `json:"challenge"` 67 + }{} 68 + if err := json.NewDecoder(resp.Body).Decode(&chall); err != nil { 69 + t.Fatalf("can't read challenge response body: %v", err) 70 + } 71 + 72 + nonce := 0 73 + elapsedTime := 420 74 + redir := "/" 75 + calcString := fmt.Sprintf("%s%d", chall.Challenge, nonce) 76 + calculated := internal.SHA256sum(calcString) 77 + 78 + req, err := http.NewRequest(http.MethodGet, ts.URL+"/.within.website/x/cmd/anubis/api/pass-challenge", nil) 79 + if err != nil { 80 + t.Fatalf("can't make request: %v", err) 81 + } 82 + 83 + q := req.URL.Query() 84 + q.Set("response", calculated) 85 + q.Set("nonce", fmt.Sprint(nonce)) 86 + q.Set("redir", redir) 87 + q.Set("elapsedTime", fmt.Sprint(elapsedTime)) 88 + req.URL.RawQuery = q.Encode() 89 + 90 + resp, err = cli.Do(req) 91 + if err != nil { 92 + t.Fatalf("can't do challenge passing") 93 + } 94 + 95 + if resp.StatusCode != http.StatusFound { 96 + t.Errorf("wanted %d, got: %d", http.StatusFound, resp.StatusCode) 97 + } 98 + 99 + var ckie *http.Cookie 100 + for _, cookie := range resp.Cookies() { 101 + t.Logf("%#v", cookie) 102 + if cookie.Name == anubis.CookieName { 103 + ckie = cookie 104 + break 105 + } 106 + } 107 + 108 + if ckie.Domain != "local.cetacean.club" { 109 + t.Errorf("cookie domain is wrong, wanted local.cetacean.club, got: %s", ckie.Domain) 110 + } 111 + 112 + if ckie.Partitioned != srv.opts.CookiePartitioned { 113 + t.Errorf("wanted partitioned flag %v, got: %v", srv.opts.CookiePartitioned, ckie.Partitioned) 114 + } 115 + 116 + if ckie == nil { 117 + t.Errorf("Cookie %q not found", anubis.CookieName) 118 + } 37 119 } 38 120 39 121 func TestCheckDefaultDifficultyMatchesPolicy(t *testing.T) {
+2 -1
lib/http.go
··· 7 7 "github.com/TecharoHQ/anubis" 8 8 ) 9 9 10 - func ClearCookie(w http.ResponseWriter) { 10 + func (s *Server) ClearCookie(w http.ResponseWriter) { 11 11 http.SetCookie(w, &http.Cookie{ 12 12 Name: anubis.CookieName, 13 13 Value: "", 14 14 Expires: time.Now().Add(-1 * time.Hour), 15 15 MaxAge: -1, 16 16 SameSite: http.SameSiteLaxMode, 17 + Domain: s.opts.CookieDomain, 17 18 }) 18 19 } 19 20