techaro.lol
Weighs the soul of incoming HTTP requests to stop AI crawlers
feat(config): add ability to customize HTTP status codes Anubis returns (#393)
Signed-off-by: Xe Iaso <me@xeiaso.net>
+242
-9
+8
data/botPolicies.yaml
+8
data/botPolicies.yaml
···
48
action: CHALLENGE
49
50
dnsbl: false
51
+
52
+
# By default, send HTTP 200 back to clients that either get issued a challenge
53
+
# or a denial. This seems weird, but this is load-bearing due to the fact that
54
+
# the most aggressive scraper bots seem to really really want an HTTP 200 and
55
+
# will stop sending requests once they get it.
56
+
status_codes:
57
+
CHALLENGE: 200
58
+
DENY: 200
+2
docs/docs/CHANGELOG.md
+2
docs/docs/CHANGELOG.md
···
11
12
## [Unreleased]
13
14
+
- Added the ability to [customize Anubis' HTTP status codes](./admin/configuration/custom-status-codes.mdx) ([#355](https://github.com/TecharoHQ/anubis/issues/355))
15
+
16
## v1.17.0: Asahi sas Brutus
17
18
- Ensure regexes can't end in newlines ([#372](https://github.com/TecharoHQ/anubis/issues/372))
+19
docs/docs/admin/configuration/custom-status-codes.mdx
+19
docs/docs/admin/configuration/custom-status-codes.mdx
···
···
1
+
# Custom status codes for Anubis errors
2
+
3
+
Out of the box, Anubis will reply with `HTTP 200` for challenge and denial pages. This is intended to make AI scrapers have a hard time with your website because when they are faced with a non-200 response, they will hammer the page over and over until they get a 200 response. This behavior may not be desirable, as such Anubis lets you customize what HTTP status codes are returned when Anubis throws challenge and denial pages.
4
+
5
+
This is configured in the `status_codes` block of your [bot policy file](../policies.mdx):
6
+
7
+
```yaml
8
+
status_codes:
9
+
CHALLENGE: 200
10
+
DENY: 200
11
+
```
12
+
13
+
To match CloudFlare's behavior, use a configuration like this:
14
+
15
+
```yaml
16
+
status_codes:
17
+
CHALLENGE: 403
18
+
DENY: 403
19
+
```
+2
-2
lib/anubis.go
+2
-2
lib/anubis.go
···
170
hash := rule.Hash()
171
172
lg.Debug("rule hash", "hash", hash)
173
-
s.respondWithStatus(w, r, fmt.Sprintf("Access Denied: error code %s", hash), http.StatusOK)
174
return true
175
case config.RuleChallenge:
176
lg.Debug("challenge requested")
···
202
203
if resp != dnsbl.AllGood {
204
lg.Info("DNSBL hit", "status", resp.String())
205
-
s.respondWithStatus(w, r, fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip), http.StatusOK)
206
return true
207
}
208
}
···
170
hash := rule.Hash()
171
172
lg.Debug("rule hash", "hash", hash)
173
+
s.respondWithStatus(w, r, fmt.Sprintf("Access Denied: error code %s", hash), s.policy.StatusCodes.Deny)
174
return true
175
case config.RuleChallenge:
176
lg.Debug("challenge requested")
···
202
203
if resp != dnsbl.AllGood {
204
lg.Info("DNSBL hit", "status", resp.String())
205
+
s.respondWithStatus(w, r, fmt.Sprintf("DroneBL reported an entry: %s, see https://dronebl.org/lookup?ip=%s", resp.String(), ip), s.policy.StatusCodes.Deny)
206
return true
207
}
208
}
+45
lib/anubis_test.go
+45
lib/anubis_test.go
···
393
})
394
}
395
}
396
+
397
+
func TestCustomStatusCodes(t *testing.T) {
398
+
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
399
+
t.Log(r.UserAgent())
400
+
w.WriteHeader(http.StatusOK)
401
+
fmt.Fprintln(w, "OK")
402
+
})
403
+
404
+
statusMap := map[string]int{
405
+
"ALLOW": 200,
406
+
"CHALLENGE": 401,
407
+
"DENY": 403,
408
+
}
409
+
410
+
pol := loadPolicies(t, "./testdata/aggressive_403.yaml")
411
+
pol.DefaultDifficulty = 4
412
+
413
+
srv := spawnAnubis(t, Options{
414
+
Next: h,
415
+
Policy: pol,
416
+
})
417
+
418
+
ts := httptest.NewServer(internal.RemoteXRealIP(true, "tcp", srv))
419
+
defer ts.Close()
420
+
421
+
for userAgent, statusCode := range statusMap {
422
+
t.Run(userAgent, func(t *testing.T) {
423
+
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL, nil)
424
+
if err != nil {
425
+
t.Fatal(err)
426
+
}
427
+
428
+
req.Header.Set("User-Agent", userAgent)
429
+
430
+
resp, err := ts.Client().Do(req)
431
+
if err != nil {
432
+
t.Fatal(err)
433
+
}
434
+
435
+
if resp.StatusCode != statusCode {
436
+
t.Errorf("wanted status code %d but got: %d", statusCode, resp.StatusCode)
437
+
}
438
+
})
439
+
}
440
+
}
+4
-1
lib/http.go
+4
-1
lib/http.go
+41
-5
lib/policy/config/config.go
+41
-5
lib/policy/config/config.go
···
6
"io"
7
"io/fs"
8
"net"
9
"os"
10
"regexp"
11
"strings"
···
28
ErrInvalidImportStatement = errors.New("config.ImportStatement: invalid source file")
29
ErrCantSetBotAndImportValuesAtOnce = errors.New("config.BotOrImport: can't set bot rules and import values at the same time")
30
ErrMustSetBotOrImportRules = errors.New("config.BotOrImport: rule definition is invalid, you must set either bot rules or an import statement, not both")
31
)
32
33
type Rule string
···
262
return ErrMustSetBotOrImportRules
263
}
264
265
type fileConfig struct {
266
-
Bots []BotOrImport `json:"bots"`
267
-
DNSBL bool `json:"dnsbl"`
268
}
269
270
func (c fileConfig) Valid() error {
···
280
}
281
}
282
283
if len(errs) != 0 {
284
return fmt.Errorf("config is not valid:\n%w", errors.Join(errs...))
285
}
···
289
290
func Load(fin io.Reader, fname string) (*Config, error) {
291
var c fileConfig
292
if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil {
293
return nil, fmt.Errorf("can't parse policy config YAML %s: %w", fname, err)
294
}
···
298
}
299
300
result := &Config{
301
-
DNSBL: c.DNSBL,
302
}
303
304
var validationErrs []error
···
331
}
332
333
type Config struct {
334
-
Bots []BotConfig
335
-
DNSBL bool
336
}
337
338
func (c Config) Valid() error {
···
6
"io"
7
"io/fs"
8
"net"
9
+
"net/http"
10
"os"
11
"regexp"
12
"strings"
···
29
ErrInvalidImportStatement = errors.New("config.ImportStatement: invalid source file")
30
ErrCantSetBotAndImportValuesAtOnce = errors.New("config.BotOrImport: can't set bot rules and import values at the same time")
31
ErrMustSetBotOrImportRules = errors.New("config.BotOrImport: rule definition is invalid, you must set either bot rules or an import statement, not both")
32
+
ErrStatusCodeNotValid = errors.New("config.StatusCode: status code not valid, must be between 100 and 599")
33
)
34
35
type Rule string
···
264
return ErrMustSetBotOrImportRules
265
}
266
267
+
type StatusCodes struct {
268
+
Challenge int `json:"CHALLENGE"`
269
+
Deny int `json:"DENY"`
270
+
}
271
+
272
+
func (sc StatusCodes) Valid() error {
273
+
var errs []error
274
+
275
+
if sc.Challenge == 0 || (sc.Challenge < 100 && sc.Challenge >= 599) {
276
+
errs = append(errs, fmt.Errorf("%w: challenge is %d", ErrStatusCodeNotValid, sc.Challenge))
277
+
}
278
+
279
+
if sc.Deny == 0 || (sc.Deny < 100 && sc.Deny >= 599) {
280
+
errs = append(errs, fmt.Errorf("%w: deny is %d", ErrStatusCodeNotValid, sc.Deny))
281
+
}
282
+
283
+
if len(errs) != 0 {
284
+
return fmt.Errorf("status codes not valid:\n%w", errors.Join(errs...))
285
+
}
286
+
287
+
return nil
288
+
}
289
+
290
type fileConfig struct {
291
+
Bots []BotOrImport `json:"bots"`
292
+
DNSBL bool `json:"dnsbl"`
293
+
StatusCodes StatusCodes `json:"status_codes"`
294
}
295
296
func (c fileConfig) Valid() error {
···
306
}
307
}
308
309
+
if err := c.StatusCodes.Valid(); err != nil {
310
+
errs = append(errs, err)
311
+
}
312
+
313
if len(errs) != 0 {
314
return fmt.Errorf("config is not valid:\n%w", errors.Join(errs...))
315
}
···
319
320
func Load(fin io.Reader, fname string) (*Config, error) {
321
var c fileConfig
322
+
c.StatusCodes = StatusCodes{
323
+
Challenge: http.StatusOK,
324
+
Deny: http.StatusOK,
325
+
}
326
if err := yaml.NewYAMLToJSONDecoder(fin).Decode(&c); err != nil {
327
return nil, fmt.Errorf("can't parse policy config YAML %s: %w", fname, err)
328
}
···
332
}
333
334
result := &Config{
335
+
DNSBL: c.DNSBL,
336
+
StatusCodes: c.StatusCodes,
337
}
338
339
var validationErrs []error
···
366
}
367
368
type Config struct {
369
+
Bots []BotConfig
370
+
DNSBL bool
371
+
StatusCodes StatusCodes
372
}
373
374
func (c Config) Valid() error {
+13
lib/policy/config/testdata/bad/status-codes-0.json
+13
lib/policy/config/testdata/bad/status-codes-0.json
+8
lib/policy/config/testdata/bad/status-codes-0.yaml
+8
lib/policy/config/testdata/bad/status-codes-0.yaml
+13
lib/policy/config/testdata/good/status-codes-paranoid.json
+13
lib/policy/config/testdata/good/status-codes-paranoid.json
+8
lib/policy/config/testdata/good/status-codes-paranoid.yaml
+8
lib/policy/config/testdata/good/status-codes-paranoid.yaml
+13
lib/policy/config/testdata/good/status-codes-rfc.json
+13
lib/policy/config/testdata/good/status-codes-rfc.json
+8
lib/policy/config/testdata/good/status-codes-rfc.yaml
+8
lib/policy/config/testdata/good/status-codes-rfc.yaml
+3
-1
lib/policy/policy.go
+3
-1
lib/policy/policy.go
+12
lib/testdata/aggressive_403.yaml
+12
lib/testdata/aggressive_403.yaml
+12
test/anubis_configs/aggressive_403.yaml
+12
test/anubis_configs/aggressive_403.yaml
+1
test/unix-socket-xff/start.sh
+1
test/unix-socket-xff/start.sh
+30
test/unix-socket-xff/test.mjs
+30
test/unix-socket-xff/test.mjs
···
···
1
+
async function testWithUserAgent(userAgent) {
2
+
const statusCode =
3
+
await fetch("https://relayd.local.cetacean.club:3004/reqmeta", {
4
+
headers: {
5
+
"User-Agent": userAgent,
6
+
}
7
+
})
8
+
.then(resp => resp.status);
9
+
return statusCode;
10
+
}
11
+
12
+
const codes = {
13
+
allow: await testWithUserAgent("ALLOW"),
14
+
challenge: await testWithUserAgent("CHALLENGE"),
15
+
deny: await testWithUserAgent("DENY")
16
+
}
17
+
18
+
const expected = {
19
+
allow: 200,
20
+
challenge: 401,
21
+
deny: 403,
22
+
};
23
+
24
+
console.log("ALLOW: ", codes.allow);
25
+
console.log("CHALLENGE:", codes.challenge);
26
+
console.log("DENY: ", codes.deny);
27
+
28
+
if (JSON.stringify(codes) !== JSON.stringify(expected)) {
29
+
throw new Error(`wanted ${JSON.stringify(expected)}, got: ${JSON.stringify(codes)}`);
30
+
}