techaro.lol
Weighs the soul of incoming HTTP requests to stop AI crawlers
fix: make ogtags and dnsbl use the Store instead of memory (#760)
Signed-off-by: Xe Iaso <me@xeiaso.net>
-15
cmd/anubis/main.go
-15
cmd/anubis/main.go
···
231
return rp, nil
232
}
233
234
-
func startDecayMapCleanup(ctx context.Context, s *libanubis.Server) {
235
-
ticker := time.NewTicker(1 * time.Hour)
236
-
defer ticker.Stop()
237
-
238
-
for {
239
-
select {
240
-
case <-ticker.C:
241
-
s.CleanupDecayMap()
242
-
case <-ctx.Done():
243
-
return
244
-
}
245
-
}
246
-
}
247
-
248
func main() {
249
flagenv.Parse()
250
flag.Parse()
···
421
wg.Add(1)
422
go metricsServer(ctx, wg.Done)
423
}
424
-
go startDecayMapCleanup(ctx, s)
425
426
var h http.Handler
427
h = s
+1
docs/docs/CHANGELOG.md
+1
docs/docs/CHANGELOG.md
···
24
- Remove the "Success" interstitial after a proof of work challenge is concluded.
25
- Anubis now has the concept of [storage backends](./admin/policies.mdx#storage-backends). These allow you to change how Anubis stores temporary data (in memory, on the disk, or in Valkey). If you run Anubis in an environment where you have a low amount of memory available for Anubis (eg: less than 64 megabytes), be sure to configure the [`bbolt`](./admin/policies.mdx#bbolt) storage backend.
26
- The challenge issuance and validation process has been rewritten from scratch. Instead of generating challenge strings from request metadata (under the assumption that the values being compared against are stable), Anubis now generates random data for each challenge. This data is stored in the active [storage backend](./admin/policies.mdx#storage-backends) for up to 30 minutes. Fixes [#564](https://github.com/TecharoHQ/anubis/issues/564), [#746](https://github.com/TecharoHQ/anubis/issues/746), and other similar instances of this issue.
27
- Add option for forcing a specific language ([#742](https://github.com/TecharoHQ/anubis/pull/742))
28
- Add translation for Turkish language ([#751](https://github.com/TecharoHQ/anubis/pull/751))
29
- Allow [Common Crawl](https://commoncrawl.org/) by default so scrapers have less incentive to scrape
···
24
- Remove the "Success" interstitial after a proof of work challenge is concluded.
25
- Anubis now has the concept of [storage backends](./admin/policies.mdx#storage-backends). These allow you to change how Anubis stores temporary data (in memory, on the disk, or in Valkey). If you run Anubis in an environment where you have a low amount of memory available for Anubis (eg: less than 64 megabytes), be sure to configure the [`bbolt`](./admin/policies.mdx#bbolt) storage backend.
26
- The challenge issuance and validation process has been rewritten from scratch. Instead of generating challenge strings from request metadata (under the assumption that the values being compared against are stable), Anubis now generates random data for each challenge. This data is stored in the active [storage backend](./admin/policies.mdx#storage-backends) for up to 30 minutes. Fixes [#564](https://github.com/TecharoHQ/anubis/issues/564), [#746](https://github.com/TecharoHQ/anubis/issues/746), and other similar instances of this issue.
27
+
- Make the [Open Graph](./admin/configuration/open-graph.mdx) subsystem and DNSBL subsystem use [storage backends](./admin/policies.mdx#storage-backends) instead of storing everything in memory by default.
28
- Add option for forcing a specific language ([#742](https://github.com/TecharoHQ/anubis/pull/742))
29
- Add translation for Turkish language ([#751](https://github.com/TecharoHQ/anubis/pull/751))
30
- Allow [Common Crawl](https://commoncrawl.org/) by default so scrapers have less incentive to scrape
+4
-9
lib/anubis.go
+4
-9
lib/anubis.go
···
70
next http.Handler
71
mux *http.ServeMux
72
policy *policy.ParsedConfig
73
-
DNSBLCache *decaymap.Impl[string, dnsbl.DroneBLResponse]
74
OGTags *ogtags.OGTagCache
75
ed25519Priv ed25519.PrivateKey
76
hs512Secret []byte
···
279
}
280
281
func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string, lg *slog.Logger) bool {
282
if s.policy.DNSBL && ip != "" {
283
-
resp, ok := s.DNSBLCache.Get(ip)
284
-
if !ok {
285
lg.Debug("looking up ip in dnsbl")
286
resp, err := dnsbl.Lookup(ip)
287
if err != nil {
288
lg.Error("can't look up ip in dnsbl", "err", err)
289
}
290
-
s.DNSBLCache.Set(ip, resp, 24*time.Hour)
291
droneBLHits.WithLabelValues(resp.String()).Inc()
292
}
293
···
551
Rules: &checker.List{},
552
}, nil
553
}
554
-
555
-
func (s *Server) CleanupDecayMap() {
556
-
s.DNSBLCache.Cleanup()
557
-
s.OGTags.Cleanup()
558
-
}
···
70
next http.Handler
71
mux *http.ServeMux
72
policy *policy.ParsedConfig
73
OGTags *ogtags.OGTagCache
74
ed25519Priv ed25519.PrivateKey
75
hs512Secret []byte
···
278
}
279
280
func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string, lg *slog.Logger) bool {
281
+
db := &store.JSON[dnsbl.DroneBLResponse]{Underlying: s.store, Prefix: "dronebl:"}
282
if s.policy.DNSBL && ip != "" {
283
+
resp, err := db.Get(r.Context(), ip)
284
+
if err != nil {
285
lg.Debug("looking up ip in dnsbl")
286
resp, err := dnsbl.Lookup(ip)
287
if err != nil {
288
lg.Error("can't look up ip in dnsbl", "err", err)
289
}
290
+
db.Set(r.Context(), ip, resp, 24*time.Hour)
291
droneBLHits.WithLabelValues(resp.String()).Inc()
292
}
293
···
551
Rules: &checker.List{},
552
}, nil
553
}
+1
-4
lib/config.go
+1
-4
lib/config.go
···
15
16
"github.com/TecharoHQ/anubis"
17
"github.com/TecharoHQ/anubis/data"
18
-
"github.com/TecharoHQ/anubis/decaymap"
19
"github.com/TecharoHQ/anubis/internal"
20
-
"github.com/TecharoHQ/anubis/internal/dnsbl"
21
"github.com/TecharoHQ/anubis/internal/ogtags"
22
"github.com/TecharoHQ/anubis/lib/challenge"
23
"github.com/TecharoHQ/anubis/lib/localization"
···
108
hs512Secret: opts.HS512Secret,
109
policy: opts.Policy,
110
opts: opts,
111
-
DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](),
112
-
OGTags: ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph),
113
store: opts.Policy.Store,
114
}
115
···
15
16
"github.com/TecharoHQ/anubis"
17
"github.com/TecharoHQ/anubis/data"
18
"github.com/TecharoHQ/anubis/internal"
19
"github.com/TecharoHQ/anubis/internal/ogtags"
20
"github.com/TecharoHQ/anubis/lib/challenge"
21
"github.com/TecharoHQ/anubis/lib/localization"
···
106
hs512Secret: opts.HS512Secret,
107
policy: opts.Policy,
108
opts: opts,
109
+
OGTags: ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph, opts.Policy.Store),
110
store: opts.Policy.Store,
111
}
112
+1
-1
lib/http.go
+1
-1
lib/http.go
+13
lib/store/interface.go
+13
lib/store/interface.go
···
43
44
type JSON[T any] struct {
45
Underlying Interface
46
}
47
48
func (j *JSON[T]) Delete(ctx context.Context, key string) error {
49
return j.Underlying.Delete(ctx, key)
50
}
51
52
func (j *JSON[T]) Get(ctx context.Context, key string) (T, error) {
53
data, err := j.Underlying.Get(ctx, key)
54
if err != nil {
55
return z[T](), err
···
64
}
65
66
func (j *JSON[T]) Set(ctx context.Context, key string, value T, expiry time.Duration) error {
67
data, err := json.Marshal(value)
68
if err != nil {
69
return fmt.Errorf("%w: %w", ErrCantEncode, err)
···
43
44
type JSON[T any] struct {
45
Underlying Interface
46
+
Prefix string
47
}
48
49
func (j *JSON[T]) Delete(ctx context.Context, key string) error {
50
+
if j.Prefix != "" {
51
+
key = j.Prefix + key
52
+
}
53
+
54
return j.Underlying.Delete(ctx, key)
55
}
56
57
func (j *JSON[T]) Get(ctx context.Context, key string) (T, error) {
58
+
if j.Prefix != "" {
59
+
key = j.Prefix + key
60
+
}
61
+
62
data, err := j.Underlying.Get(ctx, key)
63
if err != nil {
64
return z[T](), err
···
73
}
74
75
func (j *JSON[T]) Set(ctx context.Context, key string, value T, expiry time.Duration) error {
76
+
if j.Prefix != "" {
77
+
key = j.Prefix + key
78
+
}
79
+
80
data, err := json.Marshal(value)
81
if err != nil {
82
return fmt.Errorf("%w: %w", ErrCantEncode, err)
+50
lib/store/json_test.go
+50
lib/store/json_test.go
···
···
1
+
package store_test
2
+
3
+
import (
4
+
"testing"
5
+
"time"
6
+
7
+
"github.com/TecharoHQ/anubis/lib/store"
8
+
"github.com/TecharoHQ/anubis/lib/store/memory"
9
+
)
10
+
11
+
func TestJSON(t *testing.T) {
12
+
type data struct {
13
+
ID string `json:"id"`
14
+
}
15
+
16
+
st := memory.New(t.Context())
17
+
db := store.JSON[data]{
18
+
Underlying: st,
19
+
Prefix: "foo:",
20
+
}
21
+
22
+
if err := db.Set(t.Context(), "test", data{ID: t.Name()}, time.Minute); err != nil {
23
+
t.Fatal(err)
24
+
}
25
+
26
+
got, err := db.Get(t.Context(), "test")
27
+
if err != nil {
28
+
t.Fatal(err)
29
+
}
30
+
31
+
if got.ID != t.Name() {
32
+
t.Fatalf("got wrong data for key \"test\", wanted %q but got: %q", t.Name(), got.ID)
33
+
}
34
+
35
+
if err := db.Delete(t.Context(), "test"); err != nil {
36
+
t.Fatal(err)
37
+
}
38
+
39
+
if _, err := db.Get(t.Context(), "test"); err == nil {
40
+
t.Fatal("wanted invalid get to fail, it did not")
41
+
}
42
+
43
+
if err := st.Set(t.Context(), "foo:test", []byte("}"), time.Minute); err != nil {
44
+
t.Fatal(err)
45
+
}
46
+
47
+
if _, err := db.Get(t.Context(), "test"); err == nil {
48
+
t.Fatal("wanted invalid get to fail, it did not")
49
+
}
50
+
}