techaro.lol
Weighs the soul of incoming HTTP requests to stop AI crawlers
fix: make ogtags and dnsbl use the Store instead of memory (#760)
Signed-off-by: Xe Iaso <me@xeiaso.net>
-15
cmd/anubis/main.go
-15
cmd/anubis/main.go
···
231
231
return rp, nil
232
232
}
233
233
234
-
func startDecayMapCleanup(ctx context.Context, s *libanubis.Server) {
235
-
ticker := time.NewTicker(1 * time.Hour)
236
-
defer ticker.Stop()
237
-
238
-
for {
239
-
select {
240
-
case <-ticker.C:
241
-
s.CleanupDecayMap()
242
-
case <-ctx.Done():
243
-
return
244
-
}
245
-
}
246
-
}
247
-
248
234
func main() {
249
235
flagenv.Parse()
250
236
flag.Parse()
···
421
407
wg.Add(1)
422
408
go metricsServer(ctx, wg.Done)
423
409
}
424
-
go startDecayMapCleanup(ctx, s)
425
410
426
411
var h http.Handler
427
412
h = s
+1
docs/docs/CHANGELOG.md
+1
docs/docs/CHANGELOG.md
···
24
24
- Remove the "Success" interstitial after a proof of work challenge is concluded.
25
25
- Anubis now has the concept of [storage backends](./admin/policies.mdx#storage-backends). These allow you to change how Anubis stores temporary data (in memory, on the disk, or in Valkey). If you run Anubis in an environment where you have a low amount of memory available for Anubis (eg: less than 64 megabytes), be sure to configure the [`bbolt`](./admin/policies.mdx#bbolt) storage backend.
26
26
- The challenge issuance and validation process has been rewritten from scratch. Instead of generating challenge strings from request metadata (under the assumption that the values being compared against are stable), Anubis now generates random data for each challenge. This data is stored in the active [storage backend](./admin/policies.mdx#storage-backends) for up to 30 minutes. Fixes [#564](https://github.com/TecharoHQ/anubis/issues/564), [#746](https://github.com/TecharoHQ/anubis/issues/746), and other similar instances of this issue.
27
+
- Make the [Open Graph](./admin/configuration/open-graph.mdx) subsystem and DNSBL subsystem use [storage backends](./admin/policies.mdx#storage-backends) instead of storing everything in memory by default.
27
28
- Add option for forcing a specific language ([#742](https://github.com/TecharoHQ/anubis/pull/742))
28
29
- Add translation for Turkish language ([#751](https://github.com/TecharoHQ/anubis/pull/751))
29
30
- Allow [Common Crawl](https://commoncrawl.org/) by default so scrapers have less incentive to scrape
+4
-9
lib/anubis.go
+4
-9
lib/anubis.go
···
70
70
next http.Handler
71
71
mux *http.ServeMux
72
72
policy *policy.ParsedConfig
73
-
DNSBLCache *decaymap.Impl[string, dnsbl.DroneBLResponse]
74
73
OGTags *ogtags.OGTagCache
75
74
ed25519Priv ed25519.PrivateKey
76
75
hs512Secret []byte
···
279
278
}
280
279
281
280
func (s *Server) handleDNSBL(w http.ResponseWriter, r *http.Request, ip string, lg *slog.Logger) bool {
281
+
db := &store.JSON[dnsbl.DroneBLResponse]{Underlying: s.store, Prefix: "dronebl:"}
282
282
if s.policy.DNSBL && ip != "" {
283
-
resp, ok := s.DNSBLCache.Get(ip)
284
-
if !ok {
283
+
resp, err := db.Get(r.Context(), ip)
284
+
if err != nil {
285
285
lg.Debug("looking up ip in dnsbl")
286
286
resp, err := dnsbl.Lookup(ip)
287
287
if err != nil {
288
288
lg.Error("can't look up ip in dnsbl", "err", err)
289
289
}
290
-
s.DNSBLCache.Set(ip, resp, 24*time.Hour)
290
+
db.Set(r.Context(), ip, resp, 24*time.Hour)
291
291
droneBLHits.WithLabelValues(resp.String()).Inc()
292
292
}
293
293
···
551
551
Rules: &checker.List{},
552
552
}, nil
553
553
}
554
-
555
-
func (s *Server) CleanupDecayMap() {
556
-
s.DNSBLCache.Cleanup()
557
-
s.OGTags.Cleanup()
558
-
}
+1
-4
lib/config.go
+1
-4
lib/config.go
···
15
15
16
16
"github.com/TecharoHQ/anubis"
17
17
"github.com/TecharoHQ/anubis/data"
18
-
"github.com/TecharoHQ/anubis/decaymap"
19
18
"github.com/TecharoHQ/anubis/internal"
20
-
"github.com/TecharoHQ/anubis/internal/dnsbl"
21
19
"github.com/TecharoHQ/anubis/internal/ogtags"
22
20
"github.com/TecharoHQ/anubis/lib/challenge"
23
21
"github.com/TecharoHQ/anubis/lib/localization"
···
108
106
hs512Secret: opts.HS512Secret,
109
107
policy: opts.Policy,
110
108
opts: opts,
111
-
DNSBLCache: decaymap.New[string, dnsbl.DroneBLResponse](),
112
-
OGTags: ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph),
109
+
OGTags: ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph, opts.Policy.Store),
113
110
store: opts.Policy.Store,
114
111
}
115
112
+1
-1
lib/http.go
+1
-1
lib/http.go
···
138
138
var ogTags map[string]string = nil
139
139
if s.opts.OpenGraph.Enabled {
140
140
var err error
141
-
ogTags, err = s.OGTags.GetOGTags(r.URL, r.Host)
141
+
ogTags, err = s.OGTags.GetOGTags(r.Context(), r.URL, r.Host)
142
142
if err != nil {
143
143
lg.Error("failed to get OG tags", "err", err)
144
144
}
+13
lib/store/interface.go
+13
lib/store/interface.go
···
43
43
44
44
type JSON[T any] struct {
45
45
Underlying Interface
46
+
Prefix string
46
47
}
47
48
48
49
func (j *JSON[T]) Delete(ctx context.Context, key string) error {
50
+
if j.Prefix != "" {
51
+
key = j.Prefix + key
52
+
}
53
+
49
54
return j.Underlying.Delete(ctx, key)
50
55
}
51
56
52
57
func (j *JSON[T]) Get(ctx context.Context, key string) (T, error) {
58
+
if j.Prefix != "" {
59
+
key = j.Prefix + key
60
+
}
61
+
53
62
data, err := j.Underlying.Get(ctx, key)
54
63
if err != nil {
55
64
return z[T](), err
···
64
73
}
65
74
66
75
func (j *JSON[T]) Set(ctx context.Context, key string, value T, expiry time.Duration) error {
76
+
if j.Prefix != "" {
77
+
key = j.Prefix + key
78
+
}
79
+
67
80
data, err := json.Marshal(value)
68
81
if err != nil {
69
82
return fmt.Errorf("%w: %w", ErrCantEncode, err)
+50
lib/store/json_test.go
+50
lib/store/json_test.go
···
1
+
package store_test
2
+
3
+
import (
4
+
"testing"
5
+
"time"
6
+
7
+
"github.com/TecharoHQ/anubis/lib/store"
8
+
"github.com/TecharoHQ/anubis/lib/store/memory"
9
+
)
10
+
11
+
func TestJSON(t *testing.T) {
12
+
type data struct {
13
+
ID string `json:"id"`
14
+
}
15
+
16
+
st := memory.New(t.Context())
17
+
db := store.JSON[data]{
18
+
Underlying: st,
19
+
Prefix: "foo:",
20
+
}
21
+
22
+
if err := db.Set(t.Context(), "test", data{ID: t.Name()}, time.Minute); err != nil {
23
+
t.Fatal(err)
24
+
}
25
+
26
+
got, err := db.Get(t.Context(), "test")
27
+
if err != nil {
28
+
t.Fatal(err)
29
+
}
30
+
31
+
if got.ID != t.Name() {
32
+
t.Fatalf("got wrong data for key \"test\", wanted %q but got: %q", t.Name(), got.ID)
33
+
}
34
+
35
+
if err := db.Delete(t.Context(), "test"); err != nil {
36
+
t.Fatal(err)
37
+
}
38
+
39
+
if _, err := db.Get(t.Context(), "test"); err == nil {
40
+
t.Fatal("wanted invalid get to fail, it did not")
41
+
}
42
+
43
+
if err := st.Set(t.Context(), "foo:test", []byte("}"), time.Minute); err != nil {
44
+
t.Fatal(err)
45
+
}
46
+
47
+
if _, err := db.Get(t.Context(), "test"); err == nil {
48
+
t.Fatal("wanted invalid get to fail, it did not")
49
+
}
50
+
}