feat: implement a client for Thoth, the IP reputation database for Anubis (#637)

Weighs the soul of incoming HTTP requests to stop AI crawlers

* feat(internal): add Thoth client and simple ASN checker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): cached ip to asn checker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: go mod tidy

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(thoth): minor testing fixups, ensure ASNChecker is Checker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): make ASNChecker instances

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): add GeoIP checker

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): store a thoth client in a context

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: refactor Checker type to its own package

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(thoth): add thoth mocking package, ignore context deadline exceeded errors

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thoth): pre-cache private ranges

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(lib/policy/config): enable thoth ASNs and GeoIP checker parsing

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(thoth): refactor to move checker creation to the checker files

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(policy): enable thoth checks

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat(thothmock): test helper function for loading a mock thoth instance

Signed-off-by: Xe Iaso <me@xeiaso.net>

* feat: wire up Thoth, make thoth checks part of the default config

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(thoth): mend staticcheck errors

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(admin): add Thoth docs

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(policy): update Thoth links in error messages

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs: update CHANGELOG

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore(docs/manifest): enable Thoth

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: add THOTH_INSECURE for contacting Thoth over plain TCP in extreme circumstances

Signed-off-by: Xe Iaso <me@xeiaso.net>

* test(thoth): use mock thoth when credentials aren't detected in the environment

Signed-off-by: Xe Iaso <me@xeiaso.net>

* chore: spelling

Signed-off-by: Xe Iaso <me@xeiaso.net>

* fix(cmd/anubis): better warnings for half-configured Thoth setups

Signed-off-by: Xe Iaso <me@xeiaso.net>

* docs(botpolicies): link to Thoth geoip docs

Signed-off-by: Xe Iaso <me@xeiaso.net>

---------

Signed-off-by: Xe Iaso <me@xeiaso.net>

authored by

Xe Iaso and committed by

GitHub 9 months ago e3826df3 823d1be5

+1101 -82

39 changed files

expand all

unified split

.github

actions

spelling

excludes.txt

expect.txt

.vscode

settings.json

cmd

anubis

main.go

data

botPolicies.yaml

docs

CHANGELOG.md

admin

thoth.mdx

manifest

1password.yaml

deployment.yaml

kustomization.yaml

poddisruptionbudget.yaml

go.mod

go.sum

internal

test

playwright_test.go

thoth

asnchecker.go

asnchecker_test.go

auth.go

cachediptoasn.go

context.go

geoipchecker.go

geoipchecker_test.go

thoth.go

thoth_test.go

thothmock

iptoasn.go

withthothmock.go

lib

anubis.go

anubis_test.go

config.go

config_test.go

policy

bot.go

checker

checker.go

config

asn.go

config.go

geoip.go

testdata

good

challenge_cloudflare.yaml

geoip_us.yaml

policy.go

policy_test.go

+2 -1

.github/actions/spelling/excludes.txt

··· 83 83 ^\Q.github/FUNDING.yml\E$ 84 84 ^\Q.github/workflows/spelling.yml\E$ 85 85 ^data/crawlers/ 86 + ^docs/manifest/.*$ 86 87 ^docs/static/\.nojekyll$ 87 88 ignore$ 88 - robots.txt 89 + robots.txt

+19 -2

.github/actions/spelling/expect.txt

··· 9 9 apk 10 10 Applebot 11 11 archlinux 12 + asnc 13 + asnchecker 14 + asns 15 + aspirational 12 16 badregexes 13 17 bdba 14 18 berr 15 - betteralign 16 19 bingbot 17 20 bitcoin 18 21 blogging ··· 25 28 broked 26 29 Bytespider 27 30 cachebuster 31 + cachediptoasn 28 32 Caddyfile 29 33 caninetools 30 34 Cardyb ··· 89 93 forgejo 90 94 fsys 91 95 fullchain 96 + gaissmai 92 97 Galvus 98 + geoip 99 + geoipchecker 93 100 gha 101 + gipc 94 102 gitea 103 + godotenv 95 104 goland 96 105 gomod 97 106 goodbot ··· 101 110 GPG 102 111 GPT 103 112 gptbot 113 + grpcprom 104 114 grw 105 115 Hashcash 106 116 hashrate ··· 113 123 htmlc 114 124 htmx 115 125 httpdebug 126 + Huawei 116 127 hypertext 117 128 iaskspider 118 129 iat ··· 120 131 Imagesift 121 132 imgproxy 122 133 inp 134 + IPTo 135 + iptoasn 123 136 iss 124 137 isset 125 138 ivh 126 139 Jenomis 127 140 JGit 141 + joho 128 142 journalctl 129 143 jshelter 130 144 JWTs ··· 164 178 mozilla 165 179 nbf 166 180 netsurf 167 - NFlag 168 181 nginx 169 182 nobots 170 183 NONINFRINGEMENT ··· 241 254 SVCNAME 242 255 tagline 243 256 tarballs 257 + tarrif 244 258 techaro 245 259 techarohq 246 260 templ 247 261 templruntime 248 262 testarea 263 + thoth 264 + thothmock 249 265 Tik 250 266 Timpibot 251 267 torproject ··· 270 286 websites 271 287 Webzio 272 288 wildbase 289 + withthothmock 273 290 wordpress 274 291 Workaround 275 292 workdir

+19

.vscode/settings.json

··· 11 11 "zig": false, 12 12 "javascript": false, 13 13 "properties": false 14 + }, 15 + "[markdown]": { 16 + "editor.wordWrap": "wordWrapColumn", 17 + "editor.wordWrapColumn": 80, 18 + "editor.wordBasedSuggestions": "off" 19 + }, 20 + "[mdx]": { 21 + "editor.wordWrap": "wordWrapColumn", 22 + "editor.wordWrapColumn": 80, 23 + "editor.wordBasedSuggestions": "off" 24 + }, 25 + "[nunjucks]": { 26 + "editor.wordWrap": "wordWrapColumn", 27 + "editor.wordWrapColumn": 80, 28 + "editor.wordBasedSuggestions": "off" 29 + }, 30 + "cSpell.enabledFileTypes": { 31 + "mdx": true, 32 + "md": true 14 33 } 15 34 }

+25 -1

cmd/anubis/main.go

··· 30 30 "github.com/TecharoHQ/anubis" 31 31 "github.com/TecharoHQ/anubis/data" 32 32 "github.com/TecharoHQ/anubis/internal" 33 + "github.com/TecharoHQ/anubis/internal/thoth" 33 34 libanubis "github.com/TecharoHQ/anubis/lib" 34 35 botPolicy "github.com/TecharoHQ/anubis/lib/policy" 35 36 "github.com/TecharoHQ/anubis/lib/policy/config" 36 37 "github.com/TecharoHQ/anubis/web" 37 38 "github.com/facebookgo/flagenv" 39 + _ "github.com/joho/godotenv/autoload" 38 40 "github.com/prometheus/client_golang/prometheus/promhttp" 39 41 ) 40 42 ··· 70 72 webmasterEmail = flag.String("webmaster-email", "", "if set, displays webmaster's email on the reject page for appeals") 71 73 versionFlag = flag.Bool("version", false, "print Anubis version") 72 74 xffStripPrivate = flag.Bool("xff-strip-private", true, "if set, strip private addresses from X-Forwarded-For") 75 + 76 + thothInsecure = flag.Bool("thoth-insecure", false, "if set, connect to Thoth over plain HTTP/2, don't enable this unless support told you to") 77 + thothURL = flag.String("thoth-url", "", "if set, URL for Thoth, the IP reputation database for Anubis") 78 + thothToken = flag.String("thoth-token", "", "if set, API token for Thoth, the IP reputation database for Anubis") 73 79 ) 74 80 75 81 func keyFromHex(value string) (ed25519.PrivateKey, error) { ··· 233 239 } 234 240 } 235 241 236 - policy, err := libanubis.LoadPoliciesOrDefault(*policyFname, *challengeDifficulty) 242 + ctx := context.Background() 243 + 244 + // Thoth configuration 245 + switch { 246 + case *thothURL != "" && *thothToken == "": 247 + slog.Warn("THOTH_URL is set but no THOTH_TOKEN is set") 248 + case *thothURL == "" && *thothToken != "": 249 + slog.Warn("THOTH_TOKEN is set but no THOTH_URL is set") 250 + case *thothURL != "" && *thothToken != "": 251 + slog.Debug("connecting to Thoth") 252 + thothClient, err := thoth.New(ctx, *thothURL, *thothToken, *thothInsecure) 253 + if err != nil { 254 + log.Fatalf("can't dial thoth at %s: %v", *thothURL, err) 255 + } 256 + 257 + ctx = thoth.With(ctx, thothClient) 258 + } 259 + 260 + policy, err := libanubis.LoadPoliciesOrDefault(ctx, *policyFname, *challengeDifficulty) 237 261 if err != nil { 238 262 log.Fatalf("can't parse policy file: %v", err) 239 263 }

+23

data/botPolicies.yaml

··· 51 51 # report_as: 4 # lie to the operator 52 52 # algorithm: slow # intentionally waste CPU cycles and time 53 53 54 + # Requires a subscription to Thoth to use, see 55 + # https://anubis.techaro.lol/docs/admin/thoth#geoip-based-filtering 56 + - name: countries-with-aggressive-scrapers 57 + action: WEIGH 58 + geoip: 59 + counties: 60 + - BR 61 + - CN 62 + weight: 63 + adjust: 10 64 + 65 + # Requires a subscription to Thoth to use, see 66 + # https://anubis.techaro.lol/docs/admin/thoth#asn-based-filtering 67 + - name: aggressive-asns-without-functional-abuse-contact 68 + action: WEIGH 69 + asns: 70 + match: 71 + - 13335 # Cloudflare 72 + - 136907 # Huawei Cloud 73 + - 45102 # Alibaba Cloud 74 + weight: 75 + adjust: 10 76 + 54 77 # Generic catchall rule 55 78 - name: generic-browser 56 79 user_agent_regex: >-

docs/docs/CHANGELOG.md

··· 23 23 - Optimized the OGTags subsystem with reduced allocations and runtime per request by up to 66% 24 24 - Add `--strip-base-prefix` flag/envvar to strip the base prefix from request paths when forwarding to target servers 25 25 - Add `robots2policy` CLI utility to convert robots.txt files to Anubis challenge policies using CEL expressions ([#409](https://github.com/TecharoHQ/anubis/issues/409)) 26 + - Implement GeoIP and ASN based checks via [Thoth](https://anubis.techaro.lol/docs/admin/thoth) ([#206](https://github.com/TecharoHQ/anubis/issues/206)) 26 27 27 28 ## v1.19.1: Jenomis cen Lexentale - Echo 1 28 29

+81

docs/docs/admin/thoth.mdx

··· 1 + # Thoth-based advanced checks 2 + 3 + Status: Beta 4 + 5 + Anubis instances are normally isolated. Each Anubis instance has its own configuration and exists in roughly its own world without any long term memory between requests. As threats, workarounds, and AI scraper toolchains evolve, administrators will need a way to get more up to date information faster than Anubis' release cycle. 6 + 7 + Thus, Thoth is being created. Thoth is the reputation database for Anubis. Thoth feeds information to Anubis so that it can make better decisions about which traffic is innocuous and which traffic is suspicious. 8 + 9 + :::note 10 + 11 + Thoth is hosted by [Techaro](https://techaro.lol). Thoth is a paid service. Thoth is opt-in and requires manual intervention (including payment) to use. The code that powers Thoth is currently closed source. 12 + 13 + To get access to Thoth, please subscribe [on GitHub Sponsors](https://github.com/sponsors/Xe) and [email Xe](mailto:xe@techaro.lol). This will be self-service soon. 14 + 15 + ::: 16 + 17 + ## Implementation 18 + 19 + Thoth is a web service that listens over [gRPC](https://grpc.io/). Thoth's API is documented in protocol buffer definitions in the GitHub repo [TecharoHQ/thoth-proto](https://github.com/TecharoHQ/thoth-proto). 20 + 21 + Thoth is designed to be _informative_, not _authoritative_. Thoth cannot and will not arbitrarily block requests, origins, or other traffic. Thoth is there to inform Anubis and influence the weight of requests so that upstream resources can be protected. Additionally, Anubis aggressively caches data from Thoth such that over time Anubis will not need to request data very often. This makes the fast path for repeat visitors even faster and reduces the amount of data that Thoth is exposed to. 22 + 23 + ## Thoth features 24 + 25 + Thoth is currently in active development. Currently, Thoth provides the following features to Anubis: 26 + 27 + - BGP Autonomous System (ASN) based filtering 28 + - GeoIP location based filtering 29 + 30 + ### ASN-based filtering 31 + 32 + When companies link their backbone infrastructure to the Internet, they do so via a [BGP Autonomous System](<https://en.wikipedia.org/wiki/Autonomous_system_(Internet)>), denoted by a number (the Autonomous System Number or ASN). Every IP address on the Internet is owned by an ASN with a 1:1 lookup that does not change very frequently. 33 + 34 + Anubis uses Thoth to match IP addresses to BGP Autonomous Systems so that you can either issue arbitrary challenges to individual internet service providers (such as Cloudflare or Huawei Cloud) or, at the administrator's explicit instruction, block them altogether. For example, here's how you add 10 weight points to requests from Cloudflare, Huawei Cloud, and Alibaba Cloud: 35 + 36 + ```yaml 37 + - name: aggressive-asns-without-functional-abuse-contact 38 + action: WEIGH 39 + asns: 40 + match: 41 + - 13335 # Cloudflare 42 + - 136907 # Huawei Cloud 43 + - 45102 # Alibaba Cloud 44 + weight: 45 + adjust: 10 46 + ``` 47 + 48 + You can look up details for [AS13335](https://bgp.tools/as/13335) or any of these other top offenders on [bgp.tools](https://bgp.tools). 49 + 50 + ### GeoIP-based filtering 51 + 52 + In extreme cases, an administrator may have to take action against an entire country. This is not an ideal circumstance, but sometimes reality forces their hands and the administrators just want to sleep at night. 53 + 54 + Anubis uses Thoth to look up the geographic location registered to an IP address. This lookup is not the best and will get better with time, but you ship what you can so you can make it better for next time. 55 + 56 + For example, to add 10 weight points to requests from Brazil and China: 57 + 58 + ```yaml 59 + - name: countries-with-aggressive-scrapers 60 + action: WEIGH 61 + geoip: 62 + counties: 63 + - BR 64 + - CN 65 + weight: 66 + adjust: 10 67 + ``` 68 + 69 + Use this with care. 70 + 71 + ## Work-in-progress features 72 + 73 + This section is a bit aspirational and is where Thoth will end up rather than things you can use today. 74 + 75 + In general, a lot of Thoth features are focused on taking the same Anubis you know and love and making it better, smarter, and less paranoid. These include: 76 + 77 + - Private rulesets for advanced patterns, current known exploits, and other recognition tactics that need to be kept cloak and dagger for operational security reasons 78 + - Private challenge implementations via WebAssembly, including advanced browser detection logic 79 + - Reputation querying so that Thoth can arbitrarily influence the weight of requests based on the net aggregate pass rate so that the most common browsers can get through with no challenge issued at all 80 + - APIs for trusted administrators to report abusive request fingerprints so that Anubis can react to threats as they evolve 81 + - A way for Anubis to periodically report the pass rate per ASN and other fingerprints so that methodology can be improved

docs/manifest/1password.yaml

··· 1 + apiVersion: onepassword.com/v1 2 + kind: OnePasswordItem 3 + metadata: 4 + name: anubis-docs-thoth 5 + spec: 6 + itemPath: "vaults/lc5zo4zjz3if3mkeuhufjmgmui/items/pwguumqcmtxvqbeb7y4gj7l36i"

docs/manifest/deployment.yaml

··· 68 68 - ALL 69 69 seccompProfile: 70 70 type: RuntimeDefault 71 + envFrom: 72 + - secretRef: 73 + name: anubis-docs-thoth

docs/manifest/kustomization.yaml

··· 1 1 resources: 2 + - 1password.yaml 2 3 - deployment.yaml 3 4 - ingress.yaml 4 5 - onionservice.yaml 6 + - poddisruptionbudget.yaml 5 7 - service.yaml 6 8 7 9 configMapGenerator:

docs/manifest/poddisruptionbudget.yaml

··· 1 + apiVersion: policy/v1 2 + kind: PodDisruptionBudget 3 + metadata: 4 + name: anubis-docs 5 + spec: 6 + minAvailable: 1 7 + selector: 8 + matchLabels: 9 + app: anubis-docs

+11 -4

go.mod

··· 3 3 go 1.24.2 4 4 5 5 require ( 6 + github.com/TecharoHQ/thoth-proto v0.4.0 6 7 github.com/a-h/templ v0.3.898 7 8 github.com/facebookgo/flagenv v0.0.0-20160425205200-fcd59fca7456 9 + github.com/gaissmai/bart v0.20.4 8 10 github.com/golang-jwt/jwt/v5 v5.2.2 9 11 github.com/google/cel-go v0.25.0 12 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 13 + github.com/joho/godotenv v1.5.1 10 14 github.com/playwright-community/playwright-go v0.5200.0 11 15 github.com/prometheus/client_golang v1.22.0 12 16 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a 13 17 github.com/yl2chen/cidranger v1.0.2 14 18 golang.org/x/net v0.41.0 15 19 gopkg.in/yaml.v3 v3.0.1 20 + google.golang.org/grpc v1.72.2 16 21 k8s.io/apimachinery v0.33.1 17 22 sigs.k8s.io/yaml v1.4.0 18 23 ) 19 24 20 25 require ( 21 26 al.essio.dev/pkg/shellescape v1.6.0 // indirect 27 + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 // indirect 22 28 cel.dev/expr v0.23.1 // indirect 23 29 dario.cat/mergo v1.0.2 // indirect 24 30 github.com/AlekSi/pointer v1.2.0 // indirect ··· 66 72 github.com/goreleaser/chglog v0.7.0 // indirect 67 73 github.com/goreleaser/fileglob v1.3.0 // indirect 68 74 github.com/goreleaser/nfpm/v2 v2.42.1 // indirect 75 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect 69 76 github.com/huandu/xstrings v1.5.0 // indirect 70 77 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 71 78 github.com/kevinburke/ssh_config v1.2.0 // indirect ··· 86 93 github.com/shopspring/decimal v1.4.0 // indirect 87 94 github.com/skeema/knownhosts v1.3.1 // indirect 88 95 github.com/spf13/cast v1.7.1 // indirect 89 - github.com/stoewer/go-strcase v1.2.0 // indirect 96 + github.com/stoewer/go-strcase v1.3.0 // indirect 90 97 github.com/ulikunitz/xz v0.5.12 // indirect 91 98 github.com/xanzy/ssh-agent v0.3.3 // indirect 92 99 gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect ··· 102 109 golang.org/x/tools v0.33.0 // indirect 103 110 golang.org/x/vuln v1.1.4 // indirect 104 111 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect 105 - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect 106 - google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect 107 - google.golang.org/protobuf v1.36.5 // indirect 112 + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect 113 + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect 114 + google.golang.org/protobuf v1.36.6 // indirect 108 115 gopkg.in/warnings.v0 v0.1.2 // indirect 109 116 honnef.co/go/tools v0.6.1 // indirect 110 117 mvdan.cc/sh/v3 v3.11.0 // indirect

+45 -9

go.sum

··· 1 1 al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA= 2 2 al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= 3 + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1 h1:YhMSc48s25kr7kv31Z8vf7sPUIq5YJva9z1mn/hAt0M= 4 + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.6-20250425153114-8976f5be98c1.1/go.mod h1:avRlCjnFzl98VPaeCtJ24RrV/wwHFzB8sWXhj26+n/U= 3 5 cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg= 4 6 cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= 5 7 dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= ··· 28 30 github.com/ProtonMail/gopenpgp/v2 v2.7.1/go.mod h1:/BU5gfAVwqyd8EfC3Eu7zmuhwYQpKs+cGD8M//iiaxs= 29 31 github.com/Songmu/gitconfig v0.2.0 h1:pX2++u4KUq+K2k/ZCzGXLtkD3ceCqIdi0tDyb+IbSyo= 30 32 github.com/Songmu/gitconfig v0.2.0/go.mod h1:cB5bYJer+pl7W8g6RHFwL/0X6aJROVrYuHlvc7PT+hE= 33 + github.com/TecharoHQ/thoth-proto v0.4.0 h1:UbkvfgCku0Dm1R6O4ug3HOsJNnE6F3wB8x+Dpw2lzFI= 34 + github.com/TecharoHQ/thoth-proto v0.4.0/go.mod h1:IcGnZt3iYUZQVEa0Lwk5l4ix0hCeXlWUV1TJMZvbWx0= 31 35 github.com/TecharoHQ/yeet v0.6.0 h1:RCBAjr7wIlllsgy0tpvWpLX7jsZgu2tiuBY3RrprcR0= 32 36 github.com/TecharoHQ/yeet v0.6.0/go.mod h1:bj2V4Fg8qKQXoiuPZa3HuawrE8g+LsOQv/9q2WyGSsA= 33 37 github.com/a-h/parse v0.0.0-20250122154542-74294addb73e h1:HjVbSQHy+dnlS6C3XajZ69NYAb5jbGNfHanvm1+iYlo= ··· 99 103 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 100 104 github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 101 105 github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 106 + github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U= 107 + github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk= 102 108 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= 103 109 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= 104 110 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= ··· 111 117 github.com/go-git/go-git/v5 v5.14.0/go.mod h1:Z5Xhoia5PcWA3NF8vRLURn9E5FRhSl7dGj9ItW3Wk5k= 112 118 github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= 113 119 github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= 120 + github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 121 + github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 122 + github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 123 + github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 114 124 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 115 125 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= 116 126 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= ··· 134 144 github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= 135 145 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= 136 146 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= 147 + github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 148 + github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 137 149 github.com/google/cel-go v0.25.0 h1:jsFw9Fhn+3y2kBbltZR4VEz5xKkcIFRPDnuEzAGv5GY= 138 150 github.com/google/cel-go v0.25.0/go.mod h1:hjEb6r5SuOSlhCHmFoLzu8HGCERvIsDAbxDAyNU/MmI= 139 151 github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU= ··· 159 171 github.com/goreleaser/fileglob v1.3.0/go.mod h1:Jx6BoXv3mbYkEzwm9THo7xbr5egkAraxkGorbJb4RxU= 160 172 github.com/goreleaser/nfpm/v2 v2.42.1 h1:xu2pLRgQuz2ab+YZFoeIzwU/M5jjjCKDGwv1lRbVGvk= 161 173 github.com/goreleaser/nfpm/v2 v2.42.1/go.mod h1:dY53KWYKebkOocxgkmpM7SRX0Nv5hU+jEu2kIaM4/LI= 174 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= 175 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= 176 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= 177 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0/go.mod h1:XKMd7iuf/RGPSMJ/U4HP0zS2Z9Fh8Ps9a+6X26m/tmI= 162 178 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= 163 179 github.com/henvic/httpretty v0.0.6/go.mod h1:X38wLjWXHkXT7r2+uK8LjCMne9rsuNaBLJ+5cU2/Pmo= 164 180 github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= 165 181 github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= 166 182 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= 167 183 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= 184 + github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 185 + github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 168 186 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 169 187 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 170 188 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= ··· 248 266 github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= 249 267 github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 250 268 github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 251 - github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= 252 - github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= 269 + github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= 270 + github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= 253 271 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 272 + github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 273 + github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 254 274 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 255 275 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 256 - github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 257 276 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 277 + github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 278 + github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 279 + github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 258 280 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 259 281 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 260 282 github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI= ··· 269 291 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 270 292 gitlab.com/digitalxero/go-conventional-commit v1.0.7 h1:8/dO6WWG+98PMhlZowt/YjuiKhqhGlOCwlIV8SqqGh8= 271 293 gitlab.com/digitalxero/go-conventional-commit v1.0.7/go.mod h1:05Xc2BFsSyC5tKhK0y+P3bs0AwUtNuTp+mTpbCU/DZ0= 294 + go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 295 + go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 296 + go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= 297 + go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= 298 + go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= 299 + go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= 300 + go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= 301 + go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= 302 + go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= 303 + go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= 304 + go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= 305 + go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= 272 306 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 273 307 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 274 308 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= ··· 353 387 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= 354 388 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= 355 389 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= 356 - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 h1:YcyjlL1PRr2Q17/I0dPk2JmYS5CDXfcdb2Z3YRioEbw= 357 - google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= 358 - google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 h1:2035KHhUv+EpyB+hWgJnaWKJOdX1E95w2S8Rr4uWKTs= 359 - google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= 360 - google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 361 - google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 390 + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= 391 + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= 392 + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= 393 + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= 394 + google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= 395 + google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= 396 + google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 397 + google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 362 398 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 363 399 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 364 400 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

+1 -1

internal/test/playwright_test.go

··· 595 595 fmt.Fprintf(w, "<html><body><span id=anubis-test>%d</span></body></html>", time.Now().Unix()) 596 596 }) 597 597 598 - policy, err := libanubis.LoadPoliciesOrDefault("", anubis.DefaultDifficulty) 598 + policy, err := libanubis.LoadPoliciesOrDefault(t.Context(), "", anubis.DefaultDifficulty) 599 599 if err != nil { 600 600 t.Fatal(err) 601 601 }

+69

internal/thoth/asnchecker.go

··· 1 + package thoth 2 + 3 + import ( 4 + "context" 5 + "errors" 6 + "fmt" 7 + "log/slog" 8 + "net/http" 9 + "strings" 10 + "time" 11 + 12 + "github.com/TecharoHQ/anubis/internal" 13 + "github.com/TecharoHQ/anubis/lib/policy/checker" 14 + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" 15 + ) 16 + 17 + func (c *Client) ASNCheckerFor(asns []uint32) checker.Impl { 18 + asnMap := map[uint32]struct{}{} 19 + var sb strings.Builder 20 + fmt.Fprintln(&sb, "ASNChecker") 21 + for _, asn := range asns { 22 + asnMap[asn] = struct{}{} 23 + fmt.Fprintln(&sb, "AS", asn) 24 + } 25 + 26 + return &ASNChecker{ 27 + iptoasn: c.IPToASN, 28 + asns: asnMap, 29 + hash: internal.SHA256sum(sb.String()), 30 + } 31 + } 32 + 33 + type ASNChecker struct { 34 + iptoasn iptoasnv1.IpToASNServiceClient 35 + asns map[uint32]struct{} 36 + hash string 37 + } 38 + 39 + func (asnc *ASNChecker) Check(r *http.Request) (bool, error) { 40 + ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond) 41 + defer cancel() 42 + 43 + ipInfo, err := asnc.iptoasn.Lookup(ctx, &iptoasnv1.LookupRequest{ 44 + IpAddress: r.Header.Get("X-Real-Ip"), 45 + }) 46 + if err != nil { 47 + switch { 48 + case errors.Is(err, context.DeadlineExceeded): 49 + slog.Debug("error contacting thoth", "err", err, "actionable", false) 50 + return false, nil 51 + default: 52 + slog.Error("error contacting thoth, please contact support", "err", err, "actionable", true) 53 + return false, nil 54 + } 55 + } 56 + 57 + // If IP is not publicly announced, return false 58 + if !ipInfo.GetAnnounced() { 59 + return false, nil 60 + } 61 + 62 + _, ok := asnc.asns[uint32(ipInfo.GetAsNumber())] 63 + 64 + return ok, nil 65 + } 66 + 67 + func (asnc *ASNChecker) Hash() string { 68 + return asnc.hash 69 + }

+81

internal/thoth/asnchecker_test.go

··· 1 + package thoth_test 2 + 3 + import ( 4 + "fmt" 5 + "net/http/httptest" 6 + "testing" 7 + 8 + "github.com/TecharoHQ/anubis/internal/thoth" 9 + "github.com/TecharoHQ/anubis/lib/policy/checker" 10 + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" 11 + ) 12 + 13 + var _ checker.Impl = &thoth.ASNChecker{} 14 + 15 + func TestASNChecker(t *testing.T) { 16 + cli := loadSecrets(t) 17 + 18 + asnc := cli.ASNCheckerFor([]uint32{13335}) 19 + 20 + for _, cs := range []struct { 21 + ipAddress string 22 + wantMatch bool 23 + wantError bool 24 + }{ 25 + { 26 + ipAddress: "1.1.1.1", 27 + wantMatch: true, 28 + wantError: false, 29 + }, 30 + { 31 + ipAddress: "2.2.2.2", 32 + wantMatch: false, 33 + wantError: false, 34 + }, 35 + { 36 + ipAddress: "taco", 37 + wantMatch: false, 38 + wantError: false, 39 + }, 40 + { 41 + ipAddress: "127.0.0.1", 42 + wantMatch: false, 43 + wantError: false, 44 + }, 45 + } { 46 + t.Run(fmt.Sprintf("%v", cs), func(t *testing.T) { 47 + req := httptest.NewRequest("GET", "/", nil) 48 + req.Header.Set("X-Real-Ip", cs.ipAddress) 49 + 50 + match, err := asnc.Check(req) 51 + 52 + if match != cs.wantMatch { 53 + t.Errorf("Wanted match: %v, got: %v", cs.wantMatch, match) 54 + } 55 + 56 + switch { 57 + case err != nil && !cs.wantError: 58 + t.Errorf("Did not want error but got: %v", err) 59 + case err == nil && cs.wantError: 60 + t.Error("Wanted error but got none") 61 + } 62 + }) 63 + } 64 + } 65 + 66 + func BenchmarkWithCache(b *testing.B) { 67 + cli := loadSecrets(b) 68 + req := &iptoasnv1.LookupRequest{IpAddress: "1.1.1.1"} 69 + 70 + _, err := cli.IPToASN.Lookup(b.Context(), req) 71 + if err != nil { 72 + b.Error(err) 73 + } 74 + 75 + for b.Loop() { 76 + _, err := cli.IPToASN.Lookup(b.Context(), req) 77 + if err != nil { 78 + b.Error(err) 79 + } 80 + } 81 + }

+39

internal/thoth/auth.go

··· 1 + package thoth 2 + 3 + import ( 4 + "context" 5 + 6 + "google.golang.org/grpc" 7 + "google.golang.org/grpc/metadata" 8 + ) 9 + 10 + func authUnaryClientInterceptor(token string) grpc.UnaryClientInterceptor { 11 + return func( 12 + ctx context.Context, 13 + method string, 14 + req interface{}, 15 + reply interface{}, 16 + cc *grpc.ClientConn, 17 + invoker grpc.UnaryInvoker, 18 + opts ...grpc.CallOption, 19 + ) error { 20 + md := metadata.Pairs("authorization", "Bearer "+token) 21 + ctx = metadata.NewOutgoingContext(ctx, md) 22 + return invoker(ctx, method, req, reply, cc, opts...) 23 + } 24 + } 25 + 26 + func authStreamClientInterceptor(token string) grpc.StreamClientInterceptor { 27 + return func( 28 + ctx context.Context, 29 + desc *grpc.StreamDesc, 30 + cc *grpc.ClientConn, 31 + method string, 32 + streamer grpc.Streamer, 33 + opts ...grpc.CallOption, 34 + ) (grpc.ClientStream, error) { 35 + md := metadata.Pairs("authorization", "Bearer "+token) 36 + ctx = metadata.NewOutgoingContext(ctx, md) 37 + return streamer(ctx, desc, cc, method, opts...) 38 + } 39 + }

+84

internal/thoth/cachediptoasn.go

··· 1 + package thoth 2 + 3 + import ( 4 + "context" 5 + "errors" 6 + "fmt" 7 + "log/slog" 8 + "net/netip" 9 + 10 + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" 11 + "github.com/gaissmai/bart" 12 + "google.golang.org/grpc" 13 + ) 14 + 15 + type IPToASNWithCache struct { 16 + next iptoasnv1.IpToASNServiceClient 17 + table *bart.Table[*iptoasnv1.LookupResponse] 18 + } 19 + 20 + func NewIpToASNWithCache(next iptoasnv1.IpToASNServiceClient) *IPToASNWithCache { 21 + result := &IPToASNWithCache{ 22 + next: next, 23 + table: &bart.Table[*iptoasnv1.LookupResponse]{}, 24 + } 25 + 26 + for _, pfx := range []netip.Prefix{ 27 + netip.MustParsePrefix("10.0.0.0/8"), // RFC 1918 28 + netip.MustParsePrefix("172.16.0.0/12"), // RFC 1918 29 + netip.MustParsePrefix("192.168.0.0/16"), // RFC 1918 30 + netip.MustParsePrefix("127.0.0.0/8"), // Loopback 31 + netip.MustParsePrefix("169.254.0.0/16"), // Link-local 32 + netip.MustParsePrefix("100.64.0.0/10"), // CGNAT 33 + netip.MustParsePrefix("192.0.0.0/24"), // Protocol assignments 34 + netip.MustParsePrefix("192.0.2.0/24"), // TEST-NET-1 35 + netip.MustParsePrefix("198.18.0.0/15"), // Benchmarking 36 + netip.MustParsePrefix("198.51.100.0/24"), // TEST-NET-2 37 + netip.MustParsePrefix("203.0.113.0/24"), // TEST-NET-3 38 + netip.MustParsePrefix("240.0.0.0/4"), // Reserved 39 + netip.MustParsePrefix("255.255.255.255/32"), // Broadcast 40 + netip.MustParsePrefix("fc00::/7"), // Unique local address 41 + netip.MustParsePrefix("fe80::/10"), // Link-local 42 + netip.MustParsePrefix("::1/128"), // Loopback 43 + netip.MustParsePrefix("::/128"), // Unspecified 44 + netip.MustParsePrefix("100::/64"), // Discard-only 45 + netip.MustParsePrefix("2001:db8::/32"), // Documentation 46 + } { 47 + result.table.Insert(pfx, &iptoasnv1.LookupResponse{Announced: false}) 48 + } 49 + 50 + return result 51 + } 52 + 53 + func (ip2asn *IPToASNWithCache) Lookup(ctx context.Context, lr *iptoasnv1.LookupRequest, opts ...grpc.CallOption) (*iptoasnv1.LookupResponse, error) { 54 + addr, err := netip.ParseAddr(lr.GetIpAddress()) 55 + if err != nil { 56 + return nil, fmt.Errorf("input is not an IP address: %w", err) 57 + } 58 + 59 + cachedResponse, ok := ip2asn.table.Lookup(addr) 60 + if ok { 61 + return cachedResponse, nil 62 + } 63 + 64 + resp, err := ip2asn.next.Lookup(ctx, lr, opts...) 65 + if err != nil { 66 + return nil, err 67 + } 68 + 69 + var errs []error 70 + for _, cidr := range resp.GetCidr() { 71 + pfx, err := netip.ParsePrefix(cidr) 72 + if err != nil { 73 + errs = append(errs, err) 74 + continue 75 + } 76 + ip2asn.table.Insert(pfx, resp) 77 + } 78 + 79 + if len(errs) != 0 { 80 + slog.Error("errors parsing IP prefixes", "err", errors.Join(errs...)) 81 + } 82 + 83 + return resp, nil 84 + }

+14

internal/thoth/context.go

··· 1 + package thoth 2 + 3 + import "context" 4 + 5 + type ctxKey struct{} 6 + 7 + func With(ctx context.Context, cli *Client) context.Context { 8 + return context.WithValue(ctx, ctxKey{}, cli) 9 + } 10 + 11 + func FromContext(ctx context.Context) (*Client, bool) { 12 + cli, ok := ctx.Value(ctxKey{}).(*Client) 13 + return cli, ok 14 + }

+68

internal/thoth/geoipchecker.go

··· 1 + package thoth 2 + 3 + import ( 4 + "context" 5 + "errors" 6 + "fmt" 7 + "log/slog" 8 + "net/http" 9 + "strings" 10 + "time" 11 + 12 + "github.com/TecharoHQ/anubis/lib/policy/checker" 13 + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" 14 + ) 15 + 16 + func (c *Client) GeoIPCheckerFor(countries []string) checker.Impl { 17 + countryMap := map[string]struct{}{} 18 + var sb strings.Builder 19 + fmt.Fprintln(&sb, "GeoIPChecker") 20 + for _, cc := range countries { 21 + countryMap[cc] = struct{}{} 22 + fmt.Fprintln(&sb, cc) 23 + } 24 + 25 + return &GeoIPChecker{ 26 + IPToASN: c.IPToASN, 27 + Countries: countryMap, 28 + hash: sb.String(), 29 + } 30 + } 31 + 32 + type GeoIPChecker struct { 33 + IPToASN iptoasnv1.IpToASNServiceClient 34 + Countries map[string]struct{} 35 + hash string 36 + } 37 + 38 + func (gipc *GeoIPChecker) Check(r *http.Request) (bool, error) { 39 + ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond) 40 + defer cancel() 41 + 42 + ipInfo, err := gipc.IPToASN.Lookup(ctx, &iptoasnv1.LookupRequest{ 43 + IpAddress: r.Header.Get("X-Real-Ip"), 44 + }) 45 + if err != nil { 46 + switch { 47 + case errors.Is(err, context.DeadlineExceeded): 48 + slog.Debug("error contacting thoth", "err", err, "actionable", false) 49 + return false, nil 50 + default: 51 + slog.Error("error contacting thoth, please contact support", "err", err, "actionable", true) 52 + return false, nil 53 + } 54 + } 55 + 56 + // If IP is not publicly announced, return false 57 + if !ipInfo.GetAnnounced() { 58 + return false, nil 59 + } 60 + 61 + _, ok := gipc.Countries[strings.ToLower(ipInfo.GetCountryCode())] 62 + 63 + return ok, nil 64 + } 65 + 66 + func (gipc *GeoIPChecker) Hash() string { 67 + return gipc.hash 68 + }

+63

internal/thoth/geoipchecker_test.go

··· 1 + package thoth_test 2 + 3 + import ( 4 + "fmt" 5 + "net/http/httptest" 6 + "testing" 7 + 8 + "github.com/TecharoHQ/anubis/internal/thoth" 9 + "github.com/TecharoHQ/anubis/lib/policy/checker" 10 + ) 11 + 12 + var _ checker.Impl = &thoth.GeoIPChecker{} 13 + 14 + func TestGeoIPChecker(t *testing.T) { 15 + cli := loadSecrets(t) 16 + 17 + asnc := cli.GeoIPCheckerFor([]string{"us"}) 18 + 19 + for _, cs := range []struct { 20 + ipAddress string 21 + wantMatch bool 22 + wantError bool 23 + }{ 24 + { 25 + ipAddress: "1.1.1.1", 26 + wantMatch: true, 27 + wantError: false, 28 + }, 29 + { 30 + ipAddress: "2.2.2.2", 31 + wantMatch: false, 32 + wantError: false, 33 + }, 34 + { 35 + ipAddress: "taco", 36 + wantMatch: false, 37 + wantError: false, 38 + }, 39 + { 40 + ipAddress: "127.0.0.1", 41 + wantMatch: false, 42 + wantError: false, 43 + }, 44 + } { 45 + t.Run(fmt.Sprintf("%v", cs), func(t *testing.T) { 46 + req := httptest.NewRequest("GET", "/", nil) 47 + req.Header.Set("X-Real-Ip", cs.ipAddress) 48 + 49 + match, err := asnc.Check(req) 50 + 51 + if match != cs.wantMatch { 52 + t.Errorf("Wanted match: %v, got: %v", cs.wantMatch, match) 53 + } 54 + 55 + switch { 56 + case err != nil && !cs.wantError: 57 + t.Errorf("Did not want error but got: %v", err) 58 + case err == nil && cs.wantError: 59 + t.Error("Wanted error but got none") 60 + } 61 + }) 62 + } 63 + }

+88

internal/thoth/thoth.go

··· 1 + package thoth 2 + 3 + import ( 4 + "context" 5 + "crypto/tls" 6 + "fmt" 7 + "time" 8 + 9 + "github.com/TecharoHQ/anubis" 10 + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" 11 + grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" 12 + "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/timeout" 13 + "github.com/prometheus/client_golang/prometheus" 14 + "google.golang.org/grpc" 15 + "google.golang.org/grpc/credentials" 16 + "google.golang.org/grpc/credentials/insecure" 17 + healthv1 "google.golang.org/grpc/health/grpc_health_v1" 18 + ) 19 + 20 + type Client struct { 21 + conn *grpc.ClientConn 22 + health healthv1.HealthClient 23 + IPToASN iptoasnv1.IpToASNServiceClient 24 + } 25 + 26 + func New(ctx context.Context, thothURL, apiToken string, plaintext bool) (*Client, error) { 27 + clMetrics := grpcprom.NewClientMetrics( 28 + grpcprom.WithClientHandlingTimeHistogram( 29 + grpcprom.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), 30 + ), 31 + ) 32 + prometheus.DefaultRegisterer.Register(clMetrics) 33 + 34 + do := []grpc.DialOption{ 35 + grpc.WithChainUnaryInterceptor( 36 + timeout.UnaryClientInterceptor(500*time.Millisecond), 37 + clMetrics.UnaryClientInterceptor(), 38 + authUnaryClientInterceptor(apiToken), 39 + ), 40 + grpc.WithChainStreamInterceptor( 41 + clMetrics.StreamClientInterceptor(), 42 + authStreamClientInterceptor(apiToken), 43 + ), 44 + grpc.WithUserAgent(fmt.Sprint("Techaro/anubis:", anubis.Version)), 45 + } 46 + 47 + if plaintext { 48 + do = append(do, grpc.WithTransportCredentials(insecure.NewCredentials())) 49 + } else { 50 + do = append(do, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{}))) 51 + } 52 + 53 + conn, err := grpc.NewClient( 54 + thothURL, 55 + do..., 56 + ) 57 + if err != nil { 58 + return nil, fmt.Errorf("can't dial thoth at %s: %w", thothURL, err) 59 + } 60 + 61 + hc := healthv1.NewHealthClient(conn) 62 + 63 + resp, err := hc.Check(ctx, &healthv1.HealthCheckRequest{}) 64 + if err != nil { 65 + return nil, fmt.Errorf("can't verify thoth health at %s: %w", thothURL, err) 66 + } 67 + 68 + if resp.Status != healthv1.HealthCheckResponse_SERVING { 69 + return nil, fmt.Errorf("thoth is not healthy, wanted %s but got %s", healthv1.HealthCheckResponse_SERVING, resp.Status) 70 + } 71 + 72 + return &Client{ 73 + conn: conn, 74 + health: hc, 75 + IPToASN: NewIpToASNWithCache(iptoasnv1.NewIpToASNServiceClient(conn)), 76 + }, nil 77 + } 78 + 79 + func (c *Client) Close() error { 80 + if c.conn != nil { 81 + return c.conn.Close() 82 + } 83 + return nil 84 + } 85 + 86 + func (c *Client) WithIPToASNService(impl iptoasnv1.IpToASNServiceClient) { 87 + c.IPToASN = impl 88 + }

+36

internal/thoth/thoth_test.go

··· 1 + package thoth_test 2 + 3 + import ( 4 + "os" 5 + "testing" 6 + 7 + "github.com/TecharoHQ/anubis/internal/thoth" 8 + "github.com/TecharoHQ/anubis/internal/thoth/thothmock" 9 + "github.com/joho/godotenv" 10 + ) 11 + 12 + func loadSecrets(t testing.TB) *thoth.Client { 13 + t.Helper() 14 + 15 + if err := godotenv.Load(); err != nil { 16 + t.Log("using mock thoth") 17 + result := &thoth.Client{} 18 + result.WithIPToASNService(thothmock.MockIpToASNService()) 19 + return result 20 + } 21 + 22 + cli, err := thoth.New(t.Context(), os.Getenv("THOTH_URL"), os.Getenv("THOTH_API_KEY"), false) 23 + if err != nil { 24 + t.Fatal(err) 25 + } 26 + 27 + return cli 28 + } 29 + 30 + func TestNew(t *testing.T) { 31 + cli := loadSecrets(t) 32 + 33 + if err := cli.Close(); err != nil { 34 + t.Fatal(err) 35 + } 36 + }

+59

internal/thoth/thothmock/iptoasn.go

··· 1 + package thothmock 2 + 3 + import ( 4 + "context" 5 + "net/netip" 6 + 7 + iptoasnv1 "github.com/TecharoHQ/thoth-proto/gen/techaro/thoth/iptoasn/v1" 8 + "google.golang.org/grpc" 9 + "google.golang.org/grpc/codes" 10 + "google.golang.org/grpc/status" 11 + ) 12 + 13 + func MockIpToASNService() *IpToASNService { 14 + responses := map[string]*iptoasnv1.LookupResponse{ 15 + "127.0.0.1": {Announced: false}, 16 + "::1": {Announced: false}, 17 + "10.10.10.10": { 18 + Announced: true, 19 + AsNumber: 13335, 20 + Cidr: []string{"1.1.1.0/24"}, 21 + CountryCode: "US", 22 + Description: "Cloudflare", 23 + }, 24 + "2.2.2.2": { 25 + Announced: true, 26 + AsNumber: 420, 27 + Cidr: []string{"2.2.2.0/24"}, 28 + CountryCode: "CA", 29 + Description: "test canada", 30 + }, 31 + "1.1.1.1": { 32 + Announced: true, 33 + AsNumber: 13335, 34 + Cidr: []string{"1.1.1.0/24"}, 35 + CountryCode: "US", 36 + Description: "Cloudflare", 37 + }, 38 + } 39 + 40 + return &IpToASNService{Responses: responses} 41 + } 42 + 43 + type IpToASNService struct { 44 + iptoasnv1.UnimplementedIpToASNServiceServer 45 + Responses map[string]*iptoasnv1.LookupResponse 46 + } 47 + 48 + func (ip2asn *IpToASNService) Lookup(ctx context.Context, lr *iptoasnv1.LookupRequest, opts ...grpc.CallOption) (*iptoasnv1.LookupResponse, error) { 49 + if _, err := netip.ParseAddr(lr.GetIpAddress()); err != nil { 50 + return nil, err 51 + } 52 + 53 + resp, ok := ip2asn.Responses[lr.GetIpAddress()] 54 + if !ok { 55 + return nil, status.Error(codes.NotFound, "IP address not found in mock") 56 + } 57 + 58 + return resp, nil 59 + }

+17

internal/thoth/thothmock/withthothmock.go

··· 1 + package thothmock 2 + 3 + import ( 4 + "context" 5 + "testing" 6 + 7 + "github.com/TecharoHQ/anubis/internal/thoth" 8 + ) 9 + 10 + func WithMockThoth(t *testing.T) context.Context { 11 + t.Helper() 12 + 13 + thothCli := &thoth.Client{} 14 + thothCli.WithIPToASNService(MockIpToASNService()) 15 + ctx := thoth.With(t.Context(), thothCli) 16 + return ctx 17 + }

+2 -1

lib/anubis.go

··· 26 26 "github.com/TecharoHQ/anubis/internal/ogtags" 27 27 "github.com/TecharoHQ/anubis/lib/challenge" 28 28 "github.com/TecharoHQ/anubis/lib/policy" 29 + "github.com/TecharoHQ/anubis/lib/policy/checker" 29 30 "github.com/TecharoHQ/anubis/lib/policy/config" 30 31 31 32 // challenge implementations ··· 483 484 ReportAs: s.policy.DefaultDifficulty, 484 485 Algorithm: config.DefaultAlgorithm, 485 486 }, 486 - Rules: &policy.CheckerList{}, 487 + Rules: &checker.List{}, 487 488 }, nil 488 489 } 489 490

+6 -3

lib/anubis_test.go

··· 15 15 "github.com/TecharoHQ/anubis" 16 16 "github.com/TecharoHQ/anubis/data" 17 17 "github.com/TecharoHQ/anubis/internal" 18 + "github.com/TecharoHQ/anubis/internal/thoth/thothmock" 18 19 "github.com/TecharoHQ/anubis/lib/policy" 19 20 "github.com/TecharoHQ/anubis/lib/policy/config" 20 21 ) ··· 26 27 func loadPolicies(t *testing.T, fname string) *policy.ParsedConfig { 27 28 t.Helper() 28 29 29 - anubisPolicy, err := LoadPoliciesOrDefault(fname, anubis.DefaultDifficulty) 30 + ctx := thothmock.WithMockThoth(t) 31 + 32 + anubisPolicy, err := LoadPoliciesOrDefault(ctx, fname, anubis.DefaultDifficulty) 30 33 if err != nil { 31 34 t.Fatal(err) 32 35 } ··· 164 167 } 165 168 defer fin.Close() 166 169 167 - if _, err := policy.ParseConfig(fin, fname, 4); err != nil { 170 + if _, err := policy.ParseConfig(t.Context(), fin, fname, 4); err != nil { 168 171 t.Fatal(err) 169 172 } 170 173 }) ··· 313 316 314 317 for i := 1; i < 10; i++ { 315 318 t.Run(fmt.Sprint(i), func(t *testing.T) { 316 - anubisPolicy, err := LoadPoliciesOrDefault("", i) 319 + anubisPolicy, err := LoadPoliciesOrDefault(t.Context(), "", i) 317 320 if err != nil { 318 321 t.Fatal(err) 319 322 }

+3 -2

lib/config.go

··· 1 1 package lib 2 2 3 3 import ( 4 + "context" 4 5 "crypto/ed25519" 5 6 "crypto/rand" 6 7 "errors" ··· 43 44 ServeRobotsTXT bool 44 45 } 45 46 46 - func LoadPoliciesOrDefault(fname string, defaultDifficulty int) (*policy.ParsedConfig, error) { 47 + func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int) (*policy.ParsedConfig, error) { 47 48 var fin io.ReadCloser 48 49 var err error 49 50 ··· 67 68 } 68 69 }(fin) 69 70 70 - anubisPolicy, err := policy.ParseConfig(fin, fname, defaultDifficulty) 71 + anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty) 71 72 if err != nil { 72 73 return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err) 73 74 }

+15 -5

lib/config_test.go

··· 7 7 "testing" 8 8 9 9 "github.com/TecharoHQ/anubis" 10 + "github.com/TecharoHQ/anubis/internal/thoth/thothmock" 10 11 "github.com/TecharoHQ/anubis/lib/policy" 11 12 ) 12 13 13 14 func TestInvalidChallengeMethod(t *testing.T) { 14 - if _, err := LoadPoliciesOrDefault("testdata/invalid-challenge-method.yaml", 4); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) { 15 + if _, err := LoadPoliciesOrDefault(t.Context(), "testdata/invalid-challenge-method.yaml", 4); !errors.Is(err, policy.ErrChallengeRuleHasWrongAlgorithm) { 15 16 t.Fatalf("wanted error %v but got %v", policy.ErrChallengeRuleHasWrongAlgorithm, err) 16 17 } 17 18 } ··· 25 26 for _, st := range finfos { 26 27 st := st 27 28 t.Run(st.Name(), func(t *testing.T) { 28 - if _, err := LoadPoliciesOrDefault(filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err == nil { 29 + if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err == nil { 29 30 t.Fatal(err) 30 31 } else { 31 32 t.Log(err) ··· 43 44 for _, st := range finfos { 44 45 st := st 45 46 t.Run(st.Name(), func(t *testing.T) { 46 - if _, err := LoadPoliciesOrDefault(filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil { 47 - t.Fatal(err) 48 - } 47 + t.Run("with-thoth", func(t *testing.T) { 48 + ctx := thothmock.WithMockThoth(t) 49 + if _, err := LoadPoliciesOrDefault(ctx, filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil { 50 + t.Fatal(err) 51 + } 52 + }) 53 + 54 + t.Run("without-thoth", func(t *testing.T) { 55 + if _, err := LoadPoliciesOrDefault(t.Context(), filepath.Join("policy", "config", "testdata", "good", st.Name()), anubis.DefaultDifficulty); err != nil { 56 + t.Fatal(err) 57 + } 58 + }) 49 59 }) 50 60 } 51 61 }

+2 -1

lib/policy/bot.go

··· 4 4 "fmt" 5 5 6 6 "github.com/TecharoHQ/anubis/internal" 7 + "github.com/TecharoHQ/anubis/lib/policy/checker" 7 8 "github.com/TecharoHQ/anubis/lib/policy/config" 8 9 ) 9 10 10 11 type Bot struct { 11 - Rules Checker 12 + Rules checker.Impl 12 13 Challenge *config.ChallengeRules 13 14 Weight *config.Weight 14 15 Name string

+9 -39

lib/policy/checker.go

··· 9 9 "strings" 10 10 11 11 "github.com/TecharoHQ/anubis/internal" 12 + "github.com/TecharoHQ/anubis/lib/policy/checker" 12 13 "github.com/yl2chen/cidranger" 13 14 ) 14 15 ··· 16 17 ErrMisconfiguration = errors.New("[unexpected] policy: administrator misconfiguration") 17 18 ) 18 19 19 - type Checker interface { 20 - Check(*http.Request) (bool, error) 21 - Hash() string 22 - } 23 - 24 - type CheckerList []Checker 25 - 26 - func (cl CheckerList) Check(r *http.Request) (bool, error) { 27 - for _, c := range cl { 28 - ok, err := c.Check(r) 29 - if err != nil { 30 - return ok, err 31 - } 32 - if ok { 33 - return ok, nil 34 - } 35 - } 36 - 37 - return false, nil 38 - } 39 - 40 - func (cl CheckerList) Hash() string { 41 - var sb strings.Builder 42 - 43 - for _, c := range cl { 44 - fmt.Fprintln(&sb, c.Hash()) 45 - } 46 - 47 - return internal.SHA256sum(sb.String()) 48 - } 49 - 50 20 type staticHashChecker struct { 51 21 hash string 52 22 } ··· 57 27 58 28 func (s staticHashChecker) Hash() string { return s.hash } 59 29 60 - func NewStaticHashChecker(hashable string) Checker { 30 + func NewStaticHashChecker(hashable string) checker.Impl { 61 31 return staticHashChecker{hash: internal.SHA256sum(hashable)} 62 32 } 63 33 ··· 66 36 hash string 67 37 } 68 38 69 - func NewRemoteAddrChecker(cidrs []string) (Checker, error) { 39 + func NewRemoteAddrChecker(cidrs []string) (checker.Impl, error) { 70 40 ranger := cidranger.NewPCTrieRanger() 71 41 var sb strings.Builder 72 42 ··· 122 92 hash string 123 93 } 124 94 125 - func NewUserAgentChecker(rexStr string) (Checker, error) { 95 + func NewUserAgentChecker(rexStr string) (checker.Impl, error) { 126 96 return NewHeaderMatchesChecker("User-Agent", rexStr) 127 97 } 128 98 129 - func NewHeaderMatchesChecker(header, rexStr string) (Checker, error) { 99 + func NewHeaderMatchesChecker(header, rexStr string) (checker.Impl, error) { 130 100 rex, err := regexp.Compile(strings.TrimSpace(rexStr)) 131 101 if err != nil { 132 102 return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err) ··· 151 121 hash string 152 122 } 153 123 154 - func NewPathChecker(rexStr string) (Checker, error) { 124 + func NewPathChecker(rexStr string) (checker.Impl, error) { 155 125 rex, err := regexp.Compile(strings.TrimSpace(rexStr)) 156 126 if err != nil { 157 127 return nil, fmt.Errorf("%w: regex %s failed parse: %w", ErrMisconfiguration, rexStr, err) ··· 171 141 return pc.hash 172 142 } 173 143 174 - func NewHeaderExistsChecker(key string) Checker { 144 + func NewHeaderExistsChecker(key string) checker.Impl { 175 145 return headerExistsChecker{strings.TrimSpace(key)} 176 146 } 177 147 ··· 191 161 return internal.SHA256sum(hec.header) 192 162 } 193 163 194 - func NewHeadersChecker(headermap map[string]string) (Checker, error) { 195 - var result CheckerList 164 + func NewHeadersChecker(headermap map[string]string) (checker.Impl, error) { 165 + var result checker.List 196 166 var errs []error 197 167 198 168 for key, rexStr := range headermap {

+41

lib/policy/checker/checker.go

··· 1 + // Package checker defines the Checker interface and a helper utility to avoid import cycles. 2 + package checker 3 + 4 + import ( 5 + "fmt" 6 + "net/http" 7 + "strings" 8 + 9 + "github.com/TecharoHQ/anubis/internal" 10 + ) 11 + 12 + type Impl interface { 13 + Check(*http.Request) (bool, error) 14 + Hash() string 15 + } 16 + 17 + type List []Impl 18 + 19 + func (l List) Check(r *http.Request) (bool, error) { 20 + for _, c := range l { 21 + ok, err := c.Check(r) 22 + if err != nil { 23 + return ok, err 24 + } 25 + if ok { 26 + return ok, nil 27 + } 28 + } 29 + 30 + return false, nil 31 + } 32 + 33 + func (l List) Hash() string { 34 + var sb strings.Builder 35 + 36 + for _, c := range l { 37 + fmt.Fprintln(&sb, c.Hash()) 38 + } 39 + 40 + return internal.SHA256sum(sb.String()) 41 + }

+44

lib/policy/config/asn.go

··· 1 + package config 2 + 3 + import ( 4 + "errors" 5 + "fmt" 6 + ) 7 + 8 + var ( 9 + ErrPrivateASN = errors.New("bot.ASNs: you have specified a private use ASN") 10 + ) 11 + 12 + type ASNs struct { 13 + Match []uint32 `json:"match"` 14 + } 15 + 16 + func (a *ASNs) Valid() error { 17 + var errs []error 18 + 19 + for _, asn := range a.Match { 20 + if isPrivateASN(asn) { 21 + errs = append(errs, fmt.Errorf("%w: %d is private (see RFC 6996)", ErrPrivateASN, asn)) 22 + } 23 + } 24 + 25 + if len(errs) != 0 { 26 + return fmt.Errorf("bot.ASNs: invalid ASN settings: %w", errors.Join(errs...)) 27 + } 28 + 29 + return nil 30 + } 31 + 32 + // isPrivateASN checks if an ASN is in the private use area. 33 + // 34 + // Based on RFC 6996 and IANA allocations. 35 + func isPrivateASN(asn uint32) bool { 36 + switch { 37 + case asn >= 64512 && asn <= 65534: 38 + return true 39 + case asn >= 4200000000 && asn <= 4294967294: 40 + return true 41 + default: 42 + return false 43 + } 44 + }

+9 -1

lib/policy/config/config.go

··· 55 55 Name string `json:"name" yaml:"name"` 56 56 Action Rule `json:"action" yaml:"action"` 57 57 RemoteAddr []string `json:"remote_addresses,omitempty" yaml:"remote_addresses,omitempty"` 58 + 59 + // Thoth features 60 + GeoIP *GeoIP `json:"geoip,omitempty"` 61 + ASNs *ASNs `json:"asns,omitempty"` 58 62 } 59 63 60 64 func (b BotConfig) Zero() bool { ··· 66 70 b.Action != "", 67 71 len(b.RemoteAddr) != 0, 68 72 b.Challenge != nil, 73 + b.GeoIP != nil, 74 + b.ASNs != nil, 69 75 } { 70 76 if cond { 71 77 return false ··· 85 91 allFieldsEmpty := b.UserAgentRegex == nil && 86 92 b.PathRegex == nil && 87 93 len(b.RemoteAddr) == 0 && 88 - len(b.HeadersRegex) == 0 94 + len(b.HeadersRegex) == 0 && 95 + b.ASNs == nil && 96 + b.GeoIP == nil 89 97 90 98 if allFieldsEmpty && b.Expression == nil { 91 99 errs = append(errs, ErrBotMustHaveUserAgentOrPath)

+36

lib/policy/config/geoip.go

··· 1 + package config 2 + 3 + import ( 4 + "errors" 5 + "fmt" 6 + "regexp" 7 + "strings" 8 + ) 9 + 10 + var ( 11 + countryCodeRegexp = regexp.MustCompile(`^\w{2}$`) 12 + 13 + ErrNotCountryCode = errors.New("config.Bot: invalid country code") 14 + ) 15 + 16 + type GeoIP struct { 17 + Countries []string `json:"countries"` 18 + } 19 + 20 + func (g *GeoIP) Valid() error { 21 + var errs []error 22 + 23 + for i, cc := range g.Countries { 24 + if !countryCodeRegexp.MatchString(cc) { 25 + errs = append(errs, fmt.Errorf("%w: %s", ErrNotCountryCode, cc)) 26 + } 27 + 28 + g.Countries[i] = strings.ToLower(cc) 29 + } 30 + 31 + if len(errs) != 0 { 32 + return fmt.Errorf("bot.GeoIP: invalid GeoIP settings: %w", errors.Join(errs...)) 33 + } 34 + 35 + return nil 36 + }

lib/policy/config/testdata/good/challenge_cloudflare.yaml

··· 1 + bots: 2 + - name: challenge-cloudflare 3 + action: CHALLENGE 4 + asns: 5 + match: 6 + - 13335 # Cloudflare

lib/policy/config/testdata/good/geoip_us.yaml

··· 1 + bots: 2 + - name: compute-tarrif-us 3 + action: CHALLENGE 4 + geoip: 5 + countries: 6 + - US

+26 -2

lib/policy/policy.go

··· 1 1 package policy 2 2 3 3 import ( 4 + "context" 4 5 "errors" 5 6 "fmt" 6 7 "io" 8 + "log/slog" 7 9 10 + "github.com/TecharoHQ/anubis/internal/thoth" 11 + "github.com/TecharoHQ/anubis/lib/policy/checker" 8 12 "github.com/TecharoHQ/anubis/lib/policy/config" 9 13 "github.com/prometheus/client_golang/prometheus" 10 14 "github.com/prometheus/client_golang/prometheus/promauto" ··· 35 39 } 36 40 } 37 41 38 - func ParseConfig(fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) { 42 + func ParseConfig(ctx context.Context, fin io.Reader, fname string, defaultDifficulty int) (*ParsedConfig, error) { 39 43 c, err := config.Load(fin, fname) 40 44 if err != nil { 41 45 return nil, err ··· 43 47 44 48 var validationErrs []error 45 49 50 + tc, hasThothClient := thoth.FromContext(ctx) 51 + 46 52 result := NewParsedConfig(c) 47 53 result.DefaultDifficulty = defaultDifficulty 48 54 ··· 57 63 Action: b.Action, 58 64 } 59 65 60 - cl := CheckerList{} 66 + cl := checker.List{} 61 67 62 68 if len(b.RemoteAddr) > 0 { 63 69 c, err := NewRemoteAddrChecker(b.RemoteAddr) ··· 102 108 } else { 103 109 cl = append(cl, c) 104 110 } 111 + } 112 + 113 + if b.ASNs != nil { 114 + if !hasThothClient { 115 + slog.Warn("You have specified a Thoth specific check but you have no Thoth client configured. Please read https://anubis.techaro.lol/docs/admin/thoth for more information", "check", "asn", "settings", b.ASNs) 116 + continue 117 + } 118 + 119 + cl = append(cl, tc.ASNCheckerFor(b.ASNs.Match)) 120 + } 121 + 122 + if b.GeoIP != nil { 123 + if !hasThothClient { 124 + slog.Warn("You have specified a Thoth specific check but you have no Thoth client configured. Please read https://anubis.techaro.lol/docs/admin/thoth for more information", "check", "geoip", "settings", b.GeoIP) 125 + continue 126 + } 127 + 128 + cl = append(cl, tc.GeoIPCheckerFor(b.GeoIP.Countries)) 105 129 } 106 130 107 131 if b.Challenge == nil {

+31 -10

lib/policy/policy_test.go

··· 7 7 8 8 "github.com/TecharoHQ/anubis" 9 9 "github.com/TecharoHQ/anubis/data" 10 + "github.com/TecharoHQ/anubis/internal/thoth/thothmock" 10 11 ) 11 12 12 13 func TestDefaultPolicyMustParse(t *testing.T) { 14 + ctx := thothmock.WithMockThoth(t) 15 + 13 16 fin, err := data.BotPolicies.Open("botPolicies.json") 14 17 if err != nil { 15 18 t.Fatal(err) 16 19 } 17 20 defer fin.Close() 18 21 19 - if _, err := ParseConfig(fin, "botPolicies.json", anubis.DefaultDifficulty); err != nil { 22 + if _, err := ParseConfig(ctx, fin, "botPolicies.json", anubis.DefaultDifficulty); err != nil { 20 23 t.Fatalf("can't parse config: %v", err) 21 24 } 22 25 } 23 26 24 27 func TestGoodConfigs(t *testing.T) { 28 + 25 29 finfos, err := os.ReadDir("config/testdata/good") 26 30 if err != nil { 27 31 t.Fatal(err) ··· 30 34 for _, st := range finfos { 31 35 st := st 32 36 t.Run(st.Name(), func(t *testing.T) { 33 - fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) 34 - if err != nil { 35 - t.Fatal(err) 36 - } 37 - defer fin.Close() 37 + t.Run("with-thoth", func(t *testing.T) { 38 + fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) 39 + if err != nil { 40 + t.Fatal(err) 41 + } 42 + defer fin.Close() 43 + 44 + ctx := thothmock.WithMockThoth(t) 45 + if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err != nil { 46 + t.Fatal(err) 47 + } 48 + }) 49 + 50 + t.Run("without-thoth", func(t *testing.T) { 51 + fin, err := os.Open(filepath.Join("config", "testdata", "good", st.Name())) 52 + if err != nil { 53 + t.Fatal(err) 54 + } 55 + defer fin.Close() 38 56 39 - if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err != nil { 40 - t.Fatal(err) 41 - } 57 + if _, err := ParseConfig(t.Context(), fin, fin.Name(), anubis.DefaultDifficulty); err != nil { 58 + t.Fatal(err) 59 + } 60 + }) 42 61 }) 43 62 } 44 63 } 45 64 46 65 func TestBadConfigs(t *testing.T) { 66 + ctx := thothmock.WithMockThoth(t) 67 + 47 68 finfos, err := os.ReadDir("config/testdata/bad") 48 69 if err != nil { 49 70 t.Fatal(err) ··· 58 79 } 59 80 defer fin.Close() 60 81 61 - if _, err := ParseConfig(fin, fin.Name(), anubis.DefaultDifficulty); err == nil { 82 + if _, err := ParseConfig(ctx, fin, fin.Name(), anubis.DefaultDifficulty); err == nil { 62 83 t.Fatal(err) 63 84 } else { 64 85 t.Log(err)