jordanreger.com
An HTML-only Bluesky frontend
update robots.txt
+1
-116
static/robots.txt
+1
-116
static/robots.txt
···
1
-
# Our policy (taken from sr.ht, see LICENSE below)
2
-
#
3
-
# Allowed:
4
-
# - Search engine indexers
5
-
# - Archival services (e.g. IA)
6
-
#
7
-
# Disallowed:
8
-
# - Marketing or SEO crawlers
9
-
# - Bots which are too agressive by default. This is subjective, if you annoy
10
-
# our sysadmins you'll be blocked.
11
-
#
12
-
# Reach out to mail@jordanreger.com if you have questions.
13
-
14
-
# Too aggressive, marketing/SEO
15
-
User-agent: SemrushBot
16
-
Disallow: /
17
-
18
-
# Too aggressive, marketing/SEO
19
-
User-agent: SemrushBot-SA
20
-
Disallow: /
21
-
22
-
# Marketing/SEO
23
-
User-agent: AhrefsBot
24
-
Disallow: /
25
-
26
-
# Marketing/SEO
27
-
User-agent: dotbot
28
-
Disallow: /
29
-
30
-
# Marketing/SEO
31
-
User-agent: rogerbot
32
-
Disallow: /
33
-
34
-
User-agent: BLEXBot
35
-
Disallow: /
36
-
37
-
# Huwei something or another, badly behaved
38
-
User-agent: AspiegelBot
39
-
Disallow: /
40
-
41
-
# Marketing/SEO
42
-
User-agent: ZoominfoBot
43
-
Disallow: /
44
-
45
-
# YandexBot is a dickhead, too aggressive
46
-
User-agent: Yandex
47
-
Disallow: /
48
-
49
-
# Marketing/SEO
50
-
User-agent: MJ12bot
51
-
Disallow: /
52
-
53
-
# Marketing/SEO
54
-
User-agent: DataForSeoBot
55
-
Disallow: /
56
-
57
-
# Used for Alexa, I guess, who cares
58
-
User-agent: Amazonbot
1
+
User-agent: *
59
2
Disallow: /
60
-
61
-
# No
62
-
User-agent: turnitinbot
63
-
Disallow: /
64
-
65
-
User-agent: Turnitin
66
-
Disallow: /
67
-
68
-
# Does not respect * directives
69
-
User-agent: Seekport Crawler
70
-
Disallow: /
71
-
72
-
# No thanks
73
-
User-agent: GPTBot
74
-
Disallow: /
75
-
76
-
# Fairly certain that this is an LLM data vacuum
77
-
User-agent: ClaudeBot
78
-
Disallow: /
79
-
80
-
# Same
81
-
User-agent: Google-Extended
82
-
Disallow: /
83
-
84
-
# Marketing
85
-
User-agent: serpstatbot
86
-
Disallow: /
87
-
88
-
# Marketing/SEO
89
-
User-agent: barkrowler
90
-
Disallow: /
91
-
92
-
# Very aggressive, used for TikTok or something
93
-
User-agent: Bytespider
94
-
Disallow: /
95
-
96
-
97
-
# LICENSE (https://git.sr.ht/~sircmpwn/sr.ht-nginx/tree/master/item/LICENSE)
98
-
# ===============================================================================
99
-
# Copyright (c) 2020 Drew DeVault
100
-
101
-
# Permission is hereby granted, free of charge, to any person obtaining a copy of
102
-
# this software and associated documentation files (the "Software"), to deal in
103
-
# the Software without restriction, including without limitation the rights to
104
-
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
105
-
# of the Software, and to permit persons to whom the Software is furnished to do
106
-
# so, subject to the following conditions:
107
-
108
-
# The above copyright notice and this permission notice shall be included in all
109
-
# copies or substantial portions of the Software.
110
-
111
-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
112
-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
113
-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
114
-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
115
-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
116
-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
117
-
# SOFTWARE.