tijs.org
simple list of pds servers with open registration
Add multi-PDS software support with per-software version tracking
Servers running alternative PDS implementations (millipds, cirrus, cocoon,
etc.) are now detected by matching their version against known GitHub tags
and compared against their own software's latest release for the trust
score. Link-only software without version tracking gets benefit of the
doubt. Software name shows as a clickable link in the directory. JSON API
includes per-software latest versions alongside the existing field.
+9
backend/database/migrations.ts
+9
backend/database/migrations.ts
+48
-5
backend/database/queries.ts
+48
-5
backend/database/queries.ts
···
1
import { sqlite } from "https://esm.town/v/stevekrouse/sqlite?v=13";
2
import { METADATA_TABLE, SERVERS_TABLE } from "../../shared/constants.ts";
3
-
import type { PdsServer, ServerToEnrich } from "../../shared/types.ts";
4
5
// --- Server operations ---
6
···
56
countryCode?: string | null;
57
countryName?: string | null;
58
ipAddress?: string | null;
59
},
60
): Promise<void> {
61
const now = new Date().toISOString();
···
73
user_count = COALESCE(?, user_count),
74
country_code = COALESCE(?, country_code),
75
country_name = COALESCE(?, country_name),
76
-
ip_address = COALESCE(?, ip_address)
77
WHERE url = ?
78
`,
79
args: [
···
89
data.countryCode ?? null,
90
data.countryName ?? null,
91
data.ipAddress ?? null,
92
url,
93
],
94
});
···
123
}
124
125
export async function getOpenServers(
126
-
latestVersion: string | null = null,
127
): Promise<PdsServer[]> {
128
const trustScoreExpr = `(
129
(CASE WHEN contact_email IS NOT NULL THEN 20 ELSE 0 END) +
130
(CASE WHEN terms_of_service IS NOT NULL THEN 20 ELSE 0 END) +
131
(CASE WHEN privacy_policy IS NOT NULL THEN 20 ELSE 0 END) +
132
(CASE WHEN user_count > 5 THEN 20 ELSE 0 END) +
133
-
(CASE WHEN version = ? THEN 20 ELSE 0 END)
134
)`;
135
136
const result = await sqlite.execute({
···
139
WHERE is_open = 1 AND last_enriched IS NOT NULL AND version IS NOT NULL
140
ORDER BY ${trustScoreExpr} DESC, user_count DESC NULLS LAST
141
`,
142
-
args: [latestVersion ?? ""],
143
});
144
145
return result.rows.map(rowToServer);
···
194
last_seen: row.last_seen as string,
195
last_enriched: (row.last_enriched as string) || null,
196
version: (row.version as string) || null,
197
did: (row.did as string) || null,
198
invite_code_required: Boolean(row.invite_code_required),
199
phone_verification: Boolean(row.phone_verification),
···
1
import { sqlite } from "https://esm.town/v/stevekrouse/sqlite?v=13";
2
import { METADATA_TABLE, SERVERS_TABLE } from "../../shared/constants.ts";
3
+
import type {
4
+
LatestVersionMap,
5
+
PdsServer,
6
+
ServerToEnrich,
7
+
} from "../../shared/types.ts";
8
9
// --- Server operations ---
10
···
60
countryCode?: string | null;
61
countryName?: string | null;
62
ipAddress?: string | null;
63
+
pdsSoftware?: string | null;
64
},
65
): Promise<void> {
66
const now = new Date().toISOString();
···
78
user_count = COALESCE(?, user_count),
79
country_code = COALESCE(?, country_code),
80
country_name = COALESCE(?, country_name),
81
+
ip_address = COALESCE(?, ip_address),
82
+
pds_software = COALESCE(?, pds_software)
83
WHERE url = ?
84
`,
85
args: [
···
95
data.countryCode ?? null,
96
data.countryName ?? null,
97
data.ipAddress ?? null,
98
+
data.pdsSoftware ?? null,
99
url,
100
],
101
});
···
130
}
131
132
export async function getOpenServers(
133
+
latestVersions: LatestVersionMap = {},
134
): Promise<PdsServer[]> {
135
+
// Build per-software version CASE expression
136
+
// Servers with known tracked software: compare against that software's latest
137
+
// Servers with link-only software (not in latestVersions): pass (benefit of the doubt)
138
+
// Servers with NULL pds_software: default to bluesky-pds comparison
139
+
const trackedIds = Object.keys(latestVersions);
140
+
const args: (string | number)[] = [];
141
+
142
+
let versionCaseExpr: string;
143
+
if (trackedIds.length === 0) {
144
+
versionCaseExpr = "0";
145
+
} else {
146
+
const whenClauses: string[] = [];
147
+
for (const id of trackedIds) {
148
+
// Match servers assigned to this software
149
+
whenClauses.push("WHEN pds_software = ? AND version = ? THEN 20");
150
+
args.push(id, latestVersions[id]);
151
+
}
152
+
// NULL pds_software defaults to bluesky-pds comparison
153
+
if (latestVersions["bluesky-pds"]) {
154
+
whenClauses.push(
155
+
"WHEN pds_software IS NULL AND version = ? THEN 20",
156
+
);
157
+
args.push(latestVersions["bluesky-pds"]);
158
+
}
159
+
// Software with no entry in latestVersions (link-only): pass
160
+
whenClauses.push(
161
+
`WHEN pds_software IS NOT NULL AND pds_software NOT IN (${
162
+
trackedIds.map(() => "?").join(",")
163
+
}) THEN 20`,
164
+
);
165
+
args.push(...trackedIds);
166
+
167
+
versionCaseExpr = `CASE ${whenClauses.join(" ")} ELSE 0 END`;
168
+
}
169
+
170
const trustScoreExpr = `(
171
(CASE WHEN contact_email IS NOT NULL THEN 20 ELSE 0 END) +
172
(CASE WHEN terms_of_service IS NOT NULL THEN 20 ELSE 0 END) +
173
(CASE WHEN privacy_policy IS NOT NULL THEN 20 ELSE 0 END) +
174
(CASE WHEN user_count > 5 THEN 20 ELSE 0 END) +
175
+
${versionCaseExpr}
176
)`;
177
178
const result = await sqlite.execute({
···
181
WHERE is_open = 1 AND last_enriched IS NOT NULL AND version IS NOT NULL
182
ORDER BY ${trustScoreExpr} DESC, user_count DESC NULLS LAST
183
`,
184
+
args,
185
});
186
187
return result.rows.map(rowToServer);
···
236
last_seen: row.last_seen as string,
237
last_enriched: (row.last_enriched as string) || null,
238
version: (row.version as string) || null,
239
+
pds_software: (row.pds_software as string) || null,
240
did: (row.did as string) || null,
241
invite_code_required: Boolean(row.invite_code_required),
242
phone_verification: Boolean(row.phone_verification),
+14
-3
backend/routes/api.ts
+14
-3
backend/routes/api.ts
···
4
getMetadata,
5
getServerCount,
6
} from "../database/queries.ts";
7
8
const api = new Hono();
9
10
api.get("/servers", async (c) => {
11
-
const [servers, latestVersion, counts] = await Promise.all([
12
getAllServers(),
13
-
getMetadata("latest_pds_version"),
14
getServerCount(),
15
]);
16
···
20
);
21
return c.json({
22
meta: {
23
-
latest_pds_version: latestVersion,
24
total_known: counts.total,
25
total_open: counts.open,
26
updated_at: new Date().toISOString(),
···
4
getMetadata,
5
getServerCount,
6
} from "../database/queries.ts";
7
+
import { getTrackedSoftware } from "../../shared/constants.ts";
8
+
import type { LatestVersionMap } from "../../shared/types.ts";
9
10
const api = new Hono();
11
12
api.get("/servers", async (c) => {
13
+
// Fetch per-software latest versions
14
+
const tracked = getTrackedSoftware();
15
+
const latestVersions: LatestVersionMap = {};
16
+
const versionPromises = tracked.map(async (sw) => {
17
+
const v = await getMetadata(`latest_version:${sw.id}`);
18
+
if (v) latestVersions[sw.id] = v;
19
+
});
20
+
21
+
const [servers, , counts] = await Promise.all([
22
getAllServers(),
23
+
Promise.all(versionPromises),
24
getServerCount(),
25
]);
26
···
30
);
31
return c.json({
32
meta: {
33
+
latest_pds_version: latestVersions["bluesky-pds"] ?? null,
34
+
latest_versions: latestVersions,
35
total_known: counts.total,
36
total_open: counts.open,
37
updated_at: new Date().toISOString(),
+70
-17
backend/routes/pages.ts
+70
-17
backend/routes/pages.ts
···
4
getOpenServers,
5
getServerCount,
6
} from "../database/queries.ts";
7
-
import { countryFlag } from "../../shared/constants.ts";
8
-
import type { PdsServer } from "../../shared/types.ts";
9
import { isSafeHref } from "../../shared/url-validation.ts";
10
11
const pages = new Hono();
12
13
pages.get("/", async (c) => {
14
-
const [latestVersion, counts] = await Promise.all([
15
-
getMetadata("latest_pds_version"),
16
getServerCount(),
17
]);
18
19
-
const servers = await getOpenServers(latestVersion);
20
21
c.header(
22
"Cache-Control",
23
"public, max-age=3600, stale-while-revalidate=3600",
24
);
25
return c.html(renderPage(servers, {
26
-
latestVersion,
27
openCount: counts.open,
28
totalCount: counts.total,
29
}));
30
});
31
32
type PageOpts = {
33
-
latestVersion: string | null;
34
openCount: number;
35
totalCount: number;
36
};
37
38
type TrustResult = { score: number; breakdown: string };
39
40
-
function trustScore(s: PdsServer, latestVersion: string | null): TrustResult {
41
const signals: string[] = [];
42
let score = 0;
43
···
50
check(!!s.terms_of_service, "Terms of Service");
51
check(!!s.privacy_policy, "Privacy Policy");
52
check(s.user_count != null && s.user_count > 5, "Active users (>5)");
53
-
check(s.version === latestVersion, "Latest version");
54
55
return { score, breakdown: signals.join("\n") };
56
}
57
58
function renderPage(servers: PdsServer[], opts: PageOpts): string {
59
-
const { latestVersion, openCount, totalCount } = opts;
60
61
const rows = servers
62
-
.map((s) => renderRow(s, latestVersion, trustScore(s, latestVersion)))
63
.join("\n");
64
65
return `<!DOCTYPE html>
66
<html lang="en">
67
<head>
···
107
margin-bottom: 1rem;
108
font-size: 0.85rem;
109
color: var(--muted);
110
}
111
table {
112
width: 100%;
···
159
.trust-mid { background: #451a03; }
160
.trust-low { background: #1f2937; }
161
}
162
.move-btn {
163
display: inline-block;
164
padding: 0.2rem 0.5rem;
···
207
<div class="stats">
208
<span>${servers.length} servers listed</span>
209
<span>${openCount} open of ${totalCount} known</span>
210
-
${latestVersion ? `<span>Latest PDS: ${esc(latestVersion)}</span>` : ""}
211
</div>
212
${
213
servers.length > 0
···
238
<li><strong>Terms of Service</strong> — published terms for using the server</li>
239
<li><strong>Privacy Policy</strong> — published data handling policy</li>
240
<li><strong>Active users</strong> — more than 5 accounts on the server</li>
241
-
<li><strong>Latest version</strong> — running the most recent PDS software</li>
242
</ul>
243
<p>This is not an endorsement. Always review a server's policies before creating an account.</p>
244
</div>
···
253
254
function renderRow(
255
s: PdsServer,
256
-
latestVersion: string | null,
257
trust: TrustResult,
258
): string {
259
const { score, breakdown } = trust;
···
262
const flag = s.country_code ? esc(countryFlag(s.country_code)) + " " : "";
263
const country = s.country_name || "";
264
265
let versionClass = "version-outdated";
266
-
const versionText = s.version || "unknown";
267
-
if (s.version === latestVersion) versionClass = "version-current";
268
269
const users = s.user_count != null
270
? (s.user_count >= 1000 ? "1000+" : String(s.user_count))
···
294
<td class="hide-mobile">${flag}${esc(country)}</td>
295
<td><span class="version-badge ${versionClass}">${
296
esc(versionText)
297
-
}</span></td>
298
<td>${esc(users)}</td>
299
<td class="hide-mobile">${esc(firstSeen)}</td>
300
<td><a class="move-btn" href="https://pdsmoover.com/moover/${
···
4
getOpenServers,
5
getServerCount,
6
} from "../database/queries.ts";
7
+
import {
8
+
countryFlag,
9
+
getSoftwareConfig,
10
+
getTrackedSoftware,
11
+
} from "../../shared/constants.ts";
12
+
import type { LatestVersionMap, PdsServer } from "../../shared/types.ts";
13
import { isSafeHref } from "../../shared/url-validation.ts";
14
15
const pages = new Hono();
16
17
pages.get("/", async (c) => {
18
+
// Fetch per-software latest versions
19
+
const tracked = getTrackedSoftware();
20
+
const latestVersions: LatestVersionMap = {};
21
+
const versionPromises = tracked.map(async (sw) => {
22
+
const v = await getMetadata(`latest_version:${sw.id}`);
23
+
if (v) latestVersions[sw.id] = v;
24
+
});
25
+
const [, counts] = await Promise.all([
26
+
Promise.all(versionPromises),
27
getServerCount(),
28
]);
29
30
+
const servers = await getOpenServers(latestVersions);
31
32
c.header(
33
"Cache-Control",
34
"public, max-age=3600, stale-while-revalidate=3600",
35
);
36
return c.html(renderPage(servers, {
37
+
latestVersions,
38
openCount: counts.open,
39
totalCount: counts.total,
40
}));
41
});
42
43
type PageOpts = {
44
+
latestVersions: LatestVersionMap;
45
openCount: number;
46
totalCount: number;
47
};
48
49
type TrustResult = { score: number; breakdown: string };
50
51
+
function trustScore(
52
+
s: PdsServer,
53
+
latestVersions: LatestVersionMap,
54
+
): TrustResult {
55
const signals: string[] = [];
56
let score = 0;
57
···
64
check(!!s.terms_of_service, "Terms of Service");
65
check(!!s.privacy_policy, "Privacy Policy");
66
check(s.user_count != null && s.user_count > 5, "Active users (>5)");
67
+
68
+
// Version check: compare against the server's own software latest version
69
+
const softwareId = s.pds_software ?? "bluesky-pds";
70
+
const expectedVersion = latestVersions[softwareId];
71
+
if (!expectedVersion) {
72
+
// Link-only software with no tracked version: benefit of the doubt
73
+
check(true, "Latest version");
74
+
} else {
75
+
check(s.version === expectedVersion, "Latest version");
76
+
}
77
78
return { score, breakdown: signals.join("\n") };
79
}
80
81
function renderPage(servers: PdsServer[], opts: PageOpts): string {
82
+
const { latestVersions, openCount, totalCount } = opts;
83
84
const rows = servers
85
+
.map((s) => renderRow(s, latestVersions, trustScore(s, latestVersions)))
86
.join("\n");
87
88
+
// Build stats for tracked software versions
89
+
const versionStats = getTrackedSoftware()
90
+
.filter((sw) => latestVersions[sw.id])
91
+
.map((sw) =>
92
+
`<span>${esc(sw.displayName)}: ${esc(latestVersions[sw.id])}</span>`
93
+
)
94
+
.join("\n ");
95
+
96
return `<!DOCTYPE html>
97
<html lang="en">
98
<head>
···
138
margin-bottom: 1rem;
139
font-size: 0.85rem;
140
color: var(--muted);
141
+
flex-wrap: wrap;
142
}
143
table {
144
width: 100%;
···
191
.trust-mid { background: #451a03; }
192
.trust-low { background: #1f2937; }
193
}
194
+
.software-link {
195
+
font-size: 0.7rem;
196
+
color: var(--muted);
197
+
text-decoration: none;
198
+
margin-left: 0.3rem;
199
+
}
200
+
.software-link:hover { text-decoration: underline; }
201
.move-btn {
202
display: inline-block;
203
padding: 0.2rem 0.5rem;
···
246
<div class="stats">
247
<span>${servers.length} servers listed</span>
248
<span>${openCount} open of ${totalCount} known</span>
249
+
${versionStats}
250
</div>
251
${
252
servers.length > 0
···
277
<li><strong>Terms of Service</strong> — published terms for using the server</li>
278
<li><strong>Privacy Policy</strong> — published data handling policy</li>
279
<li><strong>Active users</strong> — more than 5 accounts on the server</li>
280
+
<li><strong>Latest version</strong> — running the latest release of its PDS software (servers are compared against their own software's latest version)</li>
281
</ul>
282
<p>This is not an endorsement. Always review a server's policies before creating an account.</p>
283
</div>
···
292
293
function renderRow(
294
s: PdsServer,
295
+
latestVersions: LatestVersionMap,
296
trust: TrustResult,
297
): string {
298
const { score, breakdown } = trust;
···
301
const flag = s.country_code ? esc(countryFlag(s.country_code)) + " " : "";
302
const country = s.country_name || "";
303
304
+
// Determine version badge class based on per-software comparison
305
+
const softwareId = s.pds_software ?? "bluesky-pds";
306
+
const expectedVersion = latestVersions[softwareId];
307
+
const versionText = s.version || "unknown";
308
let versionClass = "version-outdated";
309
+
if (!expectedVersion || s.version === expectedVersion) {
310
+
versionClass = "version-current";
311
+
}
312
+
313
+
// Software link (show name as clickable link if recognized)
314
+
let softwareHtml = "";
315
+
const config = s.pds_software ? getSoftwareConfig(s.pds_software) : null;
316
+
if (config && config.id !== "bluesky-pds") {
317
+
softwareHtml = `<a class="software-link" href="${
318
+
esc(config.projectUrl)
319
+
}" target="_blank" rel="noopener">${esc(config.displayName)}</a>`;
320
+
}
321
322
const users = s.user_count != null
323
? (s.user_count >= 1000 ? "1000+" : String(s.user_count))
···
347
<td class="hide-mobile">${flag}${esc(country)}</td>
348
<td><span class="version-badge ${versionClass}">${
349
esc(versionText)
350
+
}</span>${softwareHtml}</td>
351
<td>${esc(users)}</td>
352
<td class="hide-mobile">${esc(firstSeen)}</td>
353
<td><a class="move-btn" href="https://pdsmoover.com/moover/${
+2
backend/services/pds-enricher.ts
+2
backend/services/pds-enricher.ts
···
18
export type EnrichmentResult = {
19
url: string;
20
version: string | null;
21
+
pdsSoftware: string | null;
22
did: string | null;
23
phoneVerification: boolean;
24
userDomains: string[];
···
231
return {
232
url,
233
version: null,
234
+
pdsSoftware: null,
235
did: null,
236
phoneVerification: false,
237
userDomains: [],
+54
-17
backend/services/version-checker.ts
+54
-17
backend/services/version-checker.ts
···
1
-
import { PDS_REPO_NAME, PDS_REPO_OWNER } from "../../shared/constants.ts";
2
3
export type FetchVersionResult = {
4
version: string | null;
···
7
error: boolean;
8
};
9
10
-
/** Fetch latest PDS version from GitHub with ETag caching. */
11
export async function fetchLatestPdsVersion(
12
previousEtag?: string | null,
13
): Promise<FetchVersionResult> {
14
const url =
15
-
`https://api.github.com/repos/${PDS_REPO_OWNER}/${PDS_REPO_NAME}/tags?per_page=5`;
16
17
const headers: Record<string, string> = {
18
"Accept": "application/vnd.github.v3+json",
···
27
28
if (resp.status === 304) {
29
return {
30
-
version: null,
31
etag: etag ?? previousEtag ?? null,
32
notModified: true,
33
error: false,
···
35
}
36
37
if (!resp.ok) {
38
-
console.error(`GitHub API error: ${resp.status}`);
39
-
// Preserve previous etag on error so we can retry conditional request next time
40
return {
41
-
version: null,
42
etag: previousEtag ?? null,
43
notModified: false,
44
error: true,
···
46
}
47
48
const tags: Array<{ name: string }> = await resp.json();
49
-
if (tags.length === 0) {
50
-
return {
51
-
version: null,
52
-
etag: etag ?? null,
53
-
notModified: false,
54
-
error: false,
55
-
};
56
}
57
58
-
// Tags are like "v0.4.74" — strip the "v" prefix to match _health output
59
-
const version = tags[0].name.replace(/^v/, "");
60
-
return { version, etag: etag ?? null, notModified: false, error: false };
61
}
···
1
+
import type { PdsSoftwareConfig } from "../../shared/constants.ts";
2
3
export type FetchVersionResult = {
4
version: string | null;
···
7
error: boolean;
8
};
9
10
+
export type FetchRecentVersionsResult = {
11
+
latest: string | null;
12
+
versions: string[];
13
+
etag: string | null;
14
+
notModified: boolean;
15
+
error: boolean;
16
+
};
17
+
18
+
/** Fetch latest version for a PDS software from its GitHub tags with ETag caching. */
19
export async function fetchLatestPdsVersion(
20
+
software: PdsSoftwareConfig,
21
previousEtag?: string | null,
22
): Promise<FetchVersionResult> {
23
+
const result = await fetchRecentVersions(software, previousEtag);
24
+
return {
25
+
version: result.latest,
26
+
etag: result.etag,
27
+
notModified: result.notModified,
28
+
error: result.error,
29
+
};
30
+
}
31
+
32
+
/** Fetch up to 30 recent parsed versions for a PDS software from GitHub tags. */
33
+
export async function fetchRecentVersions(
34
+
software: PdsSoftwareConfig,
35
+
previousEtag?: string | null,
36
+
): Promise<FetchRecentVersionsResult> {
37
+
if (!software.githubOwner || !software.githubRepo || !software.parseVersion) {
38
+
return {
39
+
latest: null,
40
+
versions: [],
41
+
etag: null,
42
+
notModified: false,
43
+
error: true,
44
+
};
45
+
}
46
+
47
const url =
48
+
`https://api.github.com/repos/${software.githubOwner}/${software.githubRepo}/tags?per_page=30`;
49
50
const headers: Record<string, string> = {
51
"Accept": "application/vnd.github.v3+json",
···
60
61
if (resp.status === 304) {
62
return {
63
+
latest: null,
64
+
versions: [],
65
etag: etag ?? previousEtag ?? null,
66
notModified: true,
67
error: false,
···
69
}
70
71
if (!resp.ok) {
72
+
console.error(
73
+
`GitHub API error for ${software.id}: ${resp.status}`,
74
+
);
75
return {
76
+
latest: null,
77
+
versions: [],
78
etag: previousEtag ?? null,
79
notModified: false,
80
error: true,
···
82
}
83
84
const tags: Array<{ name: string }> = await resp.json();
85
+
const versions: string[] = [];
86
+
for (const tag of tags) {
87
+
const v = software.parseVersion(tag.name);
88
+
if (v) versions.push(v);
89
}
90
91
+
return {
92
+
latest: versions.length > 0 ? versions[0] : null,
93
+
versions,
94
+
etag: etag ?? null,
95
+
notModified: false,
96
+
error: false,
97
+
};
98
}
+99
-26
cron/refresh.cron.ts
+99
-26
cron/refresh.cron.ts
···
1
-
import { ENRICHMENT_BATCH_SIZE } from "../shared/constants.ts";
2
import { runMigrations } from "../backend/database/migrations.ts";
3
import {
4
getMetadata,
···
8
upsertServer,
9
} from "../backend/database/queries.ts";
10
import { fetchPdsList } from "../backend/services/pds-fetcher.ts";
11
-
import { fetchLatestPdsVersion } from "../backend/services/version-checker.ts";
12
import { enrichBatch } from "../backend/services/pds-enricher.ts";
13
14
export default async function () {
···
46
);
47
}
48
49
-
// 2. Fetch latest PDS version (with ETag caching)
50
-
const previousGithubEtag = await getMetadata("github_tags_etag");
51
-
const {
52
-
version: latestVersion,
53
-
etag: githubEtag,
54
-
notModified,
55
-
error: githubError,
56
-
} = await fetchLatestPdsVersion(previousGithubEtag);
57
58
-
if (githubEtag) {
59
-
await setMetadata("github_tags_etag", githubEtag);
60
}
61
62
-
if (notModified) {
63
-
console.log(
64
-
`GitHub tags: 304 not modified (etag=${githubEtag})`,
65
-
);
66
-
} else if (latestVersion) {
67
-
await setMetadata("latest_pds_version", latestVersion);
68
-
console.log(
69
-
`GitHub tags: new version ${latestVersion} (etag=${githubEtag})`,
70
-
);
71
-
} else if (githubError) {
72
-
console.log("GitHub tags: API error (using cached version)");
73
-
} else {
74
-
console.log("GitHub tags: no version found");
75
}
76
77
// 3. Enrich a batch of servers
78
const toEnrich = await getServersToEnrich(ENRICHMENT_BATCH_SIZE);
79
if (toEnrich.length > 0) {
···
86
const { results: enriched, stats } = await enrichBatch(toEnrich);
87
88
for (const data of enriched) {
89
await updateEnrichment(data.url, {
90
version: data.version,
91
did: data.did,
···
98
countryCode: data.countryCode,
99
countryName: data.countryName,
100
ipAddress: data.ipAddress,
101
});
102
}
103
104
const noUserCount = enriched.filter((s) => s.userCount === null).length;
105
console.log(
106
`Enriched ${enriched.length} servers: ` +
107
`DNS ${stats.cachedDns} cached/${stats.freshDns} fresh, ` +
108
`geo ${stats.cachedGeo} cached/${stats.freshGeo} fresh` +
109
-
(noUserCount > 0 ? `, ${noUserCount} without user count` : ""),
110
);
111
}
112
···
1
+
import {
2
+
ENRICHMENT_BATCH_SIZE,
3
+
getTrackedSoftware,
4
+
} from "../shared/constants.ts";
5
+
import type { LatestVersionMap, VersionSoftwareMap } from "../shared/types.ts";
6
import { runMigrations } from "../backend/database/migrations.ts";
7
import {
8
getMetadata,
···
12
upsertServer,
13
} from "../backend/database/queries.ts";
14
import { fetchPdsList } from "../backend/services/pds-fetcher.ts";
15
+
import { fetchRecentVersions } from "../backend/services/version-checker.ts";
16
import { enrichBatch } from "../backend/services/pds-enricher.ts";
17
18
export default async function () {
···
50
);
51
}
52
53
+
// 2. Fetch latest versions for all tracked PDS software
54
+
const latestVersions: LatestVersionMap = {};
55
+
const versionToSoftware: VersionSoftwareMap = {};
56
+
const trackedSoftware = getTrackedSoftware();
57
58
+
for (const software of trackedSoftware) {
59
+
const etagKey = `github_tags_etag:${software.id}`;
60
+
const previousEtag = await getMetadata(etagKey);
61
+
const result = await fetchRecentVersions(software, previousEtag);
62
+
63
+
if (result.etag) {
64
+
await setMetadata(etagKey, result.etag);
65
+
}
66
+
67
+
if (result.notModified) {
68
+
console.log(
69
+
`GitHub tags [${software.id}]: 304 not modified`,
70
+
);
71
+
// Load cached versions from metadata
72
+
const cachedLatest = await getMetadata(
73
+
`latest_version:${software.id}`,
74
+
);
75
+
if (cachedLatest) {
76
+
latestVersions[software.id] = cachedLatest;
77
+
}
78
+
const cachedVersions = await getMetadata(
79
+
`version_map:${software.id}`,
80
+
);
81
+
if (cachedVersions) {
82
+
for (const v of JSON.parse(cachedVersions)) {
83
+
versionToSoftware[v] = software.id;
84
+
}
85
+
}
86
+
} else if (result.latest) {
87
+
latestVersions[software.id] = result.latest;
88
+
await setMetadata(`latest_version:${software.id}`, result.latest);
89
+
await setMetadata(
90
+
`version_map:${software.id}`,
91
+
JSON.stringify(result.versions),
92
+
);
93
+
for (const v of result.versions) {
94
+
versionToSoftware[v] = software.id;
95
+
}
96
+
console.log(
97
+
`GitHub tags [${software.id}]: latest ${result.latest} (${result.versions.length} versions)`,
98
+
);
99
+
} else if (result.error) {
100
+
console.log(
101
+
`GitHub tags [${software.id}]: API error (using cached)`,
102
+
);
103
+
const cachedLatest = await getMetadata(
104
+
`latest_version:${software.id}`,
105
+
);
106
+
if (cachedLatest) {
107
+
latestVersions[software.id] = cachedLatest;
108
+
}
109
+
const cachedVersions = await getMetadata(
110
+
`version_map:${software.id}`,
111
+
);
112
+
if (cachedVersions) {
113
+
for (const v of JSON.parse(cachedVersions)) {
114
+
versionToSoftware[v] = software.id;
115
+
}
116
+
}
117
+
} else {
118
+
console.log(`GitHub tags [${software.id}]: no versions found`);
119
+
}
120
}
121
122
+
// Backward compat: keep latest_pds_version for bluesky-pds
123
+
if (latestVersions["bluesky-pds"]) {
124
+
await setMetadata("latest_pds_version", latestVersions["bluesky-pds"]);
125
+
}
126
+
// Also maintain legacy github_tags_etag for backward compat
127
+
const bskyEtag = await getMetadata("github_tags_etag:bluesky-pds");
128
+
if (bskyEtag) {
129
+
await setMetadata("github_tags_etag", bskyEtag);
130
}
131
132
+
console.log(
133
+
`Version map: ${
134
+
Object.keys(versionToSoftware).length
135
+
} known versions across ${trackedSoftware.length} software`,
136
+
);
137
+
138
// 3. Enrich a batch of servers
139
const toEnrich = await getServersToEnrich(ENRICHMENT_BATCH_SIZE);
140
if (toEnrich.length > 0) {
···
147
const { results: enriched, stats } = await enrichBatch(toEnrich);
148
149
for (const data of enriched) {
150
+
// Detect software by matching version against known version map
151
+
if (data.version && versionToSoftware[data.version]) {
152
+
data.pdsSoftware = versionToSoftware[data.version];
153
+
}
154
+
155
await updateEnrichment(data.url, {
156
version: data.version,
157
did: data.did,
···
164
countryCode: data.countryCode,
165
countryName: data.countryName,
166
ipAddress: data.ipAddress,
167
+
pdsSoftware: data.pdsSoftware,
168
});
169
}
170
171
const noUserCount = enriched.filter((s) => s.userCount === null).length;
172
+
const softwareCounts: Record<string, number> = {};
173
+
for (const s of enriched) {
174
+
const sw = s.pdsSoftware ?? "unknown";
175
+
softwareCounts[sw] = (softwareCounts[sw] ?? 0) + 1;
176
+
}
177
console.log(
178
`Enriched ${enriched.length} servers: ` +
179
`DNS ${stats.cachedDns} cached/${stats.freshDns} fresh, ` +
180
`geo ${stats.cachedGeo} cached/${stats.freshGeo} fresh` +
181
+
(noUserCount > 0 ? `, ${noUserCount} without user count` : "") +
182
+
`, software: ${JSON.stringify(softwareCounts)}`,
183
);
184
}
185