tijs.org
simple list of pds servers with open registration
1import {
2 ENRICHMENT_BATCH_SIZE,
3 getTrackedSoftware,
4} from "../shared/constants.ts";
5import type { LatestVersionMap, VersionSoftwareMap } from "../shared/types.ts";
6import { runMigrations } from "../backend/database/migrations.ts";
7import {
8 getMetadata,
9 getServersToEnrich,
10 setMetadata,
11 updateEnrichment,
12 upsertServer,
13} from "../backend/database/queries.ts";
14import { fetchPdsList } from "../backend/services/pds-fetcher.ts";
15import { fetchRecentVersions } from "../backend/services/version-checker.ts";
16import { enrichBatch } from "../backend/services/pds-enricher.ts";
17
18export default async function () {
19 console.log("OpenPDS refresh started:", new Date().toISOString());
20
21 try {
22 await runMigrations();
23
24 // 1. Fetch and sync state.json (with ETag caching)
25 const previousStateEtag = await getMetadata("state_json_etag");
26 const { pdsList, etag: stateEtag } = await fetchPdsList(previousStateEtag);
27
28 if (stateEtag) {
29 await setMetadata("state_json_etag", stateEtag);
30 }
31
32 if (pdsList) {
33 console.log(
34 `state.json: new data (${pdsList.length} PDSes), etag=${stateEtag}`,
35 );
36 let openCount = 0;
37 for (const pds of pdsList) {
38 await upsertServer(pds.url, {
39 inviteCodeRequired: pds.inviteCodeRequired,
40 version: pds.version,
41 errorAt: pds.errorAt,
42 isOpen: pds.isOpen,
43 });
44 if (pds.isOpen) openCount++;
45 }
46 console.log(`Synced to DB: ${openCount} open, ${pdsList.length} total`);
47 } else {
48 console.log(
49 `state.json: 304 not modified, skipping upsert (etag=${stateEtag})`,
50 );
51 }
52
53 // 2. Fetch latest versions for all tracked PDS software
54 const latestVersions: LatestVersionMap = {};
55 const versionToSoftware: VersionSoftwareMap = {};
56 const trackedSoftware = getTrackedSoftware();
57
58 for (const software of trackedSoftware) {
59 const etagKey = `github_tags_etag:${software.id}`;
60 const previousEtag = await getMetadata(etagKey);
61 const result = await fetchRecentVersions(software, previousEtag);
62
63 if (result.etag) {
64 await setMetadata(etagKey, result.etag);
65 }
66
67 if (result.notModified) {
68 console.log(
69 `GitHub tags [${software.id}]: 304 not modified`,
70 );
71 // Load cached versions from metadata
72 const cachedLatest = await getMetadata(
73 `latest_version:${software.id}`,
74 );
75 if (cachedLatest) {
76 latestVersions[software.id] = cachedLatest;
77 }
78 const cachedVersions = await getMetadata(
79 `version_map:${software.id}`,
80 );
81 if (cachedVersions) {
82 for (const v of JSON.parse(cachedVersions)) {
83 versionToSoftware[v] = software.id;
84 }
85 }
86 } else if (result.latest) {
87 latestVersions[software.id] = result.latest;
88 await setMetadata(`latest_version:${software.id}`, result.latest);
89 await setMetadata(
90 `version_map:${software.id}`,
91 JSON.stringify(result.versions),
92 );
93 for (const v of result.versions) {
94 versionToSoftware[v] = software.id;
95 }
96 console.log(
97 `GitHub tags [${software.id}]: latest ${result.latest} (${result.versions.length} versions)`,
98 );
99 } else if (result.error) {
100 console.log(
101 `GitHub tags [${software.id}]: API error (using cached)`,
102 );
103 const cachedLatest = await getMetadata(
104 `latest_version:${software.id}`,
105 );
106 if (cachedLatest) {
107 latestVersions[software.id] = cachedLatest;
108 }
109 const cachedVersions = await getMetadata(
110 `version_map:${software.id}`,
111 );
112 if (cachedVersions) {
113 for (const v of JSON.parse(cachedVersions)) {
114 versionToSoftware[v] = software.id;
115 }
116 }
117 } else {
118 console.log(`GitHub tags [${software.id}]: no versions found`);
119 }
120 }
121
122 // Backward compat: keep latest_pds_version for bluesky-pds
123 if (latestVersions["bluesky-pds"]) {
124 await setMetadata("latest_pds_version", latestVersions["bluesky-pds"]);
125 }
126 // Also maintain legacy github_tags_etag for backward compat
127 const bskyEtag = await getMetadata("github_tags_etag:bluesky-pds");
128 if (bskyEtag) {
129 await setMetadata("github_tags_etag", bskyEtag);
130 }
131
132 console.log(
133 `Version map: ${
134 Object.keys(versionToSoftware).length
135 } known versions across ${trackedSoftware.length} software`,
136 );
137
138 // 3. Enrich a batch of servers
139 const toEnrich = await getServersToEnrich(ENRICHMENT_BATCH_SIZE);
140 if (toEnrich.length > 0) {
141 const cachedIpCount = toEnrich.filter((s) => s.ipAddress).length;
142 const cachedGeoCount = toEnrich.filter((s) => s.countryCode).length;
143 console.log(
144 `Enriching ${toEnrich.length} servers (${cachedIpCount} have cached IP, ${cachedGeoCount} have cached geo)`,
145 );
146
147 const { results: enriched, stats } = await enrichBatch(toEnrich);
148
149 for (const data of enriched) {
150 // Detect software by matching version against known version map
151 if (data.version) {
152 if (versionToSoftware[data.version]) {
153 // Exact match (e.g. "0.4.74" → bluesky-pds)
154 data.pdsSoftware = versionToSoftware[data.version];
155 } else {
156 // Extract semver from strings like "millipds v0.0.5.dev17+..."
157 const m = data.version.match(/(\d+\.\d+\.\d+)/);
158 if (m && m[1] !== data.version && versionToSoftware[m[1]]) {
159 data.pdsSoftware = versionToSoftware[m[1]];
160 }
161 }
162 }
163
164 await updateEnrichment(data.url, {
165 version: data.version,
166 did: data.did,
167 phoneVerification: data.phoneVerification,
168 userDomains: data.userDomains,
169 contactEmail: data.contactEmail,
170 privacyPolicy: data.privacyPolicy,
171 termsOfService: data.termsOfService,
172 userCount: data.userCount,
173 countryCode: data.countryCode,
174 countryName: data.countryName,
175 ipAddress: data.ipAddress,
176 pdsSoftware: data.pdsSoftware,
177 });
178 }
179
180 const noUserCount = enriched.filter((s) => s.userCount === null).length;
181 const softwareCounts: Record<string, number> = {};
182 for (const s of enriched) {
183 const sw = s.pdsSoftware ?? "unknown";
184 softwareCounts[sw] = (softwareCounts[sw] ?? 0) + 1;
185 }
186 console.log(
187 `Enriched ${enriched.length} servers: ` +
188 `DNS ${stats.cachedDns} cached/${stats.freshDns} fresh, ` +
189 `geo ${stats.cachedGeo} cached/${stats.freshGeo} fresh` +
190 (noUserCount > 0 ? `, ${noUserCount} without user count` : "") +
191 `, software: ${JSON.stringify(softwareCounts)}`,
192 );
193 }
194
195 await setMetadata("last_full_refresh", new Date().toISOString());
196 console.log("OpenPDS refresh completed");
197 } catch (err) {
198 console.error("OpenPDS refresh failed:", err);
199 }
200}