+6 -1

bun.lock

··· 4 4 "workspaces": { 5 5 "": { 6 6 "name": "sitebase", 7 7 + "dependencies": { 8 8 + "@atproto/api": "^0.18.16", 9 9 + }, 7 10 "devDependencies": { 8 11 "@biomejs/biome": "^2.3.11", 9 12 "@types/node": "^25.0.9", ··· 66 69 67 70 "@atproto-labs/simple-store-memory": ["@atproto-labs/simple-store-memory@0.1.4", "", { "dependencies": { "@atproto-labs/simple-store": "0.3.0", "lru-cache": "^10.2.0" } }, "sha512-3mKY4dP8I7yKPFj9VKpYyCRzGJOi5CEpOLPlRhoJyLmgs3J4RzDrjn323Oakjz2Aj2JzRU/AIvWRAZVhpYNJHw=="], 68 71 69 69 - "@atproto/api": ["@atproto/api@0.18.14", "", { "dependencies": { "@atproto/common-web": "^0.4.12", "@atproto/lexicon": "^0.6.0", "@atproto/syntax": "^0.4.2", "@atproto/xrpc": "^0.7.7", "await-lock": "^2.2.2", "multiformats": "^9.9.0", "tlds": "^1.234.0", "zod": "^3.23.8" } }, "sha512-1pWAPbuG3RA1o8uOAwYWZOddvNjuweYOxwTvys1q/r9NCjoGkZY0uJUy1dr6LKFaDk8bjikd2O1cgsRwFfv6Fw=="], 72 72 + "@atproto/api": ["@atproto/api@0.18.16", "", { "dependencies": { "@atproto/common-web": "^0.4.12", "@atproto/lexicon": "^0.6.0", "@atproto/syntax": "^0.4.2", "@atproto/xrpc": "^0.7.7", "await-lock": "^2.2.2", "multiformats": "^9.9.0", "tlds": "^1.234.0", "zod": "^3.23.8" } }, "sha512-tRGKSWr83pP5CQpSboePU21pE+GqLDYy1XHae4HH4hjaT0pr5V8wNgu70kbKB0B02GVUumeDRpJnlHKD+eMzLg=="], 70 73 71 74 "@atproto/common-web": ["@atproto/common-web@0.4.12", "", { "dependencies": { "@atproto/lex-data": "0.0.8", "@atproto/lex-json": "0.0.8", "zod": "^3.23.8" } }, "sha512-3aCJemqM/fkHQrVPbTCHCdiVstKFI+2LkFLvUhO6XZP0EqUZa/rg/CIZBKTFUWu9I5iYiaEiXL9VwcDRpEevSw=="], 72 75 ··· 171 174 "wordwrap": ["wordwrap@1.0.0", "", {}, "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q=="], 172 175 173 176 "zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="], 177 177 + 178 178 + "@sitebase/web/@atproto/api": ["@atproto/api@0.18.14", "", { "dependencies": { "@atproto/common-web": "^0.4.12", "@atproto/lexicon": "^0.6.0", "@atproto/syntax": "^0.4.2", "@atproto/xrpc": "^0.7.7", "await-lock": "^2.2.2", "multiformats": "^9.9.0", "tlds": "^1.234.0", "zod": "^3.23.8" } }, "sha512-1pWAPbuG3RA1o8uOAwYWZOddvNjuweYOxwTvys1q/r9NCjoGkZY0uJUy1dr6LKFaDk8bjikd2O1cgsRwFfv6Fw=="], 174 179 175 180 "bun-types/@types/node": ["@types/node@25.0.8", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-powIePYMmC3ibL0UJ2i2s0WIbq6cg6UyVFQxSCpaPxxzAaziRfimGivjdF943sSGV6RADVbk0Nvlm5P/FB44Zg=="], 176 181 }

+5 -1

package.json

··· 16 16 "format:check": "biome format packages/", 17 17 "lint": "biome lint packages/ --write", 18 18 "lint:check": "biome lint packages/", 19 19 - "typecheck": "bun run --workspaces typecheck" 19 19 + "typecheck": "bun run --workspaces typecheck", 20 20 + "import": "bun run scripts/import-content.ts" 21 21 + }, 22 22 + "dependencies": { 23 23 + "@atproto/api": "^0.18.16" 20 24 } 21 25 }

+361

scripts/import-content.ts

··· 1 1 + #!/usr/bin/env bun 2 2 + /** 3 3 + * Import static site content to ATProto PDS as standard.site documents 4 4 + * 5 5 + * Usage: 6 6 + * bun run scripts/import-content.ts --content-dir <path> --publication <at-uri> --identifier <handle-or-did> --password <app-password> 7 7 + * 8 8 + * Environment variables (alternative to CLI args): 9 9 + * ATPROTO_IDENTIFIER - Your handle or DID 10 10 + * ATPROTO_PASSWORD - App password (create at https://bsky.app/settings/app-passwords) 11 11 + * PDS_URL - PDS endpoint (defaults to https://bsky.social) 12 12 + */ 13 13 + 14 14 + import { readdir, readFile, stat } from "node:fs/promises"; 15 15 + import { basename, extname, join, relative } from "node:path"; 16 16 + import { AtpAgent } from "@atproto/api"; 17 17 + 18 18 + const DOCUMENT_COLLECTION = "site.standard.document"; 19 19 + 20 20 + interface Frontmatter { 21 21 + title?: string; 22 22 + date?: string; 23 23 + tags?: string[]; 24 24 + author?: { name?: string; uri?: string }; 25 25 + description?: string; 26 26 + view?: string; 27 27 + } 28 28 + 29 29 + interface ParsedDocument { 30 30 + frontmatter: Frontmatter; 31 31 + content: string; 32 32 + filePath: string; 33 33 + relativePath: string; 34 34 + } 35 35 + 36 36 + /** 37 37 + * Parse YAML-like frontmatter from markdown content 38 38 + */ 39 39 + function parseFrontmatter(content: string): { frontmatter: Frontmatter; body: string } { 40 40 + const frontmatterMatch = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/); 41 41 + 42 42 + if (!frontmatterMatch) { 43 43 + return { frontmatter: {}, body: content }; 44 44 + } 45 45 + 46 46 + const [, yamlContent, body] = frontmatterMatch; 47 47 + const frontmatter: Frontmatter = {}; 48 48 + 49 49 + // Simple YAML parser for our needs 50 50 + const lines = yamlContent!.split("\n"); 51 51 + let currentKey: string | null = null; 52 52 + let currentArray: string[] | null = null; 53 53 + let inAuthor = false; 54 54 + let authorObj: { name?: string; uri?: string } = {}; 55 55 + 56 56 + for (const line of lines) { 57 57 + const trimmed = line.trim(); 58 58 + if (!trimmed) continue; 59 59 + 60 60 + // Array item 61 61 + if (trimmed.startsWith("- ") && currentKey) { 62 62 + const value = trimmed.slice(2).trim().replace(/^["']|["']$/g, ""); 63 63 + if (currentArray) { 64 64 + currentArray.push(value); 65 65 + } 66 66 + continue; 67 67 + } 68 68 + 69 69 + // Nested key (for author) 70 70 + if (line.startsWith(" ") && inAuthor) { 71 71 + const match = trimmed.match(/^(\w+):\s*(.*)$/); 72 72 + if (match) { 73 73 + const [, key, value] = match; 74 74 + const cleanValue = value?.replace(/^["']|["']$/g, "") || ""; 75 75 + if (key === "name") authorObj.name = cleanValue; 76 76 + if (key === "uri") authorObj.uri = cleanValue; 77 77 + } 78 78 + continue; 79 79 + } 80 80 + 81 81 + // Key-value pair 82 82 + const kvMatch = trimmed.match(/^(\w+):\s*(.*)$/); 83 83 + if (kvMatch) { 84 84 + const [, key, rawValue] = kvMatch; 85 85 + const value = rawValue?.trim(); 86 86 + 87 87 + // Save previous author object 88 88 + if (inAuthor && currentKey === "author") { 89 89 + frontmatter.author = authorObj; 90 90 + authorObj = {}; 91 91 + } 92 92 + inAuthor = false; 93 93 + currentArray = null; 94 94 + 95 95 + if (!value) { 96 96 + // Could be array or nested object 97 97 + if (key === "tags") { 98 98 + currentKey = key; 99 99 + currentArray = []; 100 100 + frontmatter.tags = currentArray; 101 101 + } else if (key === "author") { 102 102 + currentKey = key; 103 103 + inAuthor = true; 104 104 + } 105 105 + } else { 106 106 + // Direct value 107 107 + const cleanValue = value.replace(/^["']|["']$/g, ""); 108 108 + if (key === "title") frontmatter.title = cleanValue; 109 109 + if (key === "date") frontmatter.date = cleanValue; 110 110 + if (key === "description") frontmatter.description = cleanValue; 111 111 + if (key === "view") frontmatter.view = cleanValue; 112 112 + if (key === "tags" && value.startsWith("[")) { 113 113 + // Inline array like tags: ["a", "b"] 114 114 + frontmatter.tags = value 115 115 + .slice(1, -1) 116 116 + .split(",") 117 117 + .map((t) => t.trim().replace(/^["']|["']$/g, "")); 118 118 + } 119 119 + currentKey = key; 120 120 + } 121 121 + } 122 122 + } 123 123 + 124 124 + // Save final author if needed 125 125 + if (inAuthor) { 126 126 + frontmatter.author = authorObj; 127 127 + } 128 128 + 129 129 + return { frontmatter, body: body || "" }; 130 130 + } 131 131 + 132 132 + /** 133 133 + * Generate a record key from a file path 134 134 + * e.g., "notes/2025-03-16_grounding-questions.md" -> "notes-2025-03-16-grounding-questions" 135 135 + */ 136 136 + function generateRkey(relativePath: string): string { 137 137 + const withoutExt = relativePath.replace(/\.(md|html)$/, ""); 138 138 + // Replace path separators and underscores with dashes, remove invalid chars 139 139 + return withoutExt 140 140 + .replace(/[/\\]/g, "-") 141 141 + .replace(/_/g, "-") 142 142 + .replace(/[^a-zA-Z0-9-]/g, "") 143 143 + .toLowerCase() 144 144 + .slice(0, 512); // ATProto rkey max length 145 145 + } 146 146 + 147 147 + /** 148 148 + * Convert file path to URL path, removing date prefix from filename 149 149 + * e.g., "notes/2025-03-16_grounding-questions.md" -> "/notes/grounding-questions" 150 150 + */ 151 151 + function generatePath(relativePath: string): string { 152 152 + const withoutExt = relativePath.replace(/\.(md|html)$/, ""); 153 153 + // Handle index files 154 154 + if (basename(withoutExt) === "index") { 155 155 + const dir = withoutExt.replace(/\/?index$/, ""); 156 156 + return dir ? `/${dir}` : "/"; 157 157 + } 158 158 + // Remove date prefix (YYYY-MM-DD_) from filename 159 159 + const parts = withoutExt.split("/"); 160 160 + const filename = parts[parts.length - 1]; 161 161 + const filenameWithoutDate = filename?.replace(/^\d{4}-\d{2}-\d{2}_/, "") || filename; 162 162 + parts[parts.length - 1] = filenameWithoutDate; 163 163 + return `/${parts.join("/")}`; 164 164 + } 165 165 + 166 166 + /** 167 167 + * Recursively find all content files 168 168 + */ 169 169 + async function findContentFiles(dir: string, baseDir: string = dir): Promise<string[]> { 170 170 + const files: string[] = []; 171 171 + const entries = await readdir(dir, { withFileTypes: true }); 172 172 + 173 173 + for (const entry of entries) { 174 174 + const fullPath = join(dir, entry.name); 175 175 + if (entry.isDirectory()) { 176 176 + const subFiles = await findContentFiles(fullPath, baseDir); 177 177 + files.push(...subFiles); 178 178 + } else if (entry.isFile() && /\.(md|html)$/.test(entry.name)) { 179 179 + files.push(fullPath); 180 180 + } 181 181 + } 182 182 + 183 183 + return files; 184 184 + } 185 185 + 186 186 + /** 187 187 + * Parse a content file into a document 188 188 + */ 189 189 + async function parseContentFile(filePath: string, baseDir: string): Promise<ParsedDocument> { 190 190 + const content = await readFile(filePath, "utf-8"); 191 191 + const relativePath = relative(baseDir, filePath); 192 192 + const { frontmatter, body } = parseFrontmatter(content); 193 193 + 194 194 + return { 195 195 + frontmatter, 196 196 + content: body, 197 197 + filePath, 198 198 + relativePath, 199 199 + }; 200 200 + } 201 201 + 202 202 + /** 203 203 + * Create a standard.site document record 204 204 + */ 205 205 + function createDocumentRecord( 206 206 + doc: ParsedDocument, 207 207 + publicationUri: string, 208 208 + ): { 209 209 + rkey: string; 210 210 + record: Record<string, unknown>; 211 211 + } { 212 212 + const rkey = generateRkey(doc.relativePath); 213 213 + const path = generatePath(doc.relativePath); 214 214 + 215 215 + const record: Record<string, unknown> = { 216 216 + $type: DOCUMENT_COLLECTION, 217 217 + site: publicationUri, 218 218 + title: doc.frontmatter.title || basename(doc.relativePath, extname(doc.relativePath)), 219 219 + path, 220 220 + textContent: doc.content, 221 221 + createdAt: new Date().toISOString(), 222 222 + }; 223 223 + 224 224 + // Add optional fields 225 225 + if (doc.frontmatter.description) { 226 226 + record.description = doc.frontmatter.description; 227 227 + } 228 228 + 229 229 + if (doc.frontmatter.tags && doc.frontmatter.tags.length > 0) { 230 230 + record.tags = doc.frontmatter.tags; 231 231 + } 232 232 + 233 233 + if (doc.frontmatter.date) { 234 234 + // Parse date and convert to ISO string 235 235 + const date = new Date(doc.frontmatter.date); 236 236 + if (!isNaN(date.getTime())) { 237 237 + record.publishedAt = date.toISOString(); 238 238 + } 239 239 + } 240 240 + 241 241 + return { rkey, record }; 242 242 + } 243 243 + 244 244 + async function main() { 245 245 + // Parse arguments 246 246 + const args = process.argv.slice(2); 247 247 + const getArg = (name: string): string | undefined => { 248 248 + const idx = args.indexOf(`--${name}`); 249 249 + return idx !== -1 ? args[idx + 1] : undefined; 250 250 + }; 251 251 + 252 252 + const contentDir = getArg("content-dir"); 253 253 + const publicationUri = getArg("publication"); 254 254 + const identifier = getArg("identifier") || process.env.ATPROTO_IDENTIFIER; 255 255 + const password = getArg("password") || process.env.ATPROTO_PASSWORD; 256 256 + const pdsUrl = getArg("pds") || process.env.PDS_URL || "https://bsky.social"; 257 257 + const dryRun = args.includes("--dry-run"); 258 258 + 259 259 + if (!contentDir || !publicationUri || !identifier || !password) { 260 260 + console.error(`Usage: bun run scripts/import-content.ts \\ 261 261 + --content-dir <path> \\ 262 262 + --publication <at-uri> \\ 263 263 + --identifier <handle-or-did> \\ 264 264 + --password <app-password> \\ 265 265 + [--pds <pds-url>] \\ 266 266 + [--dry-run] 267 267 + 268 268 + Environment variables: 269 269 + ATPROTO_IDENTIFIER - Your handle or DID 270 270 + ATPROTO_PASSWORD - App password 271 271 + PDS_URL - PDS endpoint (default: https://bsky.social) 272 272 + `); 273 273 + process.exit(1); 274 274 + } 275 275 + 276 276 + console.log(`Content directory: ${contentDir}`); 277 277 + console.log(`Publication: ${publicationUri}`); 278 278 + console.log(`PDS: ${pdsUrl}`); 279 279 + console.log(`Dry run: ${dryRun}`); 280 280 + console.log(); 281 281 + 282 282 + // Verify content directory exists 283 283 + try { 284 284 + const stats = await stat(contentDir); 285 285 + if (!stats.isDirectory()) { 286 286 + console.error(`Error: ${contentDir} is not a directory`); 287 287 + process.exit(1); 288 288 + } 289 289 + } catch { 290 290 + console.error(`Error: ${contentDir} does not exist`); 291 291 + process.exit(1); 292 292 + } 293 293 + 294 294 + // Find and parse all content files 295 295 + console.log("Scanning for content files..."); 296 296 + const files = await findContentFiles(contentDir); 297 297 + console.log(`Found ${files.length} files\n`); 298 298 + 299 299 + const documents: ParsedDocument[] = []; 300 300 + for (const file of files) { 301 301 + const doc = await parseContentFile(file, contentDir); 302 302 + documents.push(doc); 303 303 + } 304 304 + 305 305 + // Create ATProto agent and authenticate 306 306 + const agent = new AtpAgent({ service: pdsUrl }); 307 307 + 308 308 + if (!dryRun) { 309 309 + console.log(`Authenticating as ${identifier}...`); 310 310 + await agent.login({ identifier, password }); 311 311 + console.log(`Authenticated as ${agent.session?.did}\n`); 312 312 + } 313 313 + 314 314 + // Process each document 315 315 + let created = 0; 316 316 + let skipped = 0; 317 317 + let failed = 0; 318 318 + 319 319 + for (const doc of documents) { 320 320 + const { rkey, record } = createDocumentRecord(doc, publicationUri); 321 321 + 322 322 + console.log(`Processing: ${doc.relativePath}`); 323 323 + console.log(` Title: ${record.title}`); 324 324 + console.log(` Path: ${record.path}`); 325 325 + console.log(` Rkey: ${rkey}`); 326 326 + if (record.tags) console.log(` Tags: ${(record.tags as string[]).join(", ")}`); 327 327 + if (record.publishedAt) console.log(` Published: ${record.publishedAt}`); 328 328 + 329 329 + if (dryRun) { 330 330 + console.log(` [DRY RUN] Would create record\n`); 331 331 + created++; 332 332 + continue; 333 333 + } 334 334 + 335 335 + try { 336 336 + await agent.api.com.atproto.repo.putRecord({ 337 337 + repo: agent.session!.did, 338 338 + collection: DOCUMENT_COLLECTION, 339 339 + rkey, 340 340 + record, 341 341 + }); 342 342 + console.log(` ✓ Created\n`); 343 343 + created++; 344 344 + } catch (error) { 345 345 + const message = error instanceof Error ? error.message : String(error); 346 346 + console.log(` ✗ Failed: ${message}\n`); 347 347 + failed++; 348 348 + } 349 349 + } 350 350 + 351 351 + console.log("---"); 352 352 + console.log(`Summary:`); 353 353 + console.log(` Created: ${created}`); 354 354 + console.log(` Skipped: ${skipped}`); 355 355 + console.log(` Failed: ${failed}`); 356 356 + } 357 357 + 358 358 + main().catch((error) => { 359 359 + console.error("Fatal error:", error); 360 360 + process.exit(1); 361 361 + });