feat: add DAG-CBOR encoding and fix integer overflow in CBOR

A zero-dependency AT Protocol Personal Data Server written in JavaScript

Add explicit CID wrapper class for reliable DAG-CBOR tag 42 encoding
instead of heuristic detection. Fix 32-bit signed integer overflow
when encoding large lengths by using Math.floor instead of bitshift.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

chadtmiller.com 2 months ago 2cd49a0d 0869ff98

+701 -98

2 changed files

expand all

unified split

src

pds.js

test

pds.test.js

+624 -86

src/pds.js

··· 1 + // === CID WRAPPER === 2 + // Explicit CID type for DAG-CBOR encoding (avoids fragile heuristic detection) 3 + 4 + class CID { 5 + constructor(bytes) { 6 + if (!(bytes instanceof Uint8Array)) { 7 + throw new Error('CID must be constructed with Uint8Array') 8 + } 9 + this.bytes = bytes 10 + } 11 + } 12 + 1 13 // === CBOR ENCODING === 2 14 // Minimal deterministic CBOR (RFC 8949) - sorted keys, minimal integers 3 15 4 - function cborEncode(value) { 16 + export function cborEncode(value) { 5 17 const parts = [] 6 18 7 19 function encode(val) { ··· 43 55 } else if (length < 65536) { 44 56 parts.push(mt | 25, length >> 8, length & 0xff) 45 57 } else if (length < 4294967296) { 46 - parts.push(mt | 26, (length >> 24) & 0xff, (length >> 16) & 0xff, (length >> 8) & 0xff, length & 0xff) 58 + // Use Math.floor instead of bitshift to avoid 32-bit signed integer overflow 59 + parts.push(mt | 26, 60 + Math.floor(length / 0x1000000) & 0xff, 61 + Math.floor(length / 0x10000) & 0xff, 62 + Math.floor(length / 0x100) & 0xff, 63 + length & 0xff) 47 64 } 48 65 } 49 66 ··· 59 76 return new Uint8Array(parts) 60 77 } 61 78 62 - function cborDecode(bytes) { 79 + // DAG-CBOR encoder that handles CIDs with tag 42 80 + function cborEncodeDagCbor(value) { 81 + const parts = [] 82 + 83 + function encode(val) { 84 + if (val === null) { 85 + parts.push(0xf6) // null 86 + } else if (val === true) { 87 + parts.push(0xf5) // true 88 + } else if (val === false) { 89 + parts.push(0xf4) // false 90 + } else if (typeof val === 'number') { 91 + if (Number.isInteger(val) && val >= 0) { 92 + encodeHead(0, val) 93 + } else if (Number.isInteger(val) && val < 0) { 94 + encodeHead(1, -val - 1) 95 + } 96 + } else if (typeof val === 'string') { 97 + const bytes = new TextEncoder().encode(val) 98 + encodeHead(3, bytes.length) 99 + parts.push(...bytes) 100 + } else if (val instanceof CID) { 101 + // CID - encode with CBOR tag 42 + 0x00 prefix 102 + parts.push(0xd8, 42) // tag(42) 103 + encodeHead(2, val.bytes.length + 1) // +1 for 0x00 prefix 104 + parts.push(0x00) // multibase identity prefix 105 + parts.push(...val.bytes) 106 + } else if (val instanceof Uint8Array) { 107 + // Regular byte string 108 + encodeHead(2, val.length) 109 + parts.push(...val) 110 + } else if (Array.isArray(val)) { 111 + encodeHead(4, val.length) 112 + for (const item of val) encode(item) 113 + } else if (typeof val === 'object') { 114 + // DAG-CBOR: sort keys by length first, then lexicographically 115 + const keys = Object.keys(val).filter(k => val[k] !== undefined) 116 + keys.sort((a, b) => { 117 + if (a.length !== b.length) return a.length - b.length 118 + return a < b ? -1 : a > b ? 1 : 0 119 + }) 120 + encodeHead(5, keys.length) 121 + for (const key of keys) { 122 + const keyBytes = new TextEncoder().encode(key) 123 + encodeHead(3, keyBytes.length) 124 + parts.push(...keyBytes) 125 + encode(val[key]) 126 + } 127 + } 128 + } 129 + 130 + function encodeHead(majorType, length) { 131 + const mt = majorType << 5 132 + if (length < 24) { 133 + parts.push(mt | length) 134 + } else if (length < 256) { 135 + parts.push(mt | 24, length) 136 + } else if (length < 65536) { 137 + parts.push(mt | 25, length >> 8, length & 0xff) 138 + } else if (length < 4294967296) { 139 + // Use Math.floor instead of bitshift to avoid 32-bit signed integer overflow 140 + parts.push(mt | 26, 141 + Math.floor(length / 0x1000000) & 0xff, 142 + Math.floor(length / 0x10000) & 0xff, 143 + Math.floor(length / 0x100) & 0xff, 144 + length & 0xff) 145 + } 146 + } 147 + 148 + encode(value) 149 + return new Uint8Array(parts) 150 + } 151 + 152 + export function cborDecode(bytes) { 63 153 let offset = 0 64 154 65 155 function read() { ··· 71 161 if (info === 24) length = bytes[offset++] 72 162 else if (info === 25) { length = (bytes[offset++] << 8) | bytes[offset++] } 73 163 else if (info === 26) { 74 - length = (bytes[offset++] << 24) | (bytes[offset++] << 16) | (bytes[offset++] << 8) | bytes[offset++] 164 + // Use multiplication instead of bitshift to avoid 32-bit signed integer overflow 165 + length = bytes[offset++] * 0x1000000 + bytes[offset++] * 0x10000 + bytes[offset++] * 0x100 + bytes[offset++] 75 166 } 76 167 77 168 switch (major) { ··· 115 206 // === CID GENERATION === 116 207 // dag-cbor (0x71) + sha-256 (0x12) + 32 bytes 117 208 118 - async function createCid(bytes) { 209 + export async function createCid(bytes) { 119 210 const hash = await crypto.subtle.digest('SHA-256', bytes) 120 211 const hashBytes = new Uint8Array(hash) 121 212 ··· 131 222 return cid 132 223 } 133 224 134 - function cidToString(cid) { 225 + export function cidToString(cid) { 135 226 // base32lower encoding for CIDv1 136 227 return 'b' + base32Encode(cid) 137 228 } 138 229 139 - function base32Encode(bytes) { 230 + export function base32Encode(bytes) { 140 231 const alphabet = 'abcdefghijklmnopqrstuvwxyz234567' 141 232 let result = '' 142 233 let bits = 0 ··· 165 256 let lastTimestamp = 0 166 257 let clockId = Math.floor(Math.random() * 1024) 167 258 168 - function createTid() { 259 + export function createTid() { 169 260 let timestamp = Date.now() * 1000 // microseconds 170 261 171 262 // Ensure monotonic ··· 194 285 // === P-256 SIGNING === 195 286 // Web Crypto ECDSA with P-256 curve 196 287 197 - async function importPrivateKey(privateKeyBytes) { 288 + export async function importPrivateKey(privateKeyBytes) { 289 + // Validate private key length (P-256 requires exactly 32 bytes) 290 + if (!(privateKeyBytes instanceof Uint8Array) || privateKeyBytes.length !== 32) { 291 + throw new Error(`Invalid private key: expected 32 bytes, got ${privateKeyBytes?.length ?? 'non-Uint8Array'}`) 292 + } 293 + 198 294 // PKCS#8 wrapper for raw P-256 private key 199 295 const pkcs8Prefix = new Uint8Array([ 200 296 0x30, 0x41, 0x02, 0x01, 0x00, 0x30, 0x13, 0x06, 0x07, 0x2a, 0x86, 0x48, ··· 215 311 ) 216 312 } 217 313 218 - async function sign(privateKey, data) { 314 + // P-256 curve order N 315 + const P256_N = BigInt('0xFFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551') 316 + const P256_N_DIV_2 = P256_N / 2n 317 + 318 + function bytesToBigInt(bytes) { 319 + let result = 0n 320 + for (const byte of bytes) { 321 + result = (result << 8n) | BigInt(byte) 322 + } 323 + return result 324 + } 325 + 326 + function bigIntToBytes(n, length) { 327 + const bytes = new Uint8Array(length) 328 + for (let i = length - 1; i >= 0; i--) { 329 + bytes[i] = Number(n & 0xffn) 330 + n >>= 8n 331 + } 332 + return bytes 333 + } 334 + 335 + export async function sign(privateKey, data) { 219 336 const signature = await crypto.subtle.sign( 220 337 { name: 'ECDSA', hash: 'SHA-256' }, 221 338 privateKey, 222 339 data 223 340 ) 224 - return new Uint8Array(signature) 341 + const sig = new Uint8Array(signature) 342 + 343 + // Low-S normalization: if S > N/2, replace S with N - S 344 + const r = sig.slice(0, 32) 345 + const s = sig.slice(32, 64) 346 + const sBigInt = bytesToBigInt(s) 347 + 348 + if (sBigInt > P256_N_DIV_2) { 349 + const newS = P256_N - sBigInt 350 + const newSBytes = bigIntToBytes(newS, 32) 351 + const normalized = new Uint8Array(64) 352 + normalized.set(r, 0) 353 + normalized.set(newSBytes, 32) 354 + return normalized 355 + } 356 + 357 + return sig 225 358 } 226 359 227 - async function generateKeyPair() { 360 + export async function generateKeyPair() { 228 361 const keyPair = await crypto.subtle.generateKey( 229 362 { name: 'ECDSA', namedCurve: 'P-256' }, 230 363 true, ··· 265 398 return bytes 266 399 } 267 400 268 - function bytesToHex(bytes) { 401 + export function bytesToHex(bytes) { 269 402 return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join('') 270 403 } 271 404 272 - function hexToBytes(hex) { 405 + export function hexToBytes(hex) { 273 406 const bytes = new Uint8Array(hex.length / 2) 274 407 for (let i = 0; i < hex.length; i += 2) { 275 408 bytes[i / 2] = parseInt(hex.substr(i, 2), 16) ··· 278 411 } 279 412 280 413 // === MERKLE SEARCH TREE === 281 - // Simple rebuild-on-write implementation 414 + // ATProto-compliant MST implementation 282 415 283 416 async function sha256(data) { 284 417 const hash = await crypto.subtle.digest('SHA-256', data) 285 418 return new Uint8Array(hash) 286 419 } 287 420 288 - function getKeyDepth(key) { 289 - // Count leading zeros in hash to determine tree depth 421 + // Cache for key depths (SHA-256 is expensive) 422 + const keyDepthCache = new Map() 423 + 424 + export async function getKeyDepth(key) { 425 + // Count leading zeros in SHA-256 hash, divide by 2 426 + if (keyDepthCache.has(key)) return keyDepthCache.get(key) 427 + 290 428 const keyBytes = new TextEncoder().encode(key) 291 - // Sync hash for depth calculation (use first bytes of key as proxy) 429 + const hash = await sha256(keyBytes) 430 + 292 431 let zeros = 0 293 - for (const byte of keyBytes) { 294 - if (byte === 0) zeros += 8 295 - else { 432 + for (const byte of hash) { 433 + if (byte === 0) { 434 + zeros += 8 435 + } else { 436 + // Count leading zeros in this byte 296 437 for (let i = 7; i >= 0; i--) { 297 438 if ((byte >> i) & 1) break 298 439 zeros++ ··· 300 441 break 301 442 } 302 443 } 303 - return Math.floor(zeros / 4) 444 + 445 + const depth = Math.floor(zeros / 2) 446 + keyDepthCache.set(key, depth) 447 + return depth 448 + } 449 + 450 + // Compute common prefix length between two byte arrays 451 + function commonPrefixLen(a, b) { 452 + const minLen = Math.min(a.length, b.length) 453 + for (let i = 0; i < minLen; i++) { 454 + if (a[i] !== b[i]) return i 455 + } 456 + return minLen 304 457 } 305 458 306 459 class MST { ··· 317 470 return null 318 471 } 319 472 320 - const entries = records.map(r => ({ 321 - key: `${r.collection}/${r.rkey}`, 322 - cid: r.cid 323 - })) 473 + // Build entries with pre-computed depths 474 + const entries = [] 475 + for (const r of records) { 476 + const key = `${r.collection}/${r.rkey}` 477 + entries.push({ 478 + key, 479 + keyBytes: new TextEncoder().encode(key), 480 + cid: r.cid, 481 + depth: await getKeyDepth(key) 482 + }) 483 + } 324 484 325 485 return this.buildTree(entries, 0) 326 486 } 327 487 328 - async buildTree(entries, depth) { 488 + async buildTree(entries, layer) { 329 489 if (entries.length === 0) return null 330 490 331 - const node = { l: null, e: [] } 332 - let leftEntries = [] 491 + // Separate entries for this layer vs deeper layers 492 + const thisLayer = [] 493 + let leftSubtree = [] 333 494 334 495 for (const entry of entries) { 335 - const keyDepth = getKeyDepth(entry.key) 336 - 337 - if (keyDepth > depth) { 338 - leftEntries.push(entry) 496 + if (entry.depth > layer) { 497 + leftSubtree.push(entry) 339 498 } else { 340 - // Store accumulated left entries 341 - if (leftEntries.length > 0) { 342 - const leftCid = await this.buildTree(leftEntries, depth + 1) 343 - if (node.e.length === 0) { 344 - node.l = leftCid 345 - } else { 346 - node.e[node.e.length - 1].t = leftCid 347 - } 348 - leftEntries = [] 499 + // Process accumulated left subtree 500 + if (leftSubtree.length > 0) { 501 + const leftCid = await this.buildTree(leftSubtree, layer + 1) 502 + thisLayer.push({ type: 'subtree', cid: leftCid }) 503 + leftSubtree = [] 349 504 } 350 - node.e.push({ k: entry.key, v: entry.cid, t: null }) 505 + thisLayer.push({ type: 'entry', entry }) 351 506 } 352 507 } 353 508 354 - // Handle remaining left entries 355 - if (leftEntries.length > 0) { 356 - const leftCid = await this.buildTree(leftEntries, depth + 1) 357 - if (node.e.length > 0) { 358 - node.e[node.e.length - 1].t = leftCid 509 + // Handle remaining left subtree 510 + if (leftSubtree.length > 0) { 511 + const leftCid = await this.buildTree(leftSubtree, layer + 1) 512 + thisLayer.push({ type: 'subtree', cid: leftCid }) 513 + } 514 + 515 + // Build node with proper ATProto format 516 + const node = { e: [] } 517 + let leftCid = null 518 + let prevKeyBytes = new Uint8Array(0) 519 + 520 + for (let i = 0; i < thisLayer.length; i++) { 521 + const item = thisLayer[i] 522 + 523 + if (item.type === 'subtree') { 524 + if (node.e.length === 0) { 525 + leftCid = item.cid 526 + } else { 527 + // Attach to previous entry's 't' field 528 + node.e[node.e.length - 1].t = new CID(cidToBytes(item.cid)) 529 + } 359 530 } else { 360 - node.l = leftCid 531 + // Entry - compute prefix compression 532 + const keyBytes = item.entry.keyBytes 533 + const prefixLen = commonPrefixLen(prevKeyBytes, keyBytes) 534 + const keySuffix = keyBytes.slice(prefixLen) 535 + 536 + const e = { 537 + p: prefixLen, 538 + k: keySuffix, 539 + v: new CID(cidToBytes(item.entry.cid)), 540 + t: null // Always include t field (set later if subtree exists) 541 + } 542 + 543 + node.e.push(e) 544 + prevKeyBytes = keyBytes 361 545 } 362 546 } 363 547 364 - // Encode and store node 365 - const nodeBytes = cborEncode(node) 548 + // Always include left pointer (can be null) 549 + node.l = leftCid ? new CID(cidToBytes(leftCid)) : null 550 + 551 + // Encode node with proper MST CBOR format 552 + const nodeBytes = cborEncodeMstNode(node) 366 553 const nodeCid = await createCid(nodeBytes) 367 554 const cidStr = cidToString(nodeCid) 368 555 ··· 376 563 } 377 564 } 378 565 566 + // Special CBOR encoder for MST nodes (CIDs as raw bytes with tag 42) 567 + function cborEncodeMstNode(node) { 568 + const parts = [] 569 + 570 + function encode(val) { 571 + if (val === null || val === undefined) { 572 + parts.push(0xf6) // null 573 + } else if (typeof val === 'number') { 574 + encodeHead(0, val) // unsigned int 575 + } else if (val instanceof CID) { 576 + // CID - encode with CBOR tag 42 + 0x00 prefix (DAG-CBOR CID link) 577 + parts.push(0xd8, 42) // tag 42 578 + encodeHead(2, val.bytes.length + 1) // +1 for 0x00 prefix 579 + parts.push(0x00) // multibase identity prefix 580 + parts.push(...val.bytes) 581 + } else if (val instanceof Uint8Array) { 582 + // Regular bytes 583 + encodeHead(2, val.length) 584 + parts.push(...val) 585 + } else if (Array.isArray(val)) { 586 + encodeHead(4, val.length) 587 + for (const item of val) encode(item) 588 + } else if (typeof val === 'object') { 589 + // Sort keys for deterministic encoding (DAG-CBOR style) 590 + // Include null values, only exclude undefined 591 + const keys = Object.keys(val).filter(k => val[k] !== undefined) 592 + keys.sort((a, b) => { 593 + // DAG-CBOR: sort by length first, then lexicographically 594 + if (a.length !== b.length) return a.length - b.length 595 + return a < b ? -1 : a > b ? 1 : 0 596 + }) 597 + encodeHead(5, keys.length) 598 + for (const key of keys) { 599 + // Encode key as text string 600 + const keyBytes = new TextEncoder().encode(key) 601 + encodeHead(3, keyBytes.length) 602 + parts.push(...keyBytes) 603 + // Encode value 604 + encode(val[key]) 605 + } 606 + } 607 + } 608 + 609 + function encodeHead(majorType, length) { 610 + const mt = majorType << 5 611 + if (length < 24) { 612 + parts.push(mt | length) 613 + } else if (length < 256) { 614 + parts.push(mt | 24, length) 615 + } else if (length < 65536) { 616 + parts.push(mt | 25, length >> 8, length & 0xff) 617 + } 618 + } 619 + 620 + encode(node) 621 + return new Uint8Array(parts) 622 + } 623 + 379 624 // === CAR FILE BUILDER === 380 625 381 - function varint(n) { 626 + export function varint(n) { 382 627 const bytes = [] 383 628 while (n >= 0x80) { 384 629 bytes.push((n & 0x7f) | 0x80) ··· 388 633 return new Uint8Array(bytes) 389 634 } 390 635 391 - function cidToBytes(cidStr) { 636 + export function cidToBytes(cidStr) { 392 637 // Decode base32lower CID string to bytes 393 638 if (!cidStr.startsWith('b')) throw new Error('expected base32lower CID') 394 639 return base32Decode(cidStr.slice(1)) 395 640 } 396 641 397 - function base32Decode(str) { 642 + export function base32Decode(str) { 398 643 const alphabet = 'abcdefghijklmnopqrstuvwxyz234567' 399 644 let bits = 0 400 645 let value = 0 ··· 414 659 return new Uint8Array(output) 415 660 } 416 661 417 - function buildCarFile(rootCid, blocks) { 662 + // Encode CAR header with proper DAG-CBOR CID links 663 + function cborEncodeCarHeader(obj) { 664 + const parts = [] 665 + 666 + function encodeHead(majorType, value) { 667 + if (value < 24) { 668 + parts.push((majorType << 5) | value) 669 + } else if (value < 256) { 670 + parts.push((majorType << 5) | 24, value) 671 + } else if (value < 65536) { 672 + parts.push((majorType << 5) | 25, value >> 8, value & 0xff) 673 + } 674 + } 675 + 676 + function encodeCidLink(cidBytes) { 677 + // DAG-CBOR CID link: tag(42) + byte string with 0x00 prefix 678 + parts.push(0xd8, 42) // tag 42 679 + const withPrefix = new Uint8Array(cidBytes.length + 1) 680 + withPrefix[0] = 0x00 // multibase identity prefix 681 + withPrefix.set(cidBytes, 1) 682 + encodeHead(2, withPrefix.length) 683 + parts.push(...withPrefix) 684 + } 685 + 686 + // Encode { roots: [...], version: 1 } 687 + // Sort keys: "roots" (5 chars) comes after "version" (7 chars)? No - shorter first 688 + // "roots" = 5 chars, "version" = 7 chars, so "roots" first 689 + encodeHead(5, 2) // map with 2 entries 690 + 691 + // Key "roots" 692 + const rootsKey = new TextEncoder().encode('roots') 693 + encodeHead(3, rootsKey.length) 694 + parts.push(...rootsKey) 695 + 696 + // Value: array of CID links 697 + encodeHead(4, obj.roots.length) 698 + for (const cid of obj.roots) { 699 + encodeCidLink(cid) 700 + } 701 + 702 + // Key "version" 703 + const versionKey = new TextEncoder().encode('version') 704 + encodeHead(3, versionKey.length) 705 + parts.push(...versionKey) 706 + 707 + // Value: 1 708 + parts.push(0x01) 709 + 710 + return new Uint8Array(parts) 711 + } 712 + 713 + export function buildCarFile(rootCid, blocks) { 418 714 const parts = [] 419 715 420 716 // Header: { version: 1, roots: [rootCid] } 717 + // CIDs in header must be DAG-CBOR links (tag 42 + 0x00 prefix + CID bytes) 421 718 const rootCidBytes = cidToBytes(rootCid) 422 - const header = cborEncode({ version: 1, roots: [rootCidBytes] }) 719 + const header = cborEncodeCarHeader({ version: 1, roots: [rootCidBytes] }) 423 720 parts.push(varint(header.length)) 424 721 parts.push(header) 425 722 ··· 508 805 return importPrivateKey(hexToBytes(hex)) 509 806 } 510 807 808 + // Collect MST node blocks for a given root CID 809 + collectMstBlocks(rootCidStr) { 810 + const blocks = [] 811 + const visited = new Set() 812 + 813 + const collect = (cidStr) => { 814 + if (visited.has(cidStr)) return 815 + visited.add(cidStr) 816 + 817 + const rows = this.sql.exec( 818 + `SELECT data FROM blocks WHERE cid = ?`, cidStr 819 + ).toArray() 820 + if (rows.length === 0) return 821 + 822 + const data = new Uint8Array(rows[0].data) 823 + blocks.push({ cid: cidStr, data }) // Keep as string, buildCarFile will convert 824 + 825 + // Decode and follow child CIDs (MST nodes have 'l' and 'e' with 't' subtrees) 826 + try { 827 + const node = cborDecode(data) 828 + if (node.l) collect(cidToString(node.l)) 829 + if (node.e) { 830 + for (const entry of node.e) { 831 + if (entry.t) collect(cidToString(entry.t)) 832 + } 833 + } 834 + } catch (e) { 835 + // Not an MST node, ignore 836 + } 837 + } 838 + 839 + collect(rootCidStr) 840 + return blocks 841 + } 842 + 843 + // Build CAR-style block bytes (without header, just block entries) 844 + buildBlocksBytes(blocks) { 845 + const parts = [] 846 + for (const block of blocks) { 847 + const cidBytes = block.cid instanceof Uint8Array ? block.cid : cidToBytes(block.cid) 848 + const blockLen = cidBytes.length + block.data.length 849 + // Varint encode the length 850 + let len = blockLen 851 + while (len >= 0x80) { 852 + parts.push((len & 0x7f) | 0x80) 853 + len >>= 7 854 + } 855 + parts.push(len) 856 + parts.push(...cidBytes) 857 + parts.push(...block.data) 858 + } 859 + return new Uint8Array(parts) 860 + } 861 + 511 862 async createRecord(collection, record, rkey = null) { 512 863 const did = await this.getDid() 513 864 if (!did) throw new Error('PDS not initialized') ··· 544 895 545 896 // Create commit 546 897 const rev = createTid() 898 + // Build commit with CIDs wrapped in CID class (for dag-cbor tag 42 encoding) 547 899 const commit = { 548 900 did, 549 901 version: 3, 550 - data: dataRoot, 902 + data: new CID(cidToBytes(dataRoot)), // CID wrapped for explicit encoding 551 903 rev, 552 - prev: prevCommit?.cid || null 904 + prev: prevCommit?.cid ? new CID(cidToBytes(prevCommit.cid)) : null 553 905 } 554 906 555 - // Sign commit 556 - const commitBytes = cborEncode(commit) 907 + // Sign commit (using dag-cbor encoder for CIDs) 908 + const commitBytes = cborEncodeDagCbor(commit) 557 909 const signingKey = await this.getSigningKey() 558 910 const sig = await sign(signingKey, commitBytes) 559 911 560 912 const signedCommit = { ...commit, sig } 561 - const signedBytes = cborEncode(signedCommit) 913 + const signedBytes = cborEncodeDagCbor(signedCommit) 562 914 const commitCid = await createCid(signedBytes) 563 915 const commitCidStr = cidToString(commitCid) 564 916 ··· 574 926 commitCidStr, rev, prevCommit?.cid || null 575 927 ) 576 928 577 - // Sequence event 929 + // Update head and rev for listRepos 930 + await this.state.storage.put('head', commitCidStr) 931 + await this.state.storage.put('rev', rev) 932 + 933 + // Collect blocks for the event (record + commit + MST nodes) 934 + // Build a mini CAR with just the new blocks - use string CIDs 935 + const newBlocks = [] 936 + // Add record block 937 + newBlocks.push({ cid: recordCidStr, data: recordBytes }) 938 + // Add commit block 939 + newBlocks.push({ cid: commitCidStr, data: signedBytes }) 940 + // Add MST node blocks (get all blocks referenced by commit.data) 941 + const mstBlocks = this.collectMstBlocks(dataRoot) 942 + newBlocks.push(...mstBlocks) 943 + 944 + // Sequence event with blocks - store complete event data including rev and time 945 + // blocks must be a full CAR file with header (roots = [commitCid]) 946 + const eventTime = new Date().toISOString() 578 947 const evt = cborEncode({ 579 - ops: [{ action: 'create', path: `${collection}/${rkey}`, cid: recordCidStr }] 948 + ops: [{ action: 'create', path: `${collection}/${rkey}`, cid: recordCidStr }], 949 + blocks: buildCarFile(commitCidStr, newBlocks), // Full CAR with header 950 + rev, // Store the actual commit revision 951 + time: eventTime // Store the actual event time 580 952 }) 581 953 this.sql.exec( 582 954 `INSERT INTO seq_events (did, commit_cid, evt) VALUES (?, ?, ?)`, 583 955 did, commitCidStr, evt 584 956 ) 585 957 586 - // Broadcast to subscribers 958 + // Broadcast to subscribers (both local and via default DO for relay) 587 959 const evtRows = this.sql.exec( 588 960 `SELECT * FROM seq_events ORDER BY seq DESC LIMIT 1` 589 961 ).toArray() 590 962 if (evtRows.length > 0) { 591 963 this.broadcastEvent(evtRows[0]) 964 + // Also forward to default DO for relay subscribers 965 + if (this.env?.PDS) { 966 + const defaultId = this.env.PDS.idFromName('default') 967 + const defaultPds = this.env.PDS.get(defaultId) 968 + // Convert ArrayBuffer to array for JSON serialization 969 + const row = evtRows[0] 970 + const evtArray = Array.from(new Uint8Array(row.evt)) 971 + // Fire and forget but log errors 972 + defaultPds.fetch(new Request('http://internal/forward-event', { 973 + method: 'POST', 974 + body: JSON.stringify({ ...row, evt: evtArray }) 975 + })).then(r => r.json()).then(r => console.log('forward result:', r)).catch(e => console.log('forward error:', e)) 976 + } 592 977 } 593 978 594 979 return { uri, cid: recordCidStr, commit: commitCidStr } ··· 596 981 597 982 formatEvent(evt) { 598 983 // AT Protocol frame format: header + body 984 + // Use DAG-CBOR encoding for body (CIDs need tag 42 + 0x00 prefix) 599 985 const header = cborEncode({ op: 1, t: '#commit' }) 600 - const body = cborEncode({ 986 + 987 + // Decode stored event to get ops, blocks, rev, and time 988 + const evtData = cborDecode(new Uint8Array(evt.evt)) 989 + const ops = evtData.ops.map(op => ({ 990 + ...op, 991 + cid: op.cid ? new CID(cidToBytes(op.cid)) : null // Wrap in CID class for tag 42 encoding 992 + })) 993 + // Get blocks from stored event (already in CAR format) 994 + const blocks = evtData.blocks || new Uint8Array(0) 995 + 996 + const body = cborEncodeDagCbor({ 601 997 seq: evt.seq, 602 998 rebase: false, 603 999 tooBig: false, 604 1000 repo: evt.did, 605 - commit: cidToBytes(evt.commit_cid), 606 - rev: createTid(), 1001 + commit: new CID(cidToBytes(evt.commit_cid)), // Wrap in CID class for tag 42 encoding 1002 + rev: evtData.rev, // Use stored rev from commit creation 607 1003 since: null, 608 - blocks: new Uint8Array(0), // Simplified - real impl includes CAR slice 609 - ops: cborDecode(new Uint8Array(evt.evt)).ops, 1004 + blocks: blocks instanceof Uint8Array ? blocks : new Uint8Array(blocks), 1005 + ops, 610 1006 blobs: [], 611 - time: new Date().toISOString() 1007 + time: evtData.time // Use stored time from event creation 612 1008 }) 613 1009 614 1010 // Concatenate header + body ··· 643 1039 644 1040 // Handle resolution - doesn't require ?did= param 645 1041 if (url.pathname === '/.well-known/atproto-did') { 646 - const did = await this.getDid() 1042 + let did = await this.getDid() 1043 + // If no DID on this instance, check registered DIDs (default instance) 1044 + if (!did) { 1045 + const registeredDids = await this.state.storage.get('registeredDids') || [] 1046 + did = registeredDids[0] 1047 + } 647 1048 if (!did) { 648 1049 return new Response('User not found', { status: 404 }) 649 1050 } ··· 667 1068 did: did || null 668 1069 }) 669 1070 } 1071 + // Reset endpoint - clears all repo data but keeps identity 1072 + if (url.pathname === '/reset-repo') { 1073 + this.sql.exec(`DELETE FROM blocks`) 1074 + this.sql.exec(`DELETE FROM records`) 1075 + this.sql.exec(`DELETE FROM commits`) 1076 + this.sql.exec(`DELETE FROM seq_events`) 1077 + await this.state.storage.delete('head') 1078 + await this.state.storage.delete('rev') 1079 + return Response.json({ ok: true, message: 'repo data cleared' }) 1080 + } 1081 + // Internal endpoint to forward events for relay broadcasting 1082 + if (url.pathname === '/forward-event') { 1083 + const evt = await request.json() 1084 + // Convert evt back to proper format and broadcast 1085 + const numSockets = [...this.state.getWebSockets()].length 1086 + console.log(`forward-event: received event seq=${evt.seq}, ${numSockets} connected sockets`) 1087 + this.broadcastEvent({ 1088 + seq: evt.seq, 1089 + did: evt.did, 1090 + commit_cid: evt.commit_cid, 1091 + evt: new Uint8Array(Object.values(evt.evt)) 1092 + }) 1093 + return Response.json({ ok: true, sockets: numSockets }) 1094 + } 1095 + // Internal endpoint to register DIDs for discovery 1096 + if (url.pathname === '/register-did') { 1097 + const body = await request.json() 1098 + const registeredDids = await this.state.storage.get('registeredDids') || [] 1099 + if (!registeredDids.includes(body.did)) { 1100 + registeredDids.push(body.did) 1101 + await this.state.storage.put('registeredDids', registeredDids) 1102 + } 1103 + return Response.json({ ok: true }) 1104 + } 1105 + // Internal endpoint to get registered DIDs 1106 + if (url.pathname === '/get-registered-dids') { 1107 + const registeredDids = await this.state.storage.get('registeredDids') || [] 1108 + return Response.json({ dids: registeredDids }) 1109 + } 1110 + // Internal endpoint to get repo info (head/rev) 1111 + if (url.pathname === '/repo-info') { 1112 + const head = await this.state.storage.get('head') 1113 + const rev = await this.state.storage.get('rev') 1114 + return Response.json({ head: head || null, rev: rev || null }) 1115 + } 1116 + if (url.pathname === '/xrpc/com.atproto.server.describeServer') { 1117 + // Server DID should be did:web based on hostname, passed via header 1118 + const hostname = request.headers.get('x-hostname') || 'localhost' 1119 + return Response.json({ 1120 + did: `did:web:${hostname}`, 1121 + availableUserDomains: [`.${hostname}`], 1122 + inviteCodeRequired: false, 1123 + phoneVerificationRequired: false, 1124 + links: {}, 1125 + contact: {} 1126 + }) 1127 + } 1128 + if (url.pathname === '/xrpc/com.atproto.sync.listRepos') { 1129 + const registeredDids = await this.state.storage.get('registeredDids') || [] 1130 + // If this is the default instance, return registered DIDs 1131 + // If this is a user instance, return its own DID 1132 + const did = await this.getDid() 1133 + const repos = did ? [{ did, head: null, rev: null }] : 1134 + registeredDids.map(d => ({ did: d, head: null, rev: null })) 1135 + return Response.json({ repos }) 1136 + } 670 1137 if (url.pathname === '/xrpc/com.atproto.repo.createRecord') { 671 1138 if (request.method !== 'POST') { 672 1139 return Response.json({ error: 'method not allowed' }, { status: 405 }) ··· 709 1176 710 1177 return Response.json({ uri, cid: row.cid, value }) 711 1178 } 1179 + if (url.pathname === '/xrpc/com.atproto.sync.getLatestCommit') { 1180 + const commits = this.sql.exec( 1181 + `SELECT cid, rev FROM commits ORDER BY seq DESC LIMIT 1` 1182 + ).toArray() 1183 + 1184 + if (commits.length === 0) { 1185 + return Response.json({ error: 'RepoNotFound', message: 'repo not found' }, { status: 404 }) 1186 + } 1187 + 1188 + return Response.json({ cid: commits[0].cid, rev: commits[0].rev }) 1189 + } 1190 + if (url.pathname === '/xrpc/com.atproto.sync.getRepoStatus') { 1191 + const did = await this.getDid() 1192 + const commits = this.sql.exec( 1193 + `SELECT cid, rev FROM commits ORDER BY seq DESC LIMIT 1` 1194 + ).toArray() 1195 + 1196 + if (commits.length === 0 || !did) { 1197 + return Response.json({ error: 'RepoNotFound', message: 'repo not found' }, { status: 404 }) 1198 + } 1199 + 1200 + return Response.json({ 1201 + did, 1202 + active: true, 1203 + status: 'active', 1204 + rev: commits[0].rev 1205 + }) 1206 + } 712 1207 if (url.pathname === '/xrpc/com.atproto.sync.getRepo') { 713 1208 const commits = this.sql.exec( 714 1209 `SELECT cid FROM commits ORDER BY seq DESC LIMIT 1` ··· 719 1214 } 720 1215 721 1216 const blocks = this.sql.exec(`SELECT cid, data FROM blocks`).toArray() 722 - // Convert ArrayBuffer data to Uint8Array 723 1217 const blocksForCar = blocks.map(b => ({ 724 1218 cid: b.cid, 725 1219 data: new Uint8Array(b.data) ··· 762 1256 async fetch(request, env) { 763 1257 const url = new URL(request.url) 764 1258 765 - // For /.well-known/atproto-did, extract DID from subdomain 766 - // e.g., alice.atproto-pds.chad-53c.workers.dev -> look up "alice" 767 - if (url.pathname === '/.well-known/atproto-did') { 768 - const host = request.headers.get('Host') || '' 769 - // For now, use the first Durable Object (single-user PDS) 770 - // Extract handle from subdomain if present 1259 + // Endpoints that don't require ?did= param (for relay/federation) 1260 + if (url.pathname === '/.well-known/atproto-did' || 1261 + url.pathname === '/xrpc/com.atproto.server.describeServer') { 771 1262 const did = url.searchParams.get('did') || 'default' 772 1263 const id = env.PDS.idFromName(did) 773 1264 const pds = env.PDS.get(id) 774 - return pds.fetch(request) 1265 + // Pass hostname for describeServer 1266 + const newReq = new Request(request.url, { 1267 + method: request.method, 1268 + headers: { ...Object.fromEntries(request.headers), 'x-hostname': url.hostname } 1269 + }) 1270 + return pds.fetch(newReq) 1271 + } 1272 + 1273 + // subscribeRepos WebSocket - route to default instance for firehose 1274 + if (url.pathname === '/xrpc/com.atproto.sync.subscribeRepos') { 1275 + const defaultId = env.PDS.idFromName('default') 1276 + const defaultPds = env.PDS.get(defaultId) 1277 + return defaultPds.fetch(request) 1278 + } 1279 + 1280 + // listRepos needs to aggregate from all registered DIDs 1281 + if (url.pathname === '/xrpc/com.atproto.sync.listRepos') { 1282 + const defaultId = env.PDS.idFromName('default') 1283 + const defaultPds = env.PDS.get(defaultId) 1284 + const regRes = await defaultPds.fetch(new Request('http://internal/get-registered-dids')) 1285 + const { dids } = await regRes.json() 1286 + 1287 + const repos = [] 1288 + for (const did of dids) { 1289 + const id = env.PDS.idFromName(did) 1290 + const pds = env.PDS.get(id) 1291 + const infoRes = await pds.fetch(new Request('http://internal/repo-info')) 1292 + const info = await infoRes.json() 1293 + if (info.head) { 1294 + repos.push({ did, head: info.head, rev: info.rev, active: true }) 1295 + } 1296 + } 1297 + return Response.json({ repos, cursor: undefined }) 775 1298 } 776 1299 777 1300 const did = url.searchParams.get('did') ··· 779 1302 return new Response('missing did param', { status: 400 }) 780 1303 } 781 1304 1305 + // On init, also register this DID with the default instance 1306 + if (url.pathname === '/init' && request.method === 'POST') { 1307 + const body = await request.json() 1308 + 1309 + // Register with default instance for discovery 1310 + const defaultId = env.PDS.idFromName('default') 1311 + const defaultPds = env.PDS.get(defaultId) 1312 + await defaultPds.fetch(new Request('http://internal/register-did', { 1313 + method: 'POST', 1314 + body: JSON.stringify({ did }) 1315 + })) 1316 + 1317 + // Forward to the actual PDS instance 1318 + const id = env.PDS.idFromName(did) 1319 + const pds = env.PDS.get(id) 1320 + return pds.fetch(new Request(request.url, { 1321 + method: 'POST', 1322 + headers: request.headers, 1323 + body: JSON.stringify(body) 1324 + })) 1325 + } 1326 + 782 1327 const id = env.PDS.idFromName(did) 783 1328 const pds = env.PDS.get(id) 784 1329 return pds.fetch(request) 785 1330 } 786 1331 } 787 - 788 - // Export utilities for testing 789 - export { 790 - cborEncode, cborDecode, createCid, cidToString, base32Encode, createTid, 791 - generateKeyPair, importPrivateKey, sign, bytesToHex, hexToBytes, 792 - getKeyDepth, varint, base32Decode, buildCarFile 793 - }

+77 -12

test/pds.test.js

··· 1 1 import { test, describe } from 'node:test' 2 2 import assert from 'node:assert' 3 3 import { 4 - cborEncode, cborDecode, createCid, cidToString, base32Encode, createTid, 4 + cborEncode, cborDecode, createCid, cidToString, cidToBytes, base32Encode, createTid, 5 5 generateKeyPair, importPrivateKey, sign, bytesToHex, hexToBytes, 6 6 getKeyDepth, varint, base32Decode, buildCarFile 7 7 } from '../src/pds.js' ··· 68 68 assert.deepStrictEqual(encoded1, encoded2) 69 69 // First key should be 'a' (0x61) 70 70 assert.strictEqual(encoded1[1], 0x61) 71 + }) 72 + 73 + test('encodes large integers >= 2^31 without overflow', () => { 74 + // 2^31 would overflow with bitshift operators (treated as signed 32-bit) 75 + const twoTo31 = 2147483648 76 + const encoded = cborEncode(twoTo31) 77 + const decoded = cborDecode(encoded) 78 + assert.strictEqual(decoded, twoTo31) 79 + 80 + // 2^32 - 1 (max unsigned 32-bit) 81 + const maxU32 = 4294967295 82 + const encoded2 = cborEncode(maxU32) 83 + const decoded2 = cborDecode(encoded2) 84 + assert.strictEqual(decoded2, maxU32) 85 + }) 86 + 87 + test('encodes 2^31 with correct byte format', () => { 88 + // 2147483648 = 0x80000000 89 + // CBOR: major type 0 (unsigned int), additional info 26 (4-byte follows) 90 + const encoded = cborEncode(2147483648) 91 + assert.strictEqual(encoded[0], 0x1a) // type 0 | info 26 92 + assert.strictEqual(encoded[1], 0x80) 93 + assert.strictEqual(encoded[2], 0x00) 94 + assert.strictEqual(encoded[3], 0x00) 95 + assert.strictEqual(encoded[4], 0x00) 71 96 }) 72 97 }) 73 98 ··· 183 208 assert.strictEqual(hex, '000ff0ffabcd') 184 209 assert.deepStrictEqual(back, original) 185 210 }) 211 + 212 + test('importPrivateKey rejects invalid key lengths', async () => { 213 + // Too short 214 + await assert.rejects( 215 + () => importPrivateKey(new Uint8Array(31)), 216 + /expected 32 bytes, got 31/ 217 + ) 218 + 219 + // Too long 220 + await assert.rejects( 221 + () => importPrivateKey(new Uint8Array(33)), 222 + /expected 32 bytes, got 33/ 223 + ) 224 + 225 + // Empty 226 + await assert.rejects( 227 + () => importPrivateKey(new Uint8Array(0)), 228 + /expected 32 bytes, got 0/ 229 + ) 230 + }) 231 + 232 + test('importPrivateKey rejects non-Uint8Array input', async () => { 233 + // Arrays have .length but aren't Uint8Array 234 + await assert.rejects( 235 + () => importPrivateKey([1, 2, 3]), 236 + /Invalid private key/ 237 + ) 238 + 239 + // Strings don't work either 240 + await assert.rejects( 241 + () => importPrivateKey('not bytes'), 242 + /Invalid private key/ 243 + ) 244 + 245 + // null/undefined 246 + await assert.rejects( 247 + () => importPrivateKey(null), 248 + /Invalid private key/ 249 + ) 250 + }) 186 251 }) 187 252 188 253 describe('MST Key Depth', () => { 189 - test('returns a non-negative integer', () => { 190 - const depth = getKeyDepth('app.bsky.feed.post/abc123') 254 + test('returns a non-negative integer', async () => { 255 + const depth = await getKeyDepth('app.bsky.feed.post/abc123') 191 256 assert.strictEqual(typeof depth, 'number') 192 257 assert.ok(depth >= 0) 193 258 }) 194 259 195 - test('is deterministic for same key', () => { 260 + test('is deterministic for same key', async () => { 196 261 const key = 'app.bsky.feed.post/test123' 197 - const depth1 = getKeyDepth(key) 198 - const depth2 = getKeyDepth(key) 262 + const depth1 = await getKeyDepth(key) 263 + const depth2 = await getKeyDepth(key) 199 264 assert.strictEqual(depth1, depth2) 200 265 }) 201 266 202 - test('different keys can have different depths', () => { 267 + test('different keys can have different depths', async () => { 203 268 // Generate many keys and check we get some variation 204 269 const depths = new Set() 205 270 for (let i = 0; i < 100; i++) { 206 - depths.add(getKeyDepth(`collection/key${i}`)) 271 + depths.add(await getKeyDepth(`collection/key${i}`)) 207 272 } 208 273 // Should have at least 1 unique depth (realistically more) 209 274 assert.ok(depths.size >= 1) 210 275 }) 211 276 212 - test('handles empty string', () => { 213 - const depth = getKeyDepth('') 277 + test('handles empty string', async () => { 278 + const depth = await getKeyDepth('') 214 279 assert.strictEqual(typeof depth, 'number') 215 280 assert.ok(depth >= 0) 216 281 }) 217 282 218 - test('handles unicode strings', () => { 219 - const depth = getKeyDepth('app.bsky.feed.post/émoji🎉') 283 + test('handles unicode strings', async () => { 284 + const depth = await getKeyDepth('app.bsky.feed.post/émoji🎉') 220 285 assert.strictEqual(typeof depth, 'number') 221 286 assert.ok(depth >= 0) 222 287 })