hosting service writes on cache miss, firehose service properly notifies hosting service on new updates #6

apps/firehose-service/.env.example

This file has not been changed.

apps/firehose-service/package.json

This file has not been changed.

apps/firehose-service/src/index.ts

This file has not been changed.

apps/firehose-service/src/lib/cache-invalidation.ts

This file has not been changed.

+17 -4

apps/firehose-service/src/lib/cache-writer.ts

··· 443 options?: { 444 forceRewriteHtml?: boolean; 445 skipInvalidation?: boolean; 0 446 } 447 ): Promise<void> { 448 const forceRewriteHtml = options?.forceRewriteHtml === true; 0 449 logger.info(`Processing site ${did}/${rkey}`, { 450 recordCid, 451 forceRewriteHtml, 0 452 }); 453 454 if (!record.root?.entries) { ··· 504 // Find new or changed files 505 for (const file of newFiles) { 506 const shouldForceRewrite = forceRewriteHtml && isHtmlFile(file.path); 507 - if (oldFileCids[file.path] !== file.cid || shouldForceRewrite) { 508 filesToDownload.push(file); 509 } 510 } ··· 552 // Backfill settings if a record exists for this rkey 553 const settingsRecord = await fetchSettingsRecord(did, rkey, pdsEndpoint); 554 if (settingsRecord) { 555 - await handleSettingsUpdate(did, rkey, settingsRecord.record, settingsRecord.cid); 0 0 556 } 557 558 // Notify hosting-service to invalidate its local caches ··· 590 /** 591 * Handle settings create/update event 592 */ 593 - export async function handleSettingsUpdate(did: string, rkey: string, settings: WispSettings, recordCid: string): Promise<void> { 0 0 0 0 0 0 594 logger.info(`Updating settings for ${did}/${rkey}`); 595 596 await upsertSiteSettingsCache(did, rkey, recordCid, { ··· 603 }); 604 605 // Notify hosting-service to invalidate its local caches (redirect rules depend on settings) 606 - await publishCacheInvalidation(did, rkey, 'settings'); 0 0 607 } 608 609 /**

··· 443 options?: { 444 forceRewriteHtml?: boolean; 445 skipInvalidation?: boolean; 446 + forceDownload?: boolean; 447 } 448 ): Promise<void> { 449 const forceRewriteHtml = options?.forceRewriteHtml === true; 450 + const forceDownload = options?.forceDownload === true; 451 logger.info(`Processing site ${did}/${rkey}`, { 452 recordCid, 453 forceRewriteHtml, 454 + forceDownload, 455 }); 456 457 if (!record.root?.entries) { ··· 507 // Find new or changed files 508 for (const file of newFiles) { 509 const shouldForceRewrite = forceRewriteHtml && isHtmlFile(file.path); 510 + if (forceDownload || oldFileCids[file.path] !== file.cid || shouldForceRewrite) { 511 filesToDownload.push(file); 512 } 513 } ··· 555 // Backfill settings if a record exists for this rkey 556 const settingsRecord = await fetchSettingsRecord(did, rkey, pdsEndpoint); 557 if (settingsRecord) { 558 + await handleSettingsUpdate(did, rkey, settingsRecord.record, settingsRecord.cid, { 559 + skipInvalidation: options?.skipInvalidation, 560 + }); 561 } 562 563 // Notify hosting-service to invalidate its local caches ··· 595 /** 596 * Handle settings create/update event 597 */ 598 + export async function handleSettingsUpdate( 599 + did: string, 600 + rkey: string, 601 + settings: WispSettings, 602 + recordCid: string, 603 + options?: { skipInvalidation?: boolean } 604 + ): Promise<void> { 605 logger.info(`Updating settings for ${did}/${rkey}`); 606 607 await upsertSiteSettingsCache(did, rkey, recordCid, { ··· 614 }); 615 616 // Notify hosting-service to invalidate its local caches (redirect rules depend on settings) 617 + if (!options?.skipInvalidation) { 618 + await publishCacheInvalidation(did, rkey, 'settings'); 619 + } 620 } 621 622 /**

apps/firehose-service/src/lib/db.ts

This file has not been changed.

apps/firehose-service/src/lib/firehose.ts

This file has not been changed.

+4

apps/firehose-service/src/lib/revalidate-worker.ts

··· 49 return; 50 } 51 0 0 0 52 await handleSiteCreateOrUpdate(did, rkey, record.record, record.cid, { 53 skipInvalidation: true, 0 54 }); 55 56 logger.info(`[Revalidate] Completed ${id}: ${did}/${rkey}`);

··· 49 return; 50 } 51 52 + // For storage-miss events, force re-download all files since storage is empty 53 + const forceDownload = reason.startsWith('storage-miss'); 54 + 55 await handleSiteCreateOrUpdate(did, rkey, record.record, record.cid, { 56 skipInvalidation: true, 57 + forceDownload, 58 }); 59 60 logger.info(`[Revalidate] Completed ${id}: ${did}/${rkey}`);

apps/firehose-service/src/lib/storage.ts

This file has not been changed.

apps/hosting-service/.env.example

This file has not been changed.

apps/hosting-service/src/index.ts

This file has not been changed.

apps/hosting-service/src/lib/cache-invalidation.ts

This file has not been changed.

apps/hosting-service/src/lib/db.ts

This file has not been changed.

+1 -5

apps/hosting-service/src/lib/file-serving.ts

··· 32 const result = await storage.getWithMetadata(key); 33 34 if (result) { 35 - const metadata = result.metadata; 36 - const tier = 37 - metadata && typeof metadata === 'object' && 'tier' in metadata 38 - ? String((metadata as Record<string, unknown>).tier) 39 - : 'unknown'; 40 const size = result.data ? (result.data as Uint8Array).length : 0; 41 console.log(`[Storage] Served ${filePath} from ${tier} tier (${size} bytes) - ${did}:${rkey}`); 42 }

··· 32 const result = await storage.getWithMetadata(key); 33 34 if (result) { 35 + const tier = result.source || 'unknown'; 0 0 0 0 36 const size = result.data ? (result.data as Uint8Array).length : 0; 37 console.log(`[Storage] Served ${filePath} from ${tier} tier (${size} bytes) - ${did}:${rkey}`); 38 }

apps/hosting-service/src/lib/on-demand-cache.ts

This file has not been changed.

apps/hosting-service/src/lib/revalidate-queue.ts

This file has not been changed.

apps/hosting-service/src/lib/storage.ts

This file has not been changed.

apps/hosting-service/src/server.ts

This file has not been changed.

bun.lock

This file has not been changed.

docker-compose.yml

This file has not been changed.

+2

apps/firehose-service/Dockerfile

··· 24 # Stage 2: Minimal runtime 25 FROM alpine:3.22 26 0 0 27 COPY --from=builder /app/firehose /usr/local/bin/firehose 28 29 ENV NODE_ENV=production

··· 24 # Stage 2: Minimal runtime 25 FROM alpine:3.22 26 27 + RUN apk add --no-cache libstdc++ libgcc 28 + 29 COPY --from=builder /app/firehose /usr/local/bin/firehose 30 31 ENV NODE_ENV=production