hosting service writes on cache miss, firehose service properly notifies hosting service on new updates #6

apps/firehose-service/.env.example

This file has not been changed.

apps/firehose-service/package.json

This file has not been changed.

apps/firehose-service/src/index.ts

This file has not been changed.

apps/firehose-service/src/lib/cache-invalidation.ts

This file has not been changed.

+17 -4

apps/firehose-service/src/lib/cache-writer.ts

··· 443 443 options?: { 444 444 forceRewriteHtml?: boolean; 445 445 skipInvalidation?: boolean; 446 446 + forceDownload?: boolean; 446 447 } 447 448 ): Promise<void> { 448 449 const forceRewriteHtml = options?.forceRewriteHtml === true; 450 450 + const forceDownload = options?.forceDownload === true; 449 451 logger.info(`Processing site ${did}/${rkey}`, { 450 452 recordCid, 451 453 forceRewriteHtml, 454 454 + forceDownload, 452 455 }); 453 456 454 457 if (!record.root?.entries) { ··· 504 507 // Find new or changed files 505 508 for (const file of newFiles) { 506 509 const shouldForceRewrite = forceRewriteHtml && isHtmlFile(file.path); 507 507 - if (oldFileCids[file.path] !== file.cid || shouldForceRewrite) { 510 510 + if (forceDownload || oldFileCids[file.path] !== file.cid || shouldForceRewrite) { 508 511 filesToDownload.push(file); 509 512 } 510 513 } ··· 552 555 // Backfill settings if a record exists for this rkey 553 556 const settingsRecord = await fetchSettingsRecord(did, rkey, pdsEndpoint); 554 557 if (settingsRecord) { 555 555 - await handleSettingsUpdate(did, rkey, settingsRecord.record, settingsRecord.cid); 558 558 + await handleSettingsUpdate(did, rkey, settingsRecord.record, settingsRecord.cid, { 559 559 + skipInvalidation: options?.skipInvalidation, 560 560 + }); 556 561 } 557 562 558 563 // Notify hosting-service to invalidate its local caches ··· 590 595 /** 591 596 * Handle settings create/update event 592 597 */ 593 593 - export async function handleSettingsUpdate(did: string, rkey: string, settings: WispSettings, recordCid: string): Promise<void> { 598 598 + export async function handleSettingsUpdate( 599 599 + did: string, 600 600 + rkey: string, 601 601 + settings: WispSettings, 602 602 + recordCid: string, 603 603 + options?: { skipInvalidation?: boolean } 604 604 + ): Promise<void> { 594 605 logger.info(`Updating settings for ${did}/${rkey}`); 595 606 596 607 await upsertSiteSettingsCache(did, rkey, recordCid, { ··· 603 614 }); 604 615 605 616 // Notify hosting-service to invalidate its local caches (redirect rules depend on settings) 606 606 - await publishCacheInvalidation(did, rkey, 'settings'); 617 617 + if (!options?.skipInvalidation) { 618 618 + await publishCacheInvalidation(did, rkey, 'settings'); 619 619 + } 607 620 } 608 621 609 622 /**

apps/firehose-service/src/lib/db.ts

This file has not been changed.

apps/firehose-service/src/lib/firehose.ts

This file has not been changed.

+4

apps/firehose-service/src/lib/revalidate-worker.ts

··· 49 49 return; 50 50 } 51 51 52 52 + // For storage-miss events, force re-download all files since storage is empty 53 53 + const forceDownload = reason.startsWith('storage-miss'); 54 54 + 52 55 await handleSiteCreateOrUpdate(did, rkey, record.record, record.cid, { 53 56 skipInvalidation: true, 57 57 + forceDownload, 54 58 }); 55 59 56 60 logger.info(`[Revalidate] Completed ${id}: ${did}/${rkey}`);

apps/firehose-service/src/lib/storage.ts

This file has not been changed.

apps/hosting-service/.env.example

This file has not been changed.

apps/hosting-service/src/index.ts

This file has not been changed.

apps/hosting-service/src/lib/cache-invalidation.ts

This file has not been changed.

apps/hosting-service/src/lib/db.ts

This file has not been changed.

+1 -5

apps/hosting-service/src/lib/file-serving.ts

··· 32 32 const result = await storage.getWithMetadata(key); 33 33 34 34 if (result) { 35 35 - const metadata = result.metadata; 36 36 - const tier = 37 37 - metadata && typeof metadata === 'object' && 'tier' in metadata 38 38 - ? String((metadata as Record<string, unknown>).tier) 39 39 - : 'unknown'; 35 35 + const tier = result.source || 'unknown'; 40 36 const size = result.data ? (result.data as Uint8Array).length : 0; 41 37 console.log(`[Storage] Served ${filePath} from ${tier} tier (${size} bytes) - ${did}:${rkey}`); 42 38 }

apps/hosting-service/src/lib/on-demand-cache.ts

This file has not been changed.

apps/hosting-service/src/lib/revalidate-queue.ts

This file has not been changed.

apps/hosting-service/src/lib/storage.ts

This file has not been changed.

apps/hosting-service/src/server.ts

This file has not been changed.

bun.lock

This file has not been changed.

docker-compose.yml

This file has not been changed.

+2

apps/firehose-service/Dockerfile

··· 24 24 # Stage 2: Minimal runtime 25 25 FROM alpine:3.22 26 26 27 27 + RUN apk add --no-cache libstdc++ libgcc 28 28 + 27 29 COPY --from=builder /app/firehose /usr/local/bin/firehose 28 30 29 31 ENV NODE_ENV=production