+3 -1

apps/firehose-service/src/lib/cache-writer.ts

··· 554 554 logger.debug(`Updated site cache for ${did}/${rkey} with record CID ${recordCid}`); 555 555 556 556 // Backfill settings if a record exists for this rkey 557 557 + // Always skip settings invalidation here - the 'update' invalidation below 558 558 + // already clears everything including the settings cache on the hosting service 557 559 const settingsRecord = await fetchSettingsRecord(did, rkey, pdsEndpoint); 558 560 if (settingsRecord) { 559 561 await handleSettingsUpdate(did, rkey, settingsRecord.record, settingsRecord.cid, { 560 560 - skipInvalidation: options?.skipInvalidation, 562 562 + skipInvalidation: true, 561 563 }); 562 564 } 563 565

+37 -4

apps/hosting-service/src/lib/cache-invalidation.ts

··· 7 7 */ 8 8 9 9 import Redis from 'ioredis'; 10 10 - import { storage } from './storage'; 10 10 + import type { StorageTier } from '@wispplace/tiered-storage'; 11 11 + import { hotTier, warmTier } from './storage'; 11 12 import { cache } from './cache-manager'; 12 13 13 14 const CHANNEL = 'wisp:cache-invalidate'; 14 15 15 16 let subscriber: Redis | null = null; 17 17 + 18 18 + /** 19 19 + * Directly invalidate a tier by listing and deleting all keys with the given prefix. 20 20 + * Each tier is invalidated independently so a failure in one doesn't block the others. 21 21 + */ 22 22 + async function invalidateTier( 23 23 + tier: StorageTier, 24 24 + tierName: string, 25 25 + prefix: string, 26 26 + ): Promise<number> { 27 27 + try { 28 28 + const keys: string[] = []; 29 29 + for await (const key of tier.listKeys(prefix)) { 30 30 + keys.push(key); 31 31 + } 32 32 + if (keys.length > 0) { 33 33 + await tier.deleteMany(keys); 34 34 + } 35 35 + return keys.length; 36 36 + } catch (err) { 37 37 + console.error(`[CacheInvalidation] Failed to invalidate ${tierName} tier for prefix ${prefix}:`, err); 38 38 + return 0; 39 39 + } 40 40 + } 16 41 17 42 export function startCacheInvalidationSubscriber(): void { 18 43 const redisUrl = process.env.REDIS_URL; ··· 58 83 59 84 console.log(`[CacheInvalidation] Invalidating ${did}/${rkey} (${action})`); 60 85 61 61 - // Clear tiered storage (hot + warm) for this site 62 86 const prefix = `${did}/${rkey}/`; 63 63 - const deleted = await storage.invalidate(prefix); 64 64 - console.log(`[CacheInvalidation] Cleared ${deleted} keys from tiered storage for ${did}/${rkey}`); 87 87 + 88 88 + // Invalidate each tier independently - a failure in one tier 89 89 + // (e.g. S3 listKeys timeout) must NOT prevent hot/warm from being cleared 90 90 + const hotDeleted = await invalidateTier(hotTier, 'hot', prefix); 91 91 + const warmDeleted = warmTier 92 92 + ? await invalidateTier(warmTier, 'warm', prefix) 93 93 + : 0; 94 94 + 95 95 + console.log( 96 96 + `[CacheInvalidation] Cleared ${hotDeleted} hot + ${warmDeleted} warm keys for ${did}/${rkey}`, 97 97 + ); 65 98 66 99 // Clear in-memory caches for this site 67 100 cache.delete('redirectRules', `${did}:${rkey}`);

+120 -7

apps/hosting-service/src/lib/storage.ts

··· 106 106 } 107 107 108 108 async *listKeys(prefix?: string) { 109 109 - yield* this.tier.listKeys(prefix); 109 109 + try { 110 110 + yield* this.tier.listKeys(prefix); 111 111 + } catch (err) { 112 112 + const msg = err instanceof Error ? err.message : String(err); 113 113 + console.warn(`[Storage] S3 listKeys error for prefix ${prefix}: ${msg}`); 114 114 + // Yield nothing on error - don't break invalidation 115 115 + } 110 116 } 111 117 112 118 async getStats() { ··· 152 158 } 153 159 } 154 160 161 161 + // Hot tier TTL (seconds) - safety net so stale entries expire even if invalidation fails 162 162 + const HOT_CACHE_TTL = parseInt(process.env.HOT_CACHE_TTL || '60', 10); // 60s default 163 163 + 164 164 + /** 165 165 + * Wrapper around MemoryStorageTier that enforces a short per-entry TTL. 166 166 + * This acts as a safety net: even if cache invalidation fails to clear the 167 167 + * hot tier, stale entries will expire after HOT_CACHE_TTL seconds. 168 168 + * 169 169 + * The TieredStorage defaultTTL (14 days) is too long for the hot tier - 170 170 + * we want stale hot entries to expire quickly and re-fetch from warm/cold. 171 171 + */ 172 172 + class TTLMemoryTier implements StorageTier { 173 173 + public readonly inner: MemoryStorageTier; 174 174 + private ttlMs: number; 175 175 + private insertedAt = new Map<string, number>(); 176 176 + 177 177 + constructor(config: { maxSizeBytes: number; maxItems?: number }, ttlSeconds: number) { 178 178 + this.inner = new MemoryStorageTier(config); 179 179 + this.ttlMs = ttlSeconds * 1000; 180 180 + } 181 181 + 182 182 + private isStale(key: string): boolean { 183 183 + const ts = this.insertedAt.get(key); 184 184 + if (!ts) return false; 185 185 + return Date.now() - ts > this.ttlMs; 186 186 + } 187 187 + 188 188 + private async evictIfStale(key: string): Promise<boolean> { 189 189 + if (this.isStale(key)) { 190 190 + await this.inner.delete(key); 191 191 + this.insertedAt.delete(key); 192 192 + return true; 193 193 + } 194 194 + return false; 195 195 + } 196 196 + 197 197 + async get(key: string) { 198 198 + if (await this.evictIfStale(key)) return null; 199 199 + return this.inner.get(key); 200 200 + } 201 201 + 202 202 + async getWithMetadata(key: string) { 203 203 + if (await this.evictIfStale(key)) return null; 204 204 + return this.inner.getWithMetadata(key); 205 205 + } 206 206 + 207 207 + async getStream(key: string) { 208 208 + if (await this.evictIfStale(key)) return null; 209 209 + return this.inner.getStream(key); 210 210 + } 211 211 + 212 212 + async set(key: string, data: Uint8Array, metadata: StorageMetadata) { 213 213 + this.insertedAt.set(key, Date.now()); 214 214 + return this.inner.set(key, data, metadata); 215 215 + } 216 216 + 217 217 + async setStream(key: string, stream: NodeJS.ReadableStream, metadata: StorageMetadata) { 218 218 + this.insertedAt.set(key, Date.now()); 219 219 + return this.inner.setStream(key, stream, metadata); 220 220 + } 221 221 + 222 222 + async delete(key: string) { 223 223 + this.insertedAt.delete(key); 224 224 + return this.inner.delete(key); 225 225 + } 226 226 + 227 227 + async deleteMany(keys: string[]) { 228 228 + for (const key of keys) this.insertedAt.delete(key); 229 229 + return this.inner.deleteMany(keys); 230 230 + } 231 231 + 232 232 + async exists(key: string) { 233 233 + if (await this.evictIfStale(key)) return false; 234 234 + return this.inner.exists(key); 235 235 + } 236 236 + 237 237 + async *listKeys(prefix?: string) { 238 238 + yield* this.inner.listKeys(prefix); 239 239 + } 240 240 + 241 241 + async getMetadata(key: string) { 242 242 + if (await this.evictIfStale(key)) return null; 243 243 + return this.inner.getMetadata(key); 244 244 + } 245 245 + 246 246 + async setMetadata(key: string, metadata: StorageMetadata) { 247 247 + return this.inner.setMetadata(key, metadata); 248 248 + } 249 249 + 250 250 + async getStats() { 251 251 + return this.inner.getStats(); 252 252 + } 253 253 + 254 254 + async clear() { 255 255 + this.insertedAt.clear(); 256 256 + return this.inner.clear(); 257 257 + } 258 258 + } 259 259 + 260 260 + // Exported for direct access during cache invalidation 261 261 + export let hotTier: TTLMemoryTier; 262 262 + export let warmTier: StorageTier | undefined; 263 263 + 155 264 /** 156 265 * Initialize tiered storage 157 266 * Must be called before serving requests ··· 159 268 function initializeStorage(): TieredStorage<Uint8Array> { 160 269 // Determine cold tier: S3 if configured, otherwise disk acts as cold 161 270 let coldTier: StorageTier; 162 162 - let warmTier: StorageTier | undefined; 163 271 164 272 const diskTier = new DiskStorageTier({ 165 273 directory: CACHE_DIR, ··· 195 303 console.log('[Storage] S3 not configured - using disk-only mode (disk as cold tier)'); 196 304 } 197 305 306 306 + // Hot tier with short TTL - entries expire quickly so stale data doesn't persist 307 307 + hotTier = new TTLMemoryTier( 308 308 + { maxSizeBytes: HOT_CACHE_SIZE, maxItems: HOT_CACHE_COUNT }, 309 309 + HOT_CACHE_TTL, 310 310 + ); 311 311 + 312 312 + console.log(`[Storage] Hot tier TTL: ${HOT_CACHE_TTL}s`); 313 313 + 198 314 const storage = new TieredStorage<Uint8Array>({ 199 315 tiers: { 200 200 - // Hot tier: In-memory LRU for instant serving 201 201 - hot: new MemoryStorageTier({ 202 202 - maxSizeBytes: HOT_CACHE_SIZE, 203 203 - maxItems: HOT_CACHE_COUNT, 204 204 - }), 316 316 + // Hot tier: In-memory LRU with short TTL for instant serving 317 317 + hot: hotTier, 205 318 206 319 // Warm tier: Disk-based cache (only when S3 is configured) 207 320 warm: warmTier,