notjack.space
forked from
nekomimi.pet/wisp.place-monorepo
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
add flat: boolean to subfs definition in place.wisp.fs to define if to merge flat or as a subdirectory, handle this logic better across backend and hosting service
nekomimi.pet
+8
-3
hosting-service/src/lexicon/lexicons.ts
+8
-3
hosting-service/src/lexicon/lexicons.ts
···
118
118
type: 'string',
119
119
format: 'at-uri',
120
120
description:
121
-
'AT-URI pointing to a place.wisp.subfs record containing this subtree',
121
+
'AT-URI pointing to a place.wisp.subfs record containing this subtree.',
122
+
},
123
+
flat: {
124
+
type: 'boolean',
125
+
description:
126
+
"If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure.",
122
127
},
123
128
},
124
129
},
···
131
136
main: {
132
137
type: 'record',
133
138
description:
134
-
'Virtual filesystem manifest within a place.wisp.fs record',
139
+
'Virtual filesystem subtree referenced by place.wisp.fs records. When a subfs entry is expanded, its root entries are merged (flattened) into the parent directory, allowing large directories to be split across multiple records while maintaining a flat structure.',
135
140
record: {
136
141
type: 'object',
137
142
required: ['root', 'createdAt'],
···
230
235
type: 'string',
231
236
format: 'at-uri',
232
237
description:
233
-
'AT-URI pointing to another place.wisp.subfs record for nested subtrees',
238
+
"AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures.",
234
239
},
235
240
},
236
241
},
+3
-1
hosting-service/src/lexicon/types/place/wisp/fs.ts
+3
-1
hosting-service/src/lexicon/types/place/wisp/fs.ts
···
93
93
export interface Subfs {
94
94
$type?: 'place.wisp.fs#subfs'
95
95
type: 'subfs'
96
-
/** AT-URI pointing to a place.wisp.subfs record containing this subtree */
96
+
/** AT-URI pointing to a place.wisp.subfs record containing this subtree. */
97
97
subject: string
98
+
/** If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure. */
99
+
flat?: boolean
98
100
}
99
101
100
102
const hashSubfs = 'subfs'
+1
-1
hosting-service/src/lexicon/types/place/wisp/subfs.ts
+1
-1
hosting-service/src/lexicon/types/place/wisp/subfs.ts
···
92
92
export interface Subfs {
93
93
$type?: 'place.wisp.subfs#subfs'
94
94
type: 'subfs'
95
-
/** AT-URI pointing to another place.wisp.subfs record for nested subtrees */
95
+
/** AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures. */
96
96
subject: string
97
97
}
98
98
+1
hosting-service/src/lib/firehose.ts
+1
hosting-service/src/lib/firehose.ts
+43
-7
hosting-service/src/lib/utils.ts
+43
-7
hosting-service/src/lib/utils.ts
···
295
295
const node = entry.node;
296
296
297
297
if ('type' in node && node.type === 'subfs') {
298
-
// Merge subfs entries into parent directory
298
+
// Check if this is a flat merge or subdirectory merge (default to flat if not specified)
299
+
const subfsNode = node as any;
300
+
const isFlat = subfsNode.flat !== false; // Default to true
299
301
const subfsEntries = subfsMap.get(fullPath);
302
+
300
303
if (subfsEntries) {
301
-
console.log(`Merging subfs node at ${fullPath} (${subfsEntries.length} entries)`);
302
-
// Recursively process the merged entries in case they contain nested subfs
303
-
const processedEntries = replaceSubfsInEntries(subfsEntries, currentPath);
304
-
result.push(...processedEntries);
304
+
console.log(`Merging subfs node at ${fullPath} (${subfsEntries.length} entries, flat: ${isFlat})`);
305
+
306
+
if (isFlat) {
307
+
// Flat merge: hoist entries directly into parent directory
308
+
const processedEntries = replaceSubfsInEntries(subfsEntries, currentPath);
309
+
result.push(...processedEntries);
310
+
} else {
311
+
// Subdirectory merge: create a directory with the subfs node's name
312
+
const processedEntries = replaceSubfsInEntries(subfsEntries, fullPath);
313
+
result.push({
314
+
name: entry.name,
315
+
node: {
316
+
type: 'directory',
317
+
entries: processedEntries
318
+
}
319
+
});
320
+
}
305
321
} else {
306
322
// If fetch failed, skip this entry
307
323
console.warn(`Failed to fetch subfs at ${fullPath}, skipping`);
···
491
507
492
508
// Download new/changed files concurrently - increased from 3 to 20 for much better performance
493
509
const downloadLimit = 20;
510
+
let successCount = 0;
511
+
let failureCount = 0;
512
+
494
513
for (let i = 0; i < downloadTasks.length; i += downloadLimit) {
495
514
const batch = downloadTasks.slice(i, i + downloadLimit);
496
-
await Promise.all(batch.map(task => task()));
515
+
const results = await Promise.allSettled(batch.map(task => task()));
516
+
517
+
// Count successes and failures
518
+
results.forEach((result, index) => {
519
+
if (result.status === 'fulfilled') {
520
+
successCount++;
521
+
} else {
522
+
failureCount++;
523
+
console.error(`[Cache] Failed to download file (continuing with others):`, result.reason);
524
+
}
525
+
});
526
+
497
527
if (downloadTasks.length > downloadLimit) {
498
-
console.log(`[Cache Progress] Downloaded ${Math.min(i + downloadLimit, downloadTasks.length)}/${downloadTasks.length} files`);
528
+
console.log(`[Cache Progress] Downloaded ${Math.min(i + downloadLimit, downloadTasks.length)}/${downloadTasks.length} files (${failureCount} failed)`);
499
529
}
530
+
}
531
+
532
+
if (failureCount > 0) {
533
+
console.warn(`[Cache] Completed with ${successCount} successful and ${failureCount} failed file downloads`);
500
534
}
501
535
}
502
536
···
555
589
}
556
590
557
591
const blobUrl = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`;
592
+
593
+
console.log(`[Cache] Fetching blob for file: ${filePath}, CID: ${cid}`);
558
594
559
595
// Allow up to 500MB per file blob, with 5 minute timeout
560
596
let content = await safeFetchBlob(blobUrl, { maxSize: 500 * 1024 * 1024, timeout: 300000 });
+2
-1
lexicons/fs.json
+2
-1
lexicons/fs.json
···
51
51
"required": ["type", "subject"],
52
52
"properties": {
53
53
"type": { "type": "string", "const": "subfs" },
54
-
"subject": { "type": "string", "format": "at-uri", "description": "AT-URI pointing to a place.wisp.subfs record containing this subtree. When expanded, the subfs record's root entries are merged (flattened) into the parent directory - the subfs entry itself is removed and replaced by all entries from the referenced record's root. This allows splitting large directories across multiple records while maintaining a flat structure." }
54
+
"subject": { "type": "string", "format": "at-uri", "description": "AT-URI pointing to a place.wisp.subfs record containing this subtree." },
55
+
"flat": { "type": "boolean", "description": "If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure." }
55
56
}
56
57
}
57
58
}
+8
-3
src/lexicons/lexicons.ts
+8
-3
src/lexicons/lexicons.ts
···
118
118
type: 'string',
119
119
format: 'at-uri',
120
120
description:
121
-
'AT-URI pointing to a place.wisp.subfs record containing this subtree',
121
+
'AT-URI pointing to a place.wisp.subfs record containing this subtree.',
122
+
},
123
+
flat: {
124
+
type: 'boolean',
125
+
description:
126
+
"If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure.",
122
127
},
123
128
},
124
129
},
···
131
136
main: {
132
137
type: 'record',
133
138
description:
134
-
'Virtual filesystem manifest within a place.wisp.fs record',
139
+
'Virtual filesystem subtree referenced by place.wisp.fs records. When a subfs entry is expanded, its root entries are merged (flattened) into the parent directory, allowing large directories to be split across multiple records while maintaining a flat structure.',
135
140
record: {
136
141
type: 'object',
137
142
required: ['root', 'createdAt'],
···
230
235
type: 'string',
231
236
format: 'at-uri',
232
237
description:
233
-
'AT-URI pointing to another place.wisp.subfs record for nested subtrees',
238
+
"AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures.",
234
239
},
235
240
},
236
241
},
+3
-1
src/lexicons/types/place/wisp/fs.ts
+3
-1
src/lexicons/types/place/wisp/fs.ts
···
93
93
export interface Subfs {
94
94
$type?: 'place.wisp.fs#subfs'
95
95
type: 'subfs'
96
-
/** AT-URI pointing to a place.wisp.subfs record containing this subtree */
96
+
/** AT-URI pointing to a place.wisp.subfs record containing this subtree. */
97
97
subject: string
98
+
/** If true, the subfs record's root entries are merged (flattened) into the parent directory, replacing the subfs entry. If false (default), the subfs entries are placed in a subdirectory with the subfs entry's name. Flat merging is useful for splitting large directories across multiple records while maintaining a flat structure. */
99
+
flat?: boolean
98
100
}
99
101
100
102
const hashSubfs = 'subfs'
+1
-1
src/lexicons/types/place/wisp/subfs.ts
+1
-1
src/lexicons/types/place/wisp/subfs.ts
···
92
92
export interface Subfs {
93
93
$type?: 'place.wisp.subfs#subfs'
94
94
type: 'subfs'
95
-
/** AT-URI pointing to another place.wisp.subfs record for nested subtrees */
95
+
/** AT-URI pointing to another place.wisp.subfs record for nested subtrees. When expanded, the referenced record's root entries are merged (flattened) into the parent directory, allowing recursive splitting of large directory structures. */
96
96
subject: string
97
97
}
98
98
+2
-1
src/lib/wisp-utils.ts
+2
-1
src/lib/wisp-utils.ts