atmosphere explorer

fix unpacking large repos on webkit

handle.invalid 6dbf4732 31c6e076

verified
+105 -44
+3 -19
src/views/car/explore.tsx
··· 10 10 import { TextInput } from "../../components/text-input.jsx"; 11 11 import { isTouchDevice } from "../../layout.jsx"; 12 12 import { localDateFromTimestamp } from "../../utils/date.js"; 13 + import { createDropHandler, createFileChangeHandler, handleDragOver } from "./file-handlers.js"; 13 14 import { 14 15 type Archive, 15 16 type CollectionEntry, ··· 98 99 } 99 100 }; 100 101 101 - const handleFileChange = (e: Event) => { 102 - const input = e.target as HTMLInputElement; 103 - const file = input.files?.[0]; 104 - if (file) { 105 - parseCarFile(file); 106 - } 107 - }; 108 - 109 - const handleDrop = (e: DragEvent) => { 110 - e.preventDefault(); 111 - const file = e.dataTransfer?.files?.[0]; 112 - if (file && (file.name.endsWith(".car") || file.type === "application/vnd.ipld.car")) { 113 - parseCarFile(file); 114 - } 115 - }; 116 - 117 - const handleDragOver = (e: DragEvent) => { 118 - e.preventDefault(); 119 - }; 102 + const handleFileChange = createFileChangeHandler(parseCarFile); 103 + const handleDrop = createDropHandler(parseCarFile); 120 104 121 105 const reset = () => { 122 106 setArchive(null);
+24
src/views/car/file-handlers.ts
··· 1 + export const isCarFile = (file: File): boolean => { 2 + return file.name.endsWith(".car") || file.type === "application/vnd.ipld.car"; 3 + }; 4 + 5 + export const createFileChangeHandler = (onFile: (file: File) => void) => (e: Event) => { 6 + const input = e.target as HTMLInputElement; 7 + const file = input.files?.[0]; 8 + if (file) { 9 + onFile(file); 10 + } 11 + input.value = ""; 12 + }; 13 + 14 + export const createDropHandler = (onFile: (file: File) => void) => (e: DragEvent) => { 15 + e.preventDefault(); 16 + const file = e.dataTransfer?.files?.[0]; 17 + if (file && isCarFile(file)) { 18 + onFile(file); 19 + } 20 + }; 21 + 22 + export const handleDragOver = (e: DragEvent) => { 23 + e.preventDefault(); 24 + };
+78 -25
src/views/car/unpack.tsx
··· 3 3 import { Title } from "@solidjs/meta"; 4 4 import { FileSystemWritableFileStream, showSaveFilePicker } from "native-file-system-adapter"; 5 5 import { createSignal, onCleanup } from "solid-js"; 6 + import { createDropHandler, createFileChangeHandler, handleDragOver } from "./file-handlers.js"; 6 7 import { createLogger, LoggerView } from "./logger.jsx"; 7 8 import { isIOS, toJsonValue, WelcomeView } from "./shared.jsx"; 9 + 10 + // Check if browser natively supports File System Access API 11 + const hasNativeFileSystemAccess = "showSaveFilePicker" in window; 8 12 9 13 // HACK: Disable compression on WebKit due to an error being thrown 10 14 const isWebKit = ··· 40 44 41 45 try { 42 46 let count = 0; 47 + 48 + // On Safari/browsers without native File System Access API, use blob download 49 + if (!hasNativeFileSystemAccess) { 50 + const chunks: BlobPart[] = []; 51 + 52 + const entryGenerator = async function* (): AsyncGenerator<ZipEntry> { 53 + const progress = logger.progress(`Unpacking records (0 entries)`); 54 + 55 + try { 56 + for await (const entry of repo) { 57 + if (signal.aborted) return; 58 + 59 + try { 60 + const record = toJsonValue(entry.record); 61 + const filename = `${entry.collection}/${filenamify(entry.rkey)}.json`; 62 + const data = JSON.stringify(record, null, 2); 63 + 64 + yield { filename, data, compress: isWebKit ? false : "deflate" }; 65 + count++; 66 + progress.update(`Unpacking records (${count} entries)`); 67 + } catch { 68 + // Skip entries with invalid data 69 + } 70 + } 71 + } finally { 72 + progress[Symbol.dispose]?.(); 73 + } 74 + }; 75 + 76 + for await (const chunk of zip(entryGenerator())) { 77 + if (signal.aborted) return; 78 + chunks.push(chunk as BlobPart); 79 + } 80 + 81 + if (signal.aborted) return; 82 + 83 + logger.log(`${count} records extracted`); 84 + logger.log(`Creating download...`); 85 + 86 + const blob = new Blob(chunks, { type: "application/zip" }); 87 + const url = URL.createObjectURL(blob); 88 + const a = document.createElement("a"); 89 + a.href = url; 90 + a.download = `${file.name.replace(/\.car$/, "")}.zip`; 91 + document.body.appendChild(a); 92 + a.click(); 93 + document.body.removeChild(a); 94 + URL.revokeObjectURL(url); 95 + 96 + logger.log(`Finished! Download started.`); 97 + setPending(false); 98 + return; 99 + } 100 + 101 + // Native File System Access API path 43 102 let writable: FileSystemWritableFileStream | undefined; 44 103 45 104 // Create async generator that yields ZipEntry as we read from CAR ··· 70 129 if (err instanceof DOMException && err.name === "AbortError") { 71 130 logger.warn(`File picker was cancelled`); 72 131 } else { 73 - console.warn(err); 74 132 logger.warn(`Something went wrong when opening the file picker`); 75 133 } 76 134 return undefined; 77 135 }); 78 136 79 - writable = await fd?.createWritable(); 137 + if (!fd) { 138 + logger.warn(`No file handle obtained`); 139 + return; 140 + } 141 + 142 + writable = await fd.createWritable(); 80 143 81 144 if (writable === undefined) { 145 + logger.warn(`Failed to create writable stream`); 82 146 return; 83 147 } 84 148 } finally { ··· 116 180 return; 117 181 } 118 182 writeCount++; 119 - if (writeCount % 100 !== 0) { 183 + // Await every 100th write to apply backpressure 184 + if (writeCount % 100 === 0) { 185 + await writable.write(chunk); 186 + } else { 120 187 writable.write(chunk); // Fire and forget 121 - } else { 122 - await writable.write(chunk); // Await periodically to apply backpressure 123 188 } 124 189 } 125 190 ··· 137 202 const flushProgress = logger.progress(`Flushing writes...`); 138 203 try { 139 204 await writable.close(); 205 + logger.log(`Finished! File saved successfully.`); 206 + } catch (err) { 207 + logger.error(`Failed to save file: ${err}`); 208 + throw err; // Re-throw to be caught by outer catch 140 209 } finally { 141 210 flushProgress[Symbol.dispose]?.(); 142 211 } 143 212 } 144 - 145 - logger.log(`Finished!`); 146 213 } catch (err) { 147 214 if (signal.aborted) return; 148 - console.error("Failed to unpack CAR file:", err); 149 215 logger.error(`Error: ${err}\nFile might be malformed, or might not be a CAR archive`); 150 216 } finally { 151 217 await repo?.dispose(); ··· 155 221 } 156 222 }; 157 223 158 - const handleFileChange = (e: Event) => { 159 - const input = e.target as HTMLInputElement; 160 - const file = input.files?.[0]; 161 - if (file) { 162 - unpackToZip(file); 163 - } 164 - input.value = ""; 165 - }; 224 + const handleFileChange = createFileChangeHandler(unpackToZip); 166 225 226 + // Wrap handleDrop to prevent multiple simultaneous uploads 227 + const baseDrop = createDropHandler(unpackToZip); 167 228 const handleDrop = (e: DragEvent) => { 168 - e.preventDefault(); 169 229 if (pending()) return; 170 - const file = e.dataTransfer?.files?.[0]; 171 - if (file && (file.name.endsWith(".car") || file.type === "application/vnd.ipld.car")) { 172 - unpackToZip(file); 173 - } 174 - }; 175 - 176 - const handleDragOver = (e: DragEvent) => { 177 - e.preventDefault(); 230 + baseDrop(e); 178 231 }; 179 232 180 233 return (