tree.fail / plcbundle-watch
this repo has no description plcbundle-watch.pages.dev
fork atom

bundle downloader

tree.fail f1b2c6d7 252b1909

deploy.yml
+515 -4
+7 -4
src/App.svelte
··· 4 4 import { formatDistanceToNow, addSeconds, subSeconds, formatDate, formatISO9075 } from 'date-fns'; 5 5 import { Progress, Switch } from '@skeletonlabs/skeleton-svelte'; 6 6 import orderBy from "lodash/orderBy"; 7 + import BundleDownloader from './BundleDownloader.svelte'; 7 8 import { formatNumber, formatUptime } from './lib/utils'; 8 9 import instancesData from './instances.json'; 9 10 ··· 180 181 }) 181 182 </script> 182 183 183 - <main class="w-full mt-10"> 184 + <main class="w-full mt-10 mb-16"> 184 185 <div class="max-w-5xl mx-auto px-3"> 185 186 186 187 <header class="flex items-center gap-10 flex-wrap"> ··· 299 300 <span class="opacity-75">Root:</span> <span class="font-mono text-xs">{ROOT.slice(0)}</span> 300 301 </div> 301 302 </div> 303 + <hr class="hr my-10" /> 304 + 305 + <BundleDownloader instances={instances} /> 302 306 303 - <hr class="hr mt-6" /> 307 + <hr class="hr my-6" /> 304 308 <div class="mt-2 opacity-50"> 305 309 <div> 306 310 Last updated: {formatISO9075(lastUpdated)} ··· 309 313 Source: <a href="https://tangled.org/@tree.fail/plcbundle-watch">https://tangled.org/@tree.fail/plcbundle-watch</a> 310 314 </div> 311 315 </div> 312 - 313 - 316 + 314 317 315 318 </div> 316 319 </main>
+508
src/BundleDownloader.svelte
··· 1 + <script lang="ts"> 2 + import { Progress } from '@skeletonlabs/skeleton-svelte'; 3 + import { tick } from 'svelte'; 4 + 5 + type Instance = { 6 + url: string; 7 + name?: string; 8 + } 9 + 10 + type InstanceStatus = { 11 + url: string; 12 + lastBundle: number; 13 + } 14 + 15 + type DownloadedBundle = { 16 + number: number; 17 + status: 'downloading' | 'success' | 'error' | 'cancelled'; 18 + size?: number; 19 + error?: string; 20 + source?: string; 21 + } 22 + 23 + let { instances = [] }: { instances: Instance[] } = $props(); 24 + 25 + let selectedInstance = $state('random'); 26 + let bundlesInput = $state(''); 27 + let isDownloading = $state(false); 28 + let downloadedBundles = $state<DownloadedBundle[]>([]); 29 + let progress = $state(0); 30 + let totalBundles = $state(0); 31 + let abortController: AbortController | null = null; 32 + let isStopping = $state(false); 33 + let instanceStatuses = $state<InstanceStatus[]>([]); 34 + let useDirectory = $state(true); 35 + let directoryHandle: FileSystemDirectoryHandle | null = null; 36 + let hasFileSystemAccess = $state(false); 37 + 38 + // Check if File System Access API is available 39 + $effect(() => { 40 + hasFileSystemAccess = 'showDirectoryPicker' in window; 41 + if (!hasFileSystemAccess) { 42 + useDirectory = false; 43 + } 44 + }); 45 + 46 + async function pickDirectory(): Promise<boolean> { 47 + if (!hasFileSystemAccess) { 48 + return false; 49 + } 50 + 51 + try { 52 + directoryHandle = await (window as any).showDirectoryPicker({ 53 + mode: 'readwrite' 54 + }); 55 + return true; 56 + } catch (e) { 57 + if ((e as Error).name !== 'AbortError') { 58 + console.error('Failed to pick directory:', e); 59 + } 60 + return false; 61 + } 62 + } 63 + 64 + async function fetchInstanceStatuses() { 65 + const statuses: InstanceStatus[] = []; 66 + 67 + await Promise.all(instances.map(async (instance) => { 68 + try { 69 + const response = await fetch(`${instance.url}/status`, { 70 + signal: abortController?.signal 71 + }); 72 + const data = await response.json(); 73 + statuses.push({ 74 + url: instance.url, 75 + lastBundle: data.bundles.last_bundle 76 + }); 77 + } catch (e) { 78 + console.warn(`Failed to fetch status from ${instance.url}`, e); 79 + } 80 + })); 81 + 82 + return statuses; 83 + } 84 + 85 + function getAvailableInstancesForBundle(bundleNumber: number): string[] { 86 + return instanceStatuses 87 + .filter(s => s.lastBundle >= bundleNumber) 88 + .map(s => s.url); 89 + } 90 + 91 + function getRandomInstance(bundleNumber?: number): Instance | null { 92 + let availableUrls: string[]; 93 + 94 + if (bundleNumber !== undefined && instanceStatuses.length > 0) { 95 + availableUrls = getAvailableInstancesForBundle(bundleNumber); 96 + if (availableUrls.length === 0) { 97 + return null; 98 + } 99 + } else { 100 + availableUrls = instances.map(i => i.url); 101 + } 102 + 103 + const randomUrl = availableUrls[Math.floor(Math.random() * availableUrls.length)]; 104 + return instances.find(i => i.url === randomUrl) || null; 105 + } 106 + 107 + function getInstanceUrl(bundleNumber?: number): string | null { 108 + if (selectedInstance === 'random') { 109 + const instance = getRandomInstance(bundleNumber); 110 + return instance?.url || null; 111 + } 112 + return selectedInstance; 113 + } 114 + 115 + function getInstanceName(url: string): string { 116 + const instance = instances.find(i => i.url === url); 117 + return instance?.name || new URL(url).hostname; 118 + } 119 + 120 + function parseBundlesInput(input: string): number[] | 'all' { 121 + const trimmed = input.trim(); 122 + 123 + if (!trimmed) { 124 + return 'all'; 125 + } 126 + 127 + if (trimmed.includes('-')) { 128 + const [start, end] = trimmed.split('-').map(s => parseInt(s.trim())); 129 + if (isNaN(start) || isNaN(end) || start > end) { 130 + throw new Error('Invalid range format'); 131 + } 132 + const bundles = []; 133 + for (let i = start; i <= end; i++) { 134 + bundles.push(i); 135 + } 136 + return bundles; 137 + } 138 + 139 + const num = parseInt(trimmed); 140 + if (isNaN(num)) { 141 + throw new Error('Invalid bundle number'); 142 + } 143 + return [num]; 144 + } 145 + 146 + async function getLastBundle(instanceUrl: string): Promise<number> { 147 + const response = await fetch(`${instanceUrl}/status`, { 148 + signal: abortController?.signal 149 + }); 150 + const data = await response.json(); 151 + return data.bundles.last_bundle; 152 + } 153 + 154 + async function downloadBundle(instanceUrl: string, bundleNumber: number): Promise<Blob> { 155 + const response = await fetch(`${instanceUrl}/data/${bundleNumber}`, { 156 + signal: abortController?.signal 157 + }); 158 + if (!response.ok) { 159 + throw new Error(`HTTP ${response.status}`); 160 + } 161 + return await response.blob(); 162 + } 163 + 164 + function padBundleNumber(num: number): string { 165 + return num.toString().padStart(6, '0'); 166 + } 167 + 168 + async function saveFileToDirectory(blob: Blob, bundleNumber: number) { 169 + if (!directoryHandle) { 170 + throw new Error('No directory selected'); 171 + } 172 + 173 + const fileName = `${padBundleNumber(bundleNumber)}.jsonl.zst`; 174 + const fileHandle = await directoryHandle.getFileHandle(fileName, { create: true }); 175 + const writable = await fileHandle.createWritable(); 176 + await writable.write(blob); 177 + await writable.close(); 178 + } 179 + 180 + function saveFileBrowser(blob: Blob, bundleNumber: number) { 181 + const url = URL.createObjectURL(blob); 182 + const link = document.createElement('a'); 183 + link.href = url; 184 + link.download = `${padBundleNumber(bundleNumber)}.jsonl.zst`; 185 + link.click(); 186 + URL.revokeObjectURL(url); 187 + } 188 + 189 + function stopDownload() { 190 + if (abortController) { 191 + isStopping = true; 192 + abortController.abort(); 193 + } 194 + } 195 + 196 + async function handleDownload() { 197 + if (!selectedInstance) { 198 + alert('Please select an instance'); 199 + return; 200 + } 201 + 202 + // If using directory mode, pick directory first 203 + if (useDirectory && hasFileSystemAccess) { 204 + const picked = await pickDirectory(); 205 + if (!picked) { 206 + return; // User cancelled 207 + } 208 + } 209 + 210 + let bundleNumbers: number[]; 211 + abortController = new AbortController(); 212 + isStopping = false; 213 + 214 + try { 215 + if (selectedInstance === 'random') { 216 + instanceStatuses = await fetchInstanceStatuses(); 217 + if (instanceStatuses.length === 0) { 218 + alert('No instances available'); 219 + return; 220 + } 221 + } 222 + 223 + const parsed = parseBundlesInput(bundlesInput); 224 + 225 + if (parsed === 'all') { 226 + let lastBundle: number; 227 + 228 + if (selectedInstance === 'random') { 229 + lastBundle = Math.max(...instanceStatuses.map(s => s.lastBundle)); 230 + } else { 231 + lastBundle = await getLastBundle(selectedInstance); 232 + } 233 + 234 + bundleNumbers = []; 235 + for (let i = 1; i <= lastBundle; i++) { 236 + bundleNumbers.push(i); 237 + } 238 + } else { 239 + bundleNumbers = parsed; 240 + } 241 + } catch (e) { 242 + if (e instanceof Error && e.name === 'AbortError') { 243 + return; 244 + } 245 + alert(e instanceof Error ? e.message : 'Invalid input'); 246 + return; 247 + } 248 + 249 + isDownloading = true; 250 + downloadedBundles = []; 251 + progress = 0; 252 + totalBundles = bundleNumbers.length; 253 + 254 + for (let i = 0; i < bundleNumbers.length; i++) { 255 + if (abortController?.signal.aborted) { 256 + break; 257 + } 258 + 259 + const bundleNum = bundleNumbers[i]; 260 + const instanceUrl = getInstanceUrl(bundleNum); 261 + 262 + if (!instanceUrl) { 263 + downloadedBundles = [...downloadedBundles, { 264 + number: bundleNum, 265 + status: 'error', 266 + error: 'No instance has this bundle', 267 + }]; 268 + progress = Math.round(((i + 1) / totalBundles) * 100); 269 + await tick(); 270 + continue; 271 + } 272 + 273 + downloadedBundles = [...downloadedBundles, { 274 + number: bundleNum, 275 + status: 'downloading', 276 + source: instanceUrl, 277 + }]; 278 + 279 + await tick(); 280 + 281 + try { 282 + const blob = await downloadBundle(instanceUrl, bundleNum); 283 + 284 + if (abortController?.signal.aborted) { 285 + downloadedBundles = downloadedBundles.map(b => 286 + b.number === bundleNum ? { ...b, status: 'cancelled' as const } : b 287 + ); 288 + break; 289 + } 290 + 291 + // Save file 292 + if (useDirectory && directoryHandle) { 293 + await saveFileToDirectory(blob, bundleNum); 294 + } else { 295 + saveFileBrowser(blob, bundleNum); 296 + } 297 + 298 + downloadedBundles = downloadedBundles.map(b => 299 + b.number === bundleNum 300 + ? { ...b, status: 'success' as const, size: blob.size } 301 + : b 302 + ); 303 + 304 + } catch (e) { 305 + if (e instanceof Error && e.name === 'AbortError') { 306 + downloadedBundles = downloadedBundles.map(b => 307 + b.number === bundleNum ? { ...b, status: 'cancelled' as const } : b 308 + ); 309 + break; 310 + } 311 + 312 + downloadedBundles = downloadedBundles.map(b => 313 + b.number === bundleNum 314 + ? { ...b, status: 'error' as const, error: e instanceof Error ? e.message : 'Unknown error' } 315 + : b 316 + ); 317 + } 318 + 319 + progress = Math.round(((i + 1) / totalBundles) * 100); 320 + await tick(); 321 + } 322 + 323 + isDownloading = false; 324 + isStopping = false; 325 + abortController = null; 326 + directoryHandle = null; 327 + } 328 + 329 + function formatBytes(bytes: number): string { 330 + if (bytes === 0) return '0 B'; 331 + const k = 1024; 332 + const sizes = ['B', 'KB', 'MB', 'GB']; 333 + const i = Math.floor(Math.log(bytes) / Math.log(k)); 334 + return Math.round((bytes / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i]; 335 + } 336 + 337 + function clearResults() { 338 + downloadedBundles = []; 339 + progress = 0; 340 + totalBundles = 0; 341 + } 342 + 343 + let successCount = $derived(downloadedBundles.filter(b => b.status === 'success').length); 344 + let errorCount = $derived(downloadedBundles.filter(b => b.status === 'error').length); 345 + let cancelledCount = $derived(downloadedBundles.filter(b => b.status === 'cancelled').length); 346 + let totalSize = $derived(downloadedBundles.reduce((sum, b) => sum + (b.size || 0), 0)); 347 + </script> 348 + 349 + <div class="bundle-downloader card space-y-4"> 350 + <h2 class="text-2xl">Bundle Downloader</h2> 351 + 352 + <div class="space-y-3"> 353 + <label class="label"> 354 + <span>Instance</span> 355 + <select 356 + class="select p-3 text-sm" 357 + bind:value={selectedInstance} 358 + disabled={isDownloading} 359 + > 360 + <option value="random">🎲 Random (each bundle from different source)</option> 361 + {#each instances as instance} 362 + <option value={instance.url}> 363 + {instance.name || instance.url} 364 + </option> 365 + {/each} 366 + </select> 367 + </label> 368 + 369 + <label class="label"> 370 + <span>Bundles</span> 371 + <input 372 + class="input text-sm" 373 + type="text" 374 + bind:value={bundlesInput} 375 + disabled={isDownloading} 376 + placeholder="empty = all, 5 = single, 1-10 = range" 377 + /> 378 + <p class="text-xs opacity-75 mt-1"> 379 + Leave empty for all bundles, enter a number (e.g., <code>5</code>) or range (e.g., <code>1-10</code>) 380 + </p> 381 + </label> 382 + 383 + {#if hasFileSystemAccess} 384 + <label class="flex items-center space-x-2"> 385 + <input 386 + type="checkbox" 387 + class="checkbox" 388 + bind:checked={useDirectory} 389 + disabled={isDownloading} 390 + /> 391 + <span class="text-sm"> 392 + 📁 Save to directory (recommended for multiple files) 393 + </span> 394 + </label> 395 + {:else} 396 + <div class="alert variant-ghost-warning p-2 text-xs"> 397 + <span>⚠️ Directory mode not available in this browser. Files will download individually.</span> 398 + </div> 399 + {/if} 400 + 401 + <div class="flex gap-2"> 402 + {#if !isDownloading} 403 + <button 404 + class="btn preset-tonal-primary flex-1" 405 + onclick={handleDownload} 406 + > 407 + {useDirectory && hasFileSystemAccess ? '📁 Choose Directory & Download' : '📥 Download'} 408 + </button> 409 + {#if downloadedBundles.length > 0} 410 + <button 411 + class="btn preset-tonal-surface" 412 + onclick={clearResults} 413 + > 414 + 🗑️ Clear 415 + </button> 416 + {/if} 417 + {:else} 418 + <button 419 + class="btn preset-filled-error-500 flex-1" 420 + onclick={stopDownload} 421 + disabled={isStopping} 422 + > 423 + {isStopping ? '⏳ Stopping...' : '⛔ Stop'} 424 + </button> 425 + {/if} 426 + </div> 427 + 428 + {#if isDownloading} 429 + <div class="space-y-2"> 430 + <Progress value={progress} max={100} /> 431 + <p class="text-sm text-center font-semibold"> 432 + {progress}% ({successCount}/{totalBundles}) 433 + </p> 434 + </div> 435 + {/if} 436 + </div> 437 + 438 + {#if downloadedBundles.length > 0} 439 + <div class="space-y-2"> 440 + <h3 class="text-2xl"> 441 + {isDownloading ? 'Downloading...' : 'Results'} 442 + ({successCount}/{downloadedBundles.length}) 443 + </h3> 444 + 445 + <div class="table-container max-h-64 overflow-y-auto"> 446 + <table class="table table-compact table-hover"> 447 + <thead> 448 + <tr> 449 + <th>File</th> 450 + <th>Source</th> 451 + <th>Status</th> 452 + <th class="text-right">Size</th> 453 + </tr> 454 + </thead> 455 + <tbody> 456 + {#each downloadedBundles as bundle (bundle.number)} 457 + <tr> 458 + <td class="font-mono text-xs">{padBundleNumber(bundle.number)}.jsonl.zst</td> 459 + <td class="text-xs" title={bundle.source}> 460 + {bundle.source ? getInstanceName(bundle.source) : '-'} 461 + </td> 462 + <td> 463 + {#if bundle.status === 'downloading'} 464 + <span class="badge variant-filled text-xs">⏳ Downloading</span> 465 + {:else if bundle.status === 'success'} 466 + <span class="badge variant-filled-success text-xs">✅ Success</span> 467 + {:else if bundle.status === 'cancelled'} 468 + <span class="badge variant-filled-warning text-xs">⚠️ Cancelled</span> 469 + {:else} 470 + <span class="badge variant-filled-error text-xs" title={bundle.error}>❌ Error</span> 471 + {/if} 472 + </td> 473 + <td class="text-sm text-right">{bundle.size ? formatBytes(bundle.size) : '-'}</td> 474 + </tr> 475 + {/each} 476 + </tbody> 477 + </table> 478 + </div> 479 + 480 + <div class="card p-3 variant-ghost-surface grid grid-cols-4 gap-2 text-sm"> 481 + <div> 482 + <div class="font-bold text-success-500"> 483 + {successCount} 484 + </div> 485 + <div class="opacity-75">Success</div> 486 + </div> 487 + <div> 488 + <div class="font-bold text-error-500"> 489 + {errorCount} 490 + </div> 491 + <div class="opacity-75">Failed</div> 492 + </div> 493 + <div> 494 + <div class="font-bold text-warning-500"> 495 + {cancelledCount} 496 + </div> 497 + <div class="opacity-75">Cancelled</div> 498 + </div> 499 + <div> 500 + <div class="font-bold"> 501 + {formatBytes(totalSize)} 502 + </div> 503 + <div class="opacity-75">Total</div> 504 + </div> 505 + </div> 506 + </div> 507 + {/if} 508 + </div>