mary.my.id
handy online tools for AT Protocol
boat.kelinci.net
atproto
bluesky
atcute
typescript
solidjs
refactor: use streamed repo reader
+3
src/main.tsx
+3
src/main.tsx
···
22
if (Symbol.dispose === undefined) {
23
Object.defineProperty(Symbol, 'dispose', { value: Symbol.for(`Symbol.dispose`) });
24
}
25
+
if (Symbol.asyncDispose === undefined) {
26
+
Object.defineProperty(Symbol, 'asyncDispose', { value: Symbol.for(`Symbol.asyncDispose`) });
27
+
}
28
29
render(App, document.body);
+7
-3
src/views/repository/repo-archive-explore/page.tsx
+7
-3
src/views/repository/repo-archive-explore/page.tsx
···
1
import { Match, Switch } from 'solid-js';
2
3
-
import { iterateAtpRepo } from '@atcute/car';
4
5
import { createMutation } from '~/lib/utils/mutation';
6
···
12
const ArchiveExplorePage = () => {
13
const mutation = createMutation({
14
async mutationFn({ file }: { file: File }): Promise<Archive> {
15
-
const buffer = new Uint8Array(await file.arrayBuffer());
16
17
const collections = new Map<string, RecordEntry[]>();
18
const archive: Archive = {
···
20
entries: [],
21
};
22
23
-
for (const entry of iterateAtpRepo(buffer)) {
24
let list = collections.get(entry.collection);
25
if (list === undefined) {
26
collections.set(entry.collection, (list = []));
···
41
}
42
43
return archive;
44
},
45
});
46
···
1
import { Match, Switch } from 'solid-js';
2
3
+
import { RepoReader } from '@atcute/car/v4';
4
5
import { createMutation } from '~/lib/utils/mutation';
6
···
12
const ArchiveExplorePage = () => {
13
const mutation = createMutation({
14
async mutationFn({ file }: { file: File }): Promise<Archive> {
15
+
const stream = file.stream();
16
+
await using repo = RepoReader.fromStream(stream);
17
18
const collections = new Map<string, RecordEntry[]>();
19
const archive: Archive = {
···
21
entries: [],
22
};
23
24
+
for await (const entry of repo) {
25
let list = collections.get(entry.collection);
26
if (list === undefined) {
27
collections.set(entry.collection, (list = []));
···
42
}
43
44
return archive;
45
+
},
46
+
onError(err) {
47
+
console.error(err);
48
},
49
});
50
+8
-15
src/views/repository/repo-archive-unpack.tsx
+8
-15
src/views/repository/repo-archive-unpack.tsx
···
1
import { FileSystemWritableFileStream, showSaveFilePicker } from 'native-file-system-adapter';
2
import { createSignal } from 'solid-js';
3
4
-
import { iterateAtpRepo } from '@atcute/car';
5
import { writeTarEntry } from '@mary/tar';
6
7
import { createDropZone } from '~/lib/hooks/dropzone';
···
39
});
40
41
const mutate = async (file: File, signal: AbortSignal) => {
42
-
logger.info(`Starting extraction for ${file.name}`);
43
44
-
const buf = await file.arrayBuffer();
45
-
const ui8 = new Uint8Array(buf);
46
47
-
let currentCollection: string | undefined;
48
let count = 0;
49
50
let writable: FileSystemWritableFileStream | undefined;
51
52
-
for (const { collection, rkey, record } of iterateAtpRepo(ui8)) {
53
if (writable === undefined) {
54
using _progress = logger.progress(`Waiting for the user`);
55
···
87
88
signal.throwIfAborted();
89
90
-
if (currentCollection !== collection) {
91
-
logger.log(`Current progress: ${collection}`);
92
-
currentCollection = collection;
93
-
94
-
if (yieldToScheduler === undefined) {
95
-
await yieldToIdle();
96
-
}
97
-
}
98
-
99
const entry = writeTarEntry({
100
filename: `${collection}/${filenamify(rkey)}.json`,
101
data: JSON.stringify(record, null, 2),
102
});
103
104
writable.write(entry);
105
count++;
106
···
1
import { FileSystemWritableFileStream, showSaveFilePicker } from 'native-file-system-adapter';
2
import { createSignal } from 'solid-js';
3
4
+
import { RepoReader } from '@atcute/car/v4';
5
import { writeTarEntry } from '@mary/tar';
6
7
import { createDropZone } from '~/lib/hooks/dropzone';
···
39
});
40
41
const mutate = async (file: File, signal: AbortSignal) => {
42
+
logger.log(`Starting extraction`);
43
44
+
const stream = file.stream();
45
+
await using repo = RepoReader.fromStream(stream);
46
47
let count = 0;
48
49
let writable: FileSystemWritableFileStream | undefined;
50
51
+
using progress = logger.progress(`Unpacking records (${count} entries)`);
52
+
53
+
for await (const { collection, rkey, record } of repo) {
54
if (writable === undefined) {
55
using _progress = logger.progress(`Waiting for the user`);
56
···
88
89
signal.throwIfAborted();
90
91
const entry = writeTarEntry({
92
filename: `${collection}/${filenamify(rkey)}.json`,
93
data: JSON.stringify(record, null, 2),
94
});
95
96
+
progress.update(`Unpacking records (${count} entries)`);
97
writable.write(entry);
98
count++;
99