atscan.net
PLC Bundle V1 Example Implementations
update
tree.fail
+44
-113
plcbundle.ts
+44
-113
plcbundle.ts
···
8
import path from 'path';
9
import crypto from 'crypto';
10
import { fileURLToPath } from 'url';
11
-
import { init, compress, decompress } from '@bokuweb/zstd-wasm';
12
import axios from 'axios';
13
14
const __dirname = path.dirname(fileURLToPath(import.meta.url));
···
84
};
85
86
// ============================================================================
87
-
// Bundle Loading
88
-
// ============================================================================
89
-
90
-
const loadBundle = async (dir: string, bundleNumber: number): Promise<PLCOperation[]> => {
91
-
const filename = `${String(bundleNumber).padStart(6, '0')}.jsonl.zst`;
92
-
const filepath = path.join(dir, filename);
93
-
94
-
const compressed = await fs.readFile(filepath);
95
-
const decompressed = decompress(compressed);
96
-
const jsonl = Buffer.from(decompressed).toString('utf8');
97
-
98
-
const lines = jsonl.trim().split('\n').filter(l => l);
99
-
return lines.map(line => {
100
-
const op = JSON.parse(line) as PLCOperation;
101
-
op._raw = line;
102
-
return op;
103
-
});
104
-
};
105
-
106
-
// ============================================================================
107
-
// Boundary Handling
108
-
// ============================================================================
109
-
110
-
const getBoundaryCIDs = (operations: PLCOperation[]): Set<string> => {
111
-
if (operations.length === 0) return new Set();
112
-
113
-
const lastOp = operations[operations.length - 1];
114
-
const boundaryTime = lastOp.createdAt;
115
-
const cidSet = new Set<string>();
116
-
117
-
// Walk backwards from the end to find all operations with the same timestamp
118
-
for (let i = operations.length - 1; i >= 0; i--) {
119
-
if (operations[i].createdAt === boundaryTime) {
120
-
cidSet.add(operations[i].cid);
121
-
} else {
122
-
break;
123
-
}
124
-
}
125
-
126
-
return cidSet;
127
-
};
128
-
129
-
const stripBoundaryDuplicates = (
130
-
operations: PLCOperation[],
131
-
prevBoundaryCIDs: Set<string>
132
-
): PLCOperation[] => {
133
-
if (prevBoundaryCIDs.size === 0) return operations;
134
-
if (operations.length === 0) return operations;
135
-
136
-
const boundaryTime = operations[0].createdAt;
137
-
let startIdx = 0;
138
-
139
-
// Skip operations that are in the previous bundle's boundary
140
-
for (let i = 0; i < operations.length; i++) {
141
-
const op = operations[i];
142
-
143
-
// Stop if we've moved past the boundary timestamp
144
-
if (op.createdAt > boundaryTime) {
145
-
break;
146
-
}
147
-
148
-
// Skip if this CID was in the previous boundary
149
-
if (op.createdAt === boundaryTime && prevBoundaryCIDs.has(op.cid)) {
150
-
startIdx = i + 1;
151
-
continue;
152
-
}
153
-
154
-
break;
155
-
}
156
-
157
-
const stripped = operations.slice(startIdx);
158
-
if (startIdx > 0) {
159
-
console.log(` Stripped ${startIdx} boundary duplicates`);
160
-
}
161
-
return stripped;
162
-
};
163
-
164
-
// ============================================================================
165
// PLC Directory Client
166
// ============================================================================
167
···
190
// ============================================================================
191
192
const serializeJSONL = (operations: PLCOperation[]): string => {
193
const lines = operations.map(op => {
194
const json = op._raw || JSON.stringify(op);
195
return json + '\n';
···
204
const calculateChainHash = (parent: string, contentHash: string): string => {
205
let data: string;
206
if (!parent || parent === '') {
207
data = `plcbundle:genesis:${contentHash}`;
208
} else {
209
data = `${parent}:${contentHash}`;
210
}
211
return sha256(data);
···
226
const filename = `${String(bundleNumber).padStart(6, '0')}.jsonl.zst`;
227
const filepath = path.join(dir, filename);
228
229
const jsonl = serializeJSONL(operations);
230
const uncompressedBuffer = Buffer.from(jsonl, 'utf8');
231
232
const contentHash = sha256(uncompressedBuffer);
233
const uncompressedSize = uncompressedBuffer.length;
234
235
const chainHash = calculateChainHash(parentHash, contentHash);
236
237
const compressed = compress(uncompressedBuffer, 3);
238
const compressedBuffer = Buffer.from(compressed);
239
const compressedHash = sha256(compressedBuffer);
240
const compressedSize = compressedBuffer.length;
241
242
await fs.writeFile(filepath, compressedBuffer);
243
244
const startTime = operations[0].createdAt;
245
const endTime = operations[operations.length - 1].createdAt;
246
const didCount = extractUniqueDIDs(operations);
···
251
end_time: endTime,
252
operation_count: operations.length,
253
did_count: didCount,
254
-
hash: chainHash,
255
-
content_hash: contentHash,
256
-
parent: parentHash || '',
257
compressed_hash: compressedHash,
258
compressed_size: compressedSize,
259
uncompressed_size: uncompressedSize,
···
266
// ============================================================================
267
268
const run = async (): Promise<void> => {
269
-
const dir = process.argv[2] || './plc_bundles';
270
271
console.log('PLC Bundle Fetcher');
272
console.log('==================');
···
277
278
await fs.mkdir(dir, { recursive: true });
279
280
const index = await loadIndex(dir);
281
282
let currentBundle = index.last_bundle + 1;
283
let cursor: string | null = null;
284
let parentHash = '';
285
-
let prevBoundaryCIDs = new Set<string>();
286
287
if (index.bundles.length > 0) {
288
const lastBundle = index.bundles[index.bundles.length - 1];
289
cursor = lastBundle.end_time;
290
-
parentHash = lastBundle.hash;
291
-
292
-
try {
293
-
const prevOps = await loadBundle(dir, lastBundle.bundle_number);
294
-
prevBoundaryCIDs = getBoundaryCIDs(prevOps);
295
-
console.log(`Loaded previous bundle boundary: ${prevBoundaryCIDs.size} CIDs`);
296
-
} catch (err) {
297
-
console.log(`Could not load previous bundle for boundary detection`);
298
-
}
299
-
300
console.log(`Resuming from bundle ${currentBundle}`);
301
console.log(`Last operation: ${cursor}`);
302
} else {
303
console.log('Starting from the beginning (genesis)');
304
}
···
306
console.log();
307
308
let mempool: PLCOperation[] = [];
309
-
const seenCIDs = new Set<string>(prevBoundaryCIDs);
310
let totalFetched = 0;
311
let totalBundles = 0;
312
313
while (true) {
314
try {
315
console.log(`Fetching operations (cursor: ${cursor || 'start'})...`);
316
const operations = await fetchOperations(cursor, 1000);
317
···
320
break;
321
}
322
323
-
// Deduplicate
324
-
const uniqueOps = operations.filter(op => {
325
-
if (seenCIDs.has(op.cid)) {
326
-
return false;
327
-
}
328
-
seenCIDs.add(op.cid);
329
-
return true;
330
-
});
331
332
-
console.log(` Fetched ${operations.length} operations (${uniqueOps.length} unique)`);
333
-
totalFetched += uniqueOps.length;
334
335
-
mempool.push(...uniqueOps);
336
cursor = operations[operations.length - 1].createdAt;
337
338
while (mempool.length >= BUNDLE_SIZE) {
339
const bundleOps = mempool.splice(0, BUNDLE_SIZE);
340
341
-
console.log(`\nCreating bundle ${String(currentBundle).padStart(6, '0')}...`);
342
343
const metadata = await saveBundle(dir, currentBundle, bundleOps, parentHash);
344
345
index.bundles.push(metadata);
346
index.last_bundle = currentBundle;
347
index.total_size_bytes += metadata.compressed_size;
···
351
console.log(` ✓ Bundle ${String(currentBundle).padStart(6, '0')}: ${metadata.operation_count} ops, ${metadata.did_count} DIDs`);
352
console.log(` Chain Hash: ${metadata.hash}`);
353
console.log(` Content Hash: ${metadata.content_hash}`);
354
-
console.log(` Size: ${(metadata.compressed_size / 1024).toFixed(1)} KB`);
355
-
356
-
// Get boundary CIDs for next bundle
357
-
prevBoundaryCIDs = getBoundaryCIDs(bundleOps);
358
-
console.log(` Boundary CIDs: ${prevBoundaryCIDs.size}`);
359
console.log();
360
361
parentHash = metadata.hash;
362
currentBundle++;
363
totalBundles++;
364
}
365
366
await new Promise(resolve => setTimeout(resolve, 100));
367
368
} catch (err: any) {
···
382
}
383
}
384
385
await saveIndex(dir, index);
386
387
console.log();
···
396
397
if (mempool.length > 0) {
398
console.log();
399
-
console.log(`Note: ${mempool.length} operations in mempool`);
400
}
401
};
402
···
8
import path from 'path';
9
import crypto from 'crypto';
10
import { fileURLToPath } from 'url';
11
+
import { init, compress } from '@bokuweb/zstd-wasm';
12
import axios from 'axios';
13
14
const __dirname = path.dirname(fileURLToPath(import.meta.url));
···
84
};
85
86
// ============================================================================
87
// PLC Directory Client
88
// ============================================================================
89
···
112
// ============================================================================
113
114
const serializeJSONL = (operations: PLCOperation[]): string => {
115
+
// Each operation followed by \n, but NO trailing newline at the end
116
+
// This matches the Go implementation exactly
117
const lines = operations.map(op => {
118
const json = op._raw || JSON.stringify(op);
119
return json + '\n';
···
128
const calculateChainHash = (parent: string, contentHash: string): string => {
129
let data: string;
130
if (!parent || parent === '') {
131
+
// Genesis bundle (first bundle)
132
data = `plcbundle:genesis:${contentHash}`;
133
} else {
134
+
// Subsequent bundles - chain parent hash with current content
135
data = `${parent}:${contentHash}`;
136
}
137
return sha256(data);
···
152
const filename = `${String(bundleNumber).padStart(6, '0')}.jsonl.zst`;
153
const filepath = path.join(dir, filename);
154
155
+
// Serialize to JSONL (exact format: each line ends with \n, no trailing newline)
156
const jsonl = serializeJSONL(operations);
157
const uncompressedBuffer = Buffer.from(jsonl, 'utf8');
158
159
+
console.log(` JSONL size: ${uncompressedBuffer.length} bytes`);
160
+
console.log(` First 100 chars: ${jsonl.substring(0, 100)}`);
161
+
console.log(` Last 100 chars: ${jsonl.substring(jsonl.length - 100)}`);
162
+
163
+
// Calculate content hash
164
const contentHash = sha256(uncompressedBuffer);
165
const uncompressedSize = uncompressedBuffer.length;
166
167
+
// Calculate chain hash
168
const chainHash = calculateChainHash(parentHash, contentHash);
169
170
+
// Compress with zstd level 3 (same as Go SpeedDefault)
171
const compressed = compress(uncompressedBuffer, 3);
172
const compressedBuffer = Buffer.from(compressed);
173
const compressedHash = sha256(compressedBuffer);
174
const compressedSize = compressedBuffer.length;
175
176
+
// Write file
177
await fs.writeFile(filepath, compressedBuffer);
178
179
+
// Extract metadata
180
const startTime = operations[0].createdAt;
181
const endTime = operations[operations.length - 1].createdAt;
182
const didCount = extractUniqueDIDs(operations);
···
187
end_time: endTime,
188
operation_count: operations.length,
189
did_count: didCount,
190
+
hash: chainHash, // Chain hash (primary)
191
+
content_hash: contentHash, // Content hash
192
+
parent: parentHash || '', // Parent chain hash
193
compressed_hash: compressedHash,
194
compressed_size: compressedSize,
195
uncompressed_size: uncompressedSize,
···
202
// ============================================================================
203
204
const run = async (): Promise<void> => {
205
+
const dir = process.argv[2] || './bundles';
206
207
console.log('PLC Bundle Fetcher');
208
console.log('==================');
···
213
214
await fs.mkdir(dir, { recursive: true });
215
216
+
// Load existing index
217
const index = await loadIndex(dir);
218
219
let currentBundle = index.last_bundle + 1;
220
let cursor: string | null = null;
221
let parentHash = '';
222
223
+
// If resuming, get cursor and parent from last bundle
224
if (index.bundles.length > 0) {
225
const lastBundle = index.bundles[index.bundles.length - 1];
226
cursor = lastBundle.end_time;
227
+
parentHash = lastBundle.hash; // Chain hash from previous bundle
228
console.log(`Resuming from bundle ${currentBundle}`);
229
console.log(`Last operation: ${cursor}`);
230
+
console.log(`Parent hash: ${parentHash}`);
231
} else {
232
console.log('Starting from the beginning (genesis)');
233
}
···
235
console.log();
236
237
let mempool: PLCOperation[] = [];
238
let totalFetched = 0;
239
let totalBundles = 0;
240
241
while (true) {
242
try {
243
+
// Fetch operations
244
console.log(`Fetching operations (cursor: ${cursor || 'start'})...`);
245
const operations = await fetchOperations(cursor, 1000);
246
···
249
break;
250
}
251
252
+
console.log(` Fetched ${operations.length} operations`);
253
+
totalFetched += operations.length;
254
255
+
// Add to mempool
256
+
mempool.push(...operations);
257
258
+
// Update cursor
259
cursor = operations[operations.length - 1].createdAt;
260
261
+
// Create bundles while we have enough operations
262
while (mempool.length >= BUNDLE_SIZE) {
263
const bundleOps = mempool.splice(0, BUNDLE_SIZE);
264
265
+
console.log(`Creating bundle ${String(currentBundle).padStart(6, '0')}...`);
266
267
const metadata = await saveBundle(dir, currentBundle, bundleOps, parentHash);
268
269
+
// Add to index
270
index.bundles.push(metadata);
271
index.last_bundle = currentBundle;
272
index.total_size_bytes += metadata.compressed_size;
···
276
console.log(` ✓ Bundle ${String(currentBundle).padStart(6, '0')}: ${metadata.operation_count} ops, ${metadata.did_count} DIDs`);
277
console.log(` Chain Hash: ${metadata.hash}`);
278
console.log(` Content Hash: ${metadata.content_hash}`);
279
+
if (metadata.parent) {
280
+
console.log(` Parent Hash: ${metadata.parent}`);
281
+
} else {
282
+
console.log(` Parent Hash: (genesis)`);
283
+
}
284
+
console.log(` Compressed: ${metadata.compressed_hash}`);
285
+
console.log(` Size: ${(metadata.compressed_size / 1024).toFixed(1)} KB (${(metadata.compressed_size / metadata.uncompressed_size * 100).toFixed(1)}% of original)`);
286
console.log();
287
288
+
// Update parent hash for next bundle
289
parentHash = metadata.hash;
290
+
291
currentBundle++;
292
totalBundles++;
293
}
294
295
+
// Small delay to be nice to the server
296
await new Promise(resolve => setTimeout(resolve, 100));
297
298
} catch (err: any) {
···
312
}
313
}
314
315
+
// Save final index
316
await saveIndex(dir, index);
317
318
console.log();
···
327
328
if (mempool.length > 0) {
329
console.log();
330
+
console.log(`Note: ${mempool.length} operations in mempool (need ${BUNDLE_SIZE - mempool.length} more for next bundle)`);
331
}
332
};
333