gbl08ma.com
A very experimental PLC implementation which uses BFT consensus for decentralization
Update snapshot logic to include index
The index sadly can't be rebuilt from just the tree nodes, mainly because of the validFrom-validTo logic. If we ever confirm that historical queries (at a specific tree height) are not worth supporting, we can revisit this.
+156
-55
abciapp/snapshots.go
+156
-55
abciapp/snapshots.go
···
17
17
"sync"
18
18
"time"
19
19
20
+
dbm "github.com/cometbft/cometbft-db"
20
21
abcitypes "github.com/cometbft/cometbft/abci/types"
21
22
"github.com/cosmos/iavl"
22
23
"github.com/klauspost/compress/zstd"
23
24
"github.com/palantir/stacktrace"
25
+
"tangled.org/gbl08ma.com/didplcbft/store"
24
26
)
25
27
26
28
const snapshotChunkSize = 10 * 1024 * 1024 // 10 MB
···
124
126
125
127
// LoadSnapshotChunk implements [types.Application].
126
128
func (d *DIDPLCApplication) LoadSnapshotChunk(_ context.Context, req *abcitypes.RequestLoadSnapshotChunk) (*abcitypes.ResponseLoadSnapshotChunk, error) {
127
-
if req.Format != 1 {
129
+
if req.Format != 2 {
128
130
// just in case CometBFT asks us to load a chunk of a format we didn't declare to support in ListSnapshots...
129
131
return nil, stacktrace.NewError("unsupported snapshot format")
130
132
}
···
204
206
}
205
207
}
206
208
207
-
if req.Snapshot.Format != 1 {
209
+
if req.Snapshot.Format != 2 {
208
210
return &abcitypes.ResponseOfferSnapshot{
209
211
Result: abcitypes.ResponseOfferSnapshot_REJECT_FORMAT,
210
212
}, nil
···
244
246
245
247
st := time.Now()
246
248
247
-
err = writeSnapshot(f, it)
249
+
err = writeSnapshot(f, d.indexDB, it)
248
250
if err != nil {
249
251
return stacktrace.Propagate(err, "")
250
252
}
···
282
284
return nil
283
285
}
284
286
285
-
func writeSnapshot(writerSeeker io.WriteSeeker, it *iavl.ImmutableTree) error {
287
+
func writeSnapshot(writerSeeker io.WriteSeeker, indexDB dbm.DB, it *iavl.ImmutableTree) error {
286
288
writtenUntilReservedFields := 0
287
289
288
290
bw := bufio.NewWriter(writerSeeker)
···
294
296
}
295
297
writtenUntilReservedFields += c
296
298
297
-
c, err = bw.Write([]byte{0, 0, 0, 0, 0, 1})
299
+
c, err = bw.Write([]byte{0, 0, 0, 0, 0, 2})
298
300
if err != nil {
299
301
return stacktrace.Propagate(err, "")
300
302
}
···
314
316
}
315
317
writtenUntilReservedFields += c
316
318
317
-
// reserve space for writing number of bytes, number of nodes
318
-
// 8 bytes for node list size in bytes
319
-
// 8 bytes for number of nodes
320
-
sizeOfReservedFields := 8 + 8
319
+
// reserve space for writing:
320
+
// - 8 bytes for compressed section size in bytes
321
+
// - 8 bytes for number of index entries
322
+
// - 8 bytes for number of nodes
323
+
sizeOfReservedFields := 8 * 3
321
324
b = make([]byte, sizeOfReservedFields)
322
325
_, err = bw.Write(b)
323
326
if err != nil {
···
325
328
}
326
329
327
330
zstdw, err := zstd.NewWriter(bw, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
331
+
if err != nil {
332
+
return stacktrace.Propagate(err, "")
333
+
}
334
+
335
+
numIndexEntries, err := exportIndexEntries(indexDB, it.Version(), zstdw)
328
336
if err != nil {
329
337
return stacktrace.Propagate(err, "")
330
338
}
···
344
352
return stacktrace.Propagate(err, "")
345
353
}
346
354
347
-
// find total compressed node list file size
355
+
// find total compressed section size
348
356
offset, err := writerSeeker.Seek(0, io.SeekCurrent)
349
357
if err != nil {
350
358
return stacktrace.Propagate(err, "")
351
359
}
352
-
compressedNodeListSize := offset - int64(writtenUntilReservedFields) - int64(sizeOfReservedFields)
360
+
compressedSectionSize := offset - int64(writtenUntilReservedFields) - int64(sizeOfReservedFields)
353
361
354
362
// seek back and write empty header fields
355
363
···
362
370
}
363
371
364
372
b = make([]byte, sizeOfReservedFields)
365
-
binary.BigEndian.PutUint64(b, uint64(compressedNodeListSize))
366
-
binary.BigEndian.PutUint64(b[8:], uint64(numNodes))
373
+
binary.BigEndian.PutUint64(b, uint64(compressedSectionSize))
374
+
binary.BigEndian.PutUint64(b[8:], uint64(numIndexEntries))
375
+
binary.BigEndian.PutUint64(b[16:], uint64(numNodes))
367
376
_, err = writerSeeker.Write(b)
368
377
if err != nil {
369
378
return stacktrace.Propagate(err, "")
···
372
381
return nil
373
382
}
374
383
384
+
func exportIndexEntries(indexDB dbm.DB, treeVersion int64, w io.Writer) (int64, error) {
385
+
didLogKeyStart := make([]byte, store.DIDLogKeySize)
386
+
didLogKeyStart[0] = store.DIDLogKeyPrefix
387
+
didLogKeyEnd := slices.Repeat([]byte{0xff}, store.DIDLogKeySize)
388
+
didLogKeyEnd[0] = store.DIDLogKeyPrefix
389
+
390
+
iterator, err := indexDB.Iterator(didLogKeyStart, didLogKeyEnd)
391
+
if err != nil {
392
+
return 0, stacktrace.Propagate(err, "")
393
+
}
394
+
defer iterator.Close()
395
+
396
+
numEntries := int64(0)
397
+
for iterator.Valid() {
398
+
key := iterator.Key()
399
+
value := iterator.Value()
400
+
401
+
validFromHeight, validToHeight := store.UnmarshalDIDLogValue(value)
402
+
if uint64(treeVersion) >= validFromHeight && uint64(treeVersion) <= validToHeight {
403
+
header := make([]byte, 4+4)
404
+
binary.BigEndian.PutUint32(header, uint32(len(key)))
405
+
binary.BigEndian.PutUint32(header[4:], uint32(len(value)))
406
+
w.Write(header)
407
+
w.Write(key)
408
+
w.Write(value)
409
+
410
+
numEntries++
411
+
}
412
+
413
+
iterator.Next()
414
+
}
415
+
return numEntries, nil
416
+
}
417
+
375
418
func exportNodes(it *iavl.ImmutableTree, w io.Writer) (int64, error) {
376
419
exporter, err := it.Export()
377
420
if err != nil {
···
471
514
}
472
515
473
516
type snapshotApplier struct {
517
+
indexBatch dbm.Batch
474
518
tree *iavl.MutableTree
475
519
treeVersion int64
476
520
expectedFinalHash []byte
···
484
528
compressImporter iavl.NodeImporter
485
529
importerWg sync.WaitGroup
486
530
487
-
numImportedNodes int
488
-
claimedNodeCount int
489
-
done bool
531
+
numImportedNodes int
532
+
claimedNodeCount int
533
+
numImportedIndexEntries int
534
+
claimedIndexEntryCount int
535
+
done bool
490
536
}
491
537
492
538
var errMalformedChunk = errors.New("malformed chunk")
···
523
569
}
524
570
525
571
return &snapshotApplier{
572
+
indexBatch: d.indexDB.NewBatch(),
526
573
tree: d.tree,
527
574
treeVersion: treeVersion,
528
575
expectedFinalHash: expectedFinalHash,
···
547
594
}
548
595
549
596
if chunkIndex == 0 {
550
-
if len(chunkBytes) < 80 {
597
+
if len(chunkBytes) < 88 {
551
598
return stacktrace.Propagate(errMalformedChunk, "chunk too small")
552
599
}
553
600
···
555
602
return stacktrace.Propagate(errMalformedChunk, "invalid file magic")
556
603
}
557
604
558
-
if binary.BigEndian.Uint32(chunkBytes[20:]) != 1 {
605
+
if binary.BigEndian.Uint32(chunkBytes[20:]) != 2 {
559
606
return stacktrace.Propagate(errMalformedChunk, "invalid snapshot format")
560
607
}
561
608
···
567
614
return stacktrace.Propagate(errMalformedChunk, "mismatched declared tree hash")
568
615
}
569
616
570
-
declaredFileSize := 80 + binary.BigEndian.Uint64(chunkBytes[64:])
617
+
declaredFileSize := 88 + binary.BigEndian.Uint64(chunkBytes[64:])
571
618
minExpectedSize := uint64((len(a.expectedChunkHashes) - 1) * snapshotChunkSize)
572
619
maxExpectedSize := uint64(len(a.expectedChunkHashes) * snapshotChunkSize)
573
620
if declaredFileSize < minExpectedSize ||
574
621
declaredFileSize > maxExpectedSize {
575
-
return stacktrace.Propagate(errMalformedChunk, "unexpected compressed node list length")
622
+
return stacktrace.Propagate(errMalformedChunk, "unexpected compressed section length")
576
623
}
577
624
578
-
a.claimedNodeCount = int(binary.BigEndian.Uint64(chunkBytes[72:]))
625
+
a.claimedIndexEntryCount = int(binary.BigEndian.Uint64(chunkBytes[72:]))
626
+
a.claimedNodeCount = int(binary.BigEndian.Uint64(chunkBytes[80:]))
579
627
580
628
// move to the start of the compressed portion
581
-
chunkBytes = chunkBytes[80:]
629
+
chunkBytes = chunkBytes[88:]
582
630
583
631
a.importerWg.Go(a.streamingImporter)
584
632
}
···
602
650
// wait for importer to finish reading and importing everything
603
651
a.importerWg.Wait()
604
652
653
+
if a.numImportedIndexEntries != a.claimedIndexEntryCount {
654
+
return stacktrace.Propagate(errTreeHashMismatch, "imported index entry count mismatch")
655
+
}
656
+
605
657
if a.numImportedNodes != a.claimedNodeCount {
606
658
return stacktrace.Propagate(errTreeHashMismatch, "imported node count mismatch")
607
659
}
608
660
609
-
err := a.importer.Commit()
661
+
err := a.indexBatch.Write()
662
+
if err != nil {
663
+
return stacktrace.Propagate(err, "")
664
+
}
665
+
666
+
err = a.importer.Commit()
610
667
if err != nil {
611
668
if strings.Contains(err.Error(), "invalid node structure") {
612
669
return stacktrace.Propagate(errors.Join(errMalformedChunk, err), "")
···
627
684
628
685
func (a *snapshotApplier) streamingImporter() {
629
686
for {
630
-
nodeHeader := make([]byte, 9+4+4)
631
-
n, err := io.ReadFull(a.zstdReader, nodeHeader)
632
-
if err != nil || n != 9+4+4 {
633
-
// err may be EOF here, which is expected
634
-
return
635
-
}
687
+
if a.numImportedIndexEntries < a.claimedIndexEntryCount {
688
+
entryHeader := make([]byte, 4+4)
689
+
n, err := io.ReadFull(a.zstdReader, entryHeader)
690
+
if err != nil || n != 8 {
691
+
// err may be EOF here, which is expected
692
+
return
693
+
}
636
694
637
-
// validate lengths against sensible limits to prevent OOM DoS by malicious third parties
638
-
keyLength := binary.BigEndian.Uint32(nodeHeader[9:13])
639
-
var key []byte
640
-
if keyLength != 0xffffffff {
641
-
if keyLength > 1024*1024 {
695
+
// validate lengths against sensible limits to prevent OOM DoS by malicious third parties
696
+
keyLength := binary.BigEndian.Uint32(entryHeader[0:4])
697
+
valueLength := binary.BigEndian.Uint32(entryHeader[4:8])
698
+
if keyLength > 1024*1024 || valueLength > 1024*1024 {
642
699
return
643
700
}
644
-
key = make([]byte, keyLength)
701
+
702
+
key := make([]byte, keyLength)
645
703
646
704
n, err = io.ReadFull(a.zstdReader, key)
647
705
if err != nil || n != len(key) {
···
649
707
// we can return silently here because since we didn't import all nodes, the tree hash won't match anyway
650
708
return
651
709
}
652
-
}
710
+
711
+
value := make([]byte, valueLength)
712
+
n, err = io.ReadFull(a.zstdReader, value)
713
+
if err != nil || n != len(value) {
714
+
return
715
+
}
653
716
654
-
valueLength := binary.BigEndian.Uint32(nodeHeader[13:17])
655
-
var value []byte
656
-
if valueLength != 0xffffffff {
657
-
if valueLength > 1024*1024 {
717
+
err = a.indexBatch.Set(key, value)
718
+
if err != nil {
719
+
// we can return silently here because since we didn't import all nodes, the tree hash won't match anyway
658
720
return
659
721
}
660
-
value = make([]byte, valueLength)
661
-
n, err = io.ReadFull(a.zstdReader, value)
662
-
if err != nil || n != len(value) {
722
+
723
+
a.numImportedIndexEntries++
724
+
} else {
725
+
nodeHeader := make([]byte, 9+4+4)
726
+
n, err := io.ReadFull(a.zstdReader, nodeHeader)
727
+
if err != nil || n != 9+4+4 {
728
+
// err may be EOF here, which is expected
663
729
return
664
730
}
665
-
}
731
+
732
+
// validate lengths against sensible limits to prevent OOM DoS by malicious third parties
733
+
keyLength := binary.BigEndian.Uint32(nodeHeader[9:13])
734
+
var key []byte
735
+
if keyLength != 0xffffffff {
736
+
if keyLength > 1024*1024 {
737
+
return
738
+
}
739
+
key = make([]byte, keyLength)
666
740
667
-
err = a.compressImporter.Add(&iavl.ExportNode{
668
-
Height: int8(nodeHeader[0]),
669
-
Version: int64(binary.BigEndian.Uint64(nodeHeader[1:9])),
670
-
Key: key,
671
-
Value: value,
672
-
})
673
-
if err != nil {
674
-
// this shouldn't happen unless the data is corrupt
675
-
// we can return silently here because since we didn't import all nodes, the tree hash won't match anyway
676
-
return
741
+
n, err = io.ReadFull(a.zstdReader, key)
742
+
if err != nil || n != len(key) {
743
+
// this shouldn't happen unless the data is corrupt
744
+
// we can return silently here because since we didn't import all nodes, the tree hash won't match anyway
745
+
return
746
+
}
747
+
}
748
+
749
+
valueLength := binary.BigEndian.Uint32(nodeHeader[13:17])
750
+
var value []byte
751
+
if valueLength != 0xffffffff {
752
+
if valueLength > 1024*1024 {
753
+
return
754
+
}
755
+
value = make([]byte, valueLength)
756
+
n, err = io.ReadFull(a.zstdReader, value)
757
+
if err != nil || n != len(value) {
758
+
return
759
+
}
760
+
}
761
+
762
+
err = a.compressImporter.Add(&iavl.ExportNode{
763
+
Height: int8(nodeHeader[0]),
764
+
Version: int64(binary.BigEndian.Uint64(nodeHeader[1:9])),
765
+
Key: key,
766
+
Value: value,
767
+
})
768
+
if err != nil {
769
+
// this shouldn't happen unless the data is corrupt
770
+
// we can return silently here because since we didn't import all nodes, the tree hash won't match anyway
771
+
return
772
+
}
773
+
a.numImportedNodes++
677
774
}
678
-
a.numImportedNodes++
679
775
}
680
776
}
681
777
···
705
801
}
706
802
707
803
err = a.pipeWriter.Close()
804
+
if err != nil {
805
+
return stacktrace.Propagate(err, "")
806
+
}
807
+
808
+
err = a.indexBatch.Close()
708
809
if err != nil {
709
810
return stacktrace.Propagate(err, "")
710
811
}
+9
-6
store/tree.go
+9
-6
store/tree.go
···
73
73
}
74
74
75
75
sequence := unmarshalDIDLogSequence(didLogIterator.Key())
76
-
validFromHeight, validToHeight := unmarshalDIDLogValue(didLogIterator.Value())
76
+
validFromHeight, validToHeight := UnmarshalDIDLogValue(didLogIterator.Value())
77
77
78
78
opKey := marshalOperationKey(sequence)
79
79
···
146
146
}
147
147
148
148
sequence := unmarshalDIDLogSequence(didLogIterator.Key())
149
-
validFromHeight, validToHeight := unmarshalDIDLogValue(didLogIterator.Value())
149
+
validFromHeight, validToHeight := UnmarshalDIDLogValue(didLogIterator.Value())
150
150
151
151
opKey := marshalOperationKey(sequence)
152
152
···
245
245
}
246
246
247
247
sequence := unmarshalDIDLogSequence(didLogIterator.Key())
248
-
validFromHeight, validToHeight := unmarshalDIDLogValue(didLogIterator.Value())
248
+
validFromHeight, validToHeight := UnmarshalDIDLogValue(didLogIterator.Value())
249
249
250
250
opKey := marshalOperationKey(sequence)
251
251
···
390
390
return did, nil
391
391
}
392
392
393
+
const DIDLogKeySize = 1 + 15 + 8
394
+
const DIDLogKeyPrefix = 'l'
395
+
393
396
func marshalDIDLogKey(didBytes []byte, sequence uint64) []byte {
394
-
key := make([]byte, 1+15+8)
395
-
key[0] = 'l'
397
+
key := make([]byte, DIDLogKeySize)
398
+
key[0] = DIDLogKeyPrefix
396
399
copy(key[1:16], didBytes)
397
400
binary.BigEndian.PutUint64(key[16:], sequence)
398
401
return key
···
410
413
return value
411
414
}
412
415
413
-
func unmarshalDIDLogValue(value []byte) (validFromHeight, validToHeight uint64) {
416
+
func UnmarshalDIDLogValue(value []byte) (validFromHeight, validToHeight uint64) {
414
417
validFromHeight = binary.BigEndian.Uint64(value[0:8])
415
418
validToHeight = binary.BigEndian.Uint64(value[8:16])
416
419
return