gbl08ma.com
A very experimental PLC implementation which uses BFT consensus for decentralization
Compress keys outside of iavl tree for better guarantees of determinism
+6
-1
abciapp/app.go
+6
-1
abciapp/app.go
···
11
dbm "github.com/cometbft/cometbft-db"
12
abcitypes "github.com/cometbft/cometbft/abci/types"
13
"github.com/cosmos/iavl"
14
"github.com/palantir/stacktrace"
15
"github.com/samber/lo"
16
"tangled.org/gbl08ma.com/didplcbft/dbadapter"
17
"tangled.org/gbl08ma.com/didplcbft/plc"
18
"tangled.org/gbl08ma.com/didplcbft/store"
19
"tangled.org/gbl08ma.com/didplcbft/transaction"
···
42
// store and plc must be able to share transaction objects
43
func NewDIDPLCApplication(treeDB dbm.DB, indexDB dbm.DB, clearData func(), snapshotDirectory string) (*DIDPLCApplication, *transaction.Factory, plc.PLC, func(), error) {
44
mkTree := func() *iavl.MutableTree {
45
-
return iavl.NewMutableTree(dbadapter.Adapt(treeDB), 500000, false, iavl.NewNopLogger(), iavl.AsyncPruningOption(false))
46
}
47
48
tree := mkTree()
···
11
dbm "github.com/cometbft/cometbft-db"
12
abcitypes "github.com/cometbft/cometbft/abci/types"
13
"github.com/cosmos/iavl"
14
+
"github.com/klauspost/compress/zstd"
15
"github.com/palantir/stacktrace"
16
"github.com/samber/lo"
17
"tangled.org/gbl08ma.com/didplcbft/dbadapter"
18
+
"tangled.org/gbl08ma.com/didplcbft/dbadapter/zstddict"
19
"tangled.org/gbl08ma.com/didplcbft/plc"
20
"tangled.org/gbl08ma.com/didplcbft/store"
21
"tangled.org/gbl08ma.com/didplcbft/transaction"
···
44
// store and plc must be able to share transaction objects
45
func NewDIDPLCApplication(treeDB dbm.DB, indexDB dbm.DB, clearData func(), snapshotDirectory string) (*DIDPLCApplication, *transaction.Factory, plc.PLC, func(), error) {
46
mkTree := func() *iavl.MutableTree {
47
+
// Using SpeedDefault appears to cause the processing time for ExecuteOperation to double on average
48
+
// Using SpeedBetterCompression appears to cause the processing time to double again
49
+
// By using SpeedFastest we seem to give up on like 5% size reduction, it's not worth using the slower speeds
50
+
return iavl.NewMutableTree(dbadapter.AdaptWithCompression(treeDB, zstd.SpeedFastest, zstddict.PLCZstdDict), 500000, false, iavl.NewNopLogger(), iavl.AsyncPruningOption(false))
51
}
52
53
tree := mkTree()
+102
-7
dbadapter/adapter.go
+102
-7
dbadapter/adapter.go
···
4
"cosmossdk.io/core/store"
5
dbm "github.com/cometbft/cometbft-db"
6
iavldbm "github.com/cosmos/iavl/db"
7
)
8
9
type AdaptedDB struct {
10
underlying dbm.DB
11
}
12
13
func Adapt(underlying dbm.DB) *AdaptedDB {
···
16
}
17
}
18
19
var _ iavldbm.DB = (*AdaptedDB)(nil)
20
21
// Close implements [iavldbm.DB].
···
25
26
// Get implements [iavldbm.DB].
27
func (b *AdaptedDB) Get(key []byte) ([]byte, error) {
28
-
return b.underlying.Get(key)
29
}
30
31
// Has implements [iavldbm.DB].
···
35
36
// AdaptedIterator adapts badger.Iterator to store.Iterator
37
type AdaptedIterator struct {
38
underlying dbm.Iterator
39
calledNextOnce bool
40
}
···
60
}
61
62
func (i *AdaptedIterator) Value() []byte {
63
-
return i.underlying.Value()
64
}
65
66
func (i *AdaptedIterator) Error() error {
···
77
if err != nil {
78
return nil, err
79
}
80
-
return &AdaptedIterator{underlying: i}, nil
81
}
82
83
// ReverseIterator implements [iavldbm.DB].
···
86
if err != nil {
87
return nil, err
88
}
89
-
return &AdaptedIterator{underlying: i}, nil
90
}
91
92
// NewBatch implements [db.DB].
93
func (b *AdaptedDB) NewBatch() store.Batch {
94
-
return &AdaptedBatch{b.underlying.NewBatch()}
95
}
96
97
// NewBatchWithSize implements [db.DB].
98
func (b *AdaptedDB) NewBatchWithSize(int) store.Batch {
99
-
return &AdaptedBatch{b.underlying.NewBatch()}
100
}
101
102
type AdaptedBatch struct {
103
-
dbm.Batch
104
}
105
106
// GetByteSize implements [store.Batch].
107
func (a *AdaptedBatch) GetByteSize() (int, error) {
108
return 0, nil
109
}
···
4
"cosmossdk.io/core/store"
5
dbm "github.com/cometbft/cometbft-db"
6
iavldbm "github.com/cosmos/iavl/db"
7
+
"github.com/klauspost/compress/zstd"
8
+
"github.com/palantir/stacktrace"
9
)
10
11
type AdaptedDB struct {
12
underlying dbm.DB
13
+
14
+
// these two may be nil when not compressing:
15
+
zstdEncoder *zstd.Encoder
16
+
zstdDecoder *zstd.Decoder
17
}
18
19
func Adapt(underlying dbm.DB) *AdaptedDB {
···
22
}
23
}
24
25
+
func AdaptWithCompression(underlying dbm.DB, level zstd.EncoderLevel, dict []byte) *AdaptedDB {
26
+
zstdEncoder, _ := zstd.NewWriter(nil, zstd.WithEncoderDict(dict), zstd.WithEncoderLevel(level))
27
+
zstdDecoder, _ := zstd.NewReader(nil, zstd.WithDecoderDicts(dict))
28
+
29
+
return &AdaptedDB{
30
+
underlying: underlying,
31
+
zstdEncoder: zstdEncoder,
32
+
zstdDecoder: zstdDecoder,
33
+
}
34
+
}
35
+
36
var _ iavldbm.DB = (*AdaptedDB)(nil)
37
38
// Close implements [iavldbm.DB].
···
42
43
// Get implements [iavldbm.DB].
44
func (b *AdaptedDB) Get(key []byte) ([]byte, error) {
45
+
v, err := b.underlying.Get(key)
46
+
if err != nil {
47
+
return nil, stacktrace.Propagate(err, "")
48
+
}
49
+
v, err = decompressValue(b.zstdDecoder, v)
50
+
return v, stacktrace.Propagate(err, "")
51
}
52
53
// Has implements [iavldbm.DB].
···
57
58
// AdaptedIterator adapts badger.Iterator to store.Iterator
59
type AdaptedIterator struct {
60
+
zstdDecoder *zstd.Decoder
61
underlying dbm.Iterator
62
calledNextOnce bool
63
}
···
83
}
84
85
func (i *AdaptedIterator) Value() []byte {
86
+
v, _ := decompressValue(i.zstdDecoder, i.underlying.Value())
87
+
return v
88
}
89
90
func (i *AdaptedIterator) Error() error {
···
101
if err != nil {
102
return nil, err
103
}
104
+
return &AdaptedIterator{underlying: i, zstdDecoder: b.zstdDecoder}, nil
105
}
106
107
// ReverseIterator implements [iavldbm.DB].
···
110
if err != nil {
111
return nil, err
112
}
113
+
return &AdaptedIterator{underlying: i, zstdDecoder: b.zstdDecoder}, nil
114
}
115
116
// NewBatch implements [db.DB].
117
func (b *AdaptedDB) NewBatch() store.Batch {
118
+
return &AdaptedBatch{
119
+
underlying: b.underlying.NewBatch(),
120
+
zstdEncoder: b.zstdEncoder,
121
+
}
122
}
123
124
// NewBatchWithSize implements [db.DB].
125
func (b *AdaptedDB) NewBatchWithSize(int) store.Batch {
126
+
return b.NewBatch()
127
}
128
129
type AdaptedBatch struct {
130
+
underlying dbm.Batch
131
+
zstdEncoder *zstd.Encoder
132
+
}
133
+
134
+
// Close implements [store.Batch].
135
+
func (a *AdaptedBatch) Close() error {
136
+
return a.underlying.Close()
137
+
}
138
+
139
+
// Delete implements [store.Batch].
140
+
func (a *AdaptedBatch) Delete(key []byte) error {
141
+
return a.underlying.Delete(key)
142
+
}
143
+
144
+
// Set implements [store.Batch].
145
+
func (a *AdaptedBatch) Set(key []byte, value []byte) error {
146
+
v := compressValue(a.zstdEncoder, value)
147
+
return stacktrace.Propagate(a.underlying.Set(key, v), "")
148
+
}
149
+
150
+
// Write implements [store.Batch].
151
+
func (a *AdaptedBatch) Write() error {
152
+
return a.underlying.Write()
153
+
}
154
+
155
+
// WriteSync implements [store.Batch].
156
+
func (a *AdaptedBatch) WriteSync() error {
157
+
return a.underlying.WriteSync()
158
}
159
160
// GetByteSize implements [store.Batch].
161
func (a *AdaptedBatch) GetByteSize() (int, error) {
162
return 0, nil
163
}
164
+
165
+
func compressValue(encoder *zstd.Encoder, value []byte) []byte {
166
+
if encoder == nil {
167
+
return value
168
+
}
169
+
if len(value) < 192 {
170
+
// this is probably a inner node of the iavl tree and we don't gain anything from compressing those 50-ish byte values
171
+
return prepend(value, 0x00)
172
+
}
173
+
buf := make([]byte, 0, len(value)+5) // a bit of an extra buffer because, rarely, the value increases in size and this way we save on one reallocation
174
+
return prepend(encoder.EncodeAll(value, buf), 0x01)
175
+
}
176
+
177
+
func decompressValue(decoder *zstd.Decoder, value []byte) ([]byte, error) {
178
+
if decoder == nil || len(value) == 0 {
179
+
return value, nil
180
+
} else if value[0] == 0x00 {
181
+
return value[1:], nil
182
+
}
183
+
// passing a nil output buffer to DecodeAll means it'll optimistically start by allocating len(value)*2
184
+
// but we observe compression ratios better than 50% frequently, so we allocate a slice ourselves with cap len(value)*3
185
+
value, err := decoder.DecodeAll(value[1:], make([]byte, 0, len(value)*3))
186
+
return value, stacktrace.Propagate(err, "")
187
+
}
188
+
189
+
// this is a simplified version of slices.Insert for prepending a single element to a slice, returning the modified slice
190
+
func prepend[S ~[]E, E any](s S, v E) S {
191
+
n := len(s)
192
+
193
+
if n >= cap(s) {
194
+
s2 := make(S, n+1)
195
+
s2[0] = v
196
+
copy(s2[1:], s)
197
+
return s2
198
+
}
199
+
s = s[:n+1]
200
+
copy(s[1:], s)
201
+
s[0] = v
202
+
203
+
return s
204
+
}
dbadapter/zstddict/plcvalues
dbadapter/zstddict/plcvalues
This is a binary file and will not be displayed.
+6
dbadapter/zstddict/plcvalues.go
+6
dbadapter/zstddict/plcvalues.go
-2
plc/impl.go
-2
plc/impl.go
+6
-32
store/tree.go
+6
-32
store/tree.go
···
15
ics23 "github.com/cosmos/ics23/go"
16
"github.com/did-method-plc/go-didplc"
17
cbornode "github.com/ipfs/go-ipld-cbor"
18
-
"github.com/klauspost/compress/zstd"
19
"github.com/palantir/stacktrace"
20
"github.com/polydawn/refmt/obj/atlas"
21
"github.com/samber/lo"
···
49
50
// TreeStore exists just to groups methods nicely
51
type TreeStore struct{}
52
-
53
-
//go:embed zstddict/plcops
54
-
var plcOpsZstdDict []byte
55
-
56
-
// Using SpeedDefault appears to cause the processing time for ExecuteOperation to double on average
57
-
// Using SpeedBetterCompression appears to cause the processing time to double again
58
-
// By using SpeedFastest we seem to give up on like 5% size reduction, it's not worth using the slower speeds
59
-
var zstdOpEncoder, _ = zstd.NewWriter(nil, zstd.WithEncoderDict(plcOpsZstdDict), zstd.WithEncoderLevel(zstd.SpeedFastest))
60
-
var zstdOpDecoder, _ = zstd.NewReader(nil, zstd.WithDecoderDicts(plcOpsZstdDict))
61
62
func (t *TreeStore) ProduceOperationExamples(tx transaction.Read, interval, count int) iter.Seq[[]byte] {
63
return func(yield func([]byte) bool) {
···
290
if err != nil {
291
return stacktrace.Propagate(err, "")
292
}
293
-
operationValue, err = markCompressedOperationValueNullified(operationValue)
294
-
if err != nil {
295
-
return stacktrace.Propagate(err, "")
296
-
}
297
298
-
updated, err := tx.Tree().Set(opKey, operationValue)
299
if err != nil {
300
return stacktrace.Propagate(err, "")
301
}
···
477
binary.BigEndian.PutUint64(o[16:24], ts)
478
copy(o[24:], opAsBytes)
479
480
-
return zstdOpEncoder.EncodeAll(o, make([]byte, 0, len(o)))
481
}
482
483
func unmarshalOperationValue(value []byte) (bool, string, time.Time, didplc.OpEnum, error) {
484
-
// passing a nil output buffer to DecodeAll means it'll optimistically start by allocating len(value)*2
485
-
// but we observe compression ratios better than 50% sometimes, so allocate len(value)*3 instead
486
-
value, err := zstdOpDecoder.DecodeAll(value, make([]byte, 0, len(value)*3))
487
-
if err != nil {
488
-
return false, "", time.Time{}, didplc.OpEnum{}, stacktrace.Propagate(err, "")
489
-
}
490
-
491
nullified := value[0] != 0
492
493
did, err := bytesToDID(value[1:16])
···
506
return nullified, did, createdAt, opEnum, nil
507
}
508
509
-
func markCompressedOperationValueNullified(value []byte) ([]byte, error) {
510
-
value, err := zstdOpDecoder.DecodeAll(value, make([]byte, 0, len(value)*3))
511
-
if err != nil {
512
-
return nil, stacktrace.Propagate(err, "")
513
-
}
514
-
515
-
value[0] = 1
516
-
517
-
return zstdOpEncoder.EncodeAll(value, make([]byte, 0, len(value))), nil
518
}
519
520
func unmarshalLogEntry(operationKey, operationValue []byte) (types.SequencedLogEntry, error) {
···
15
ics23 "github.com/cosmos/ics23/go"
16
"github.com/did-method-plc/go-didplc"
17
cbornode "github.com/ipfs/go-ipld-cbor"
18
"github.com/palantir/stacktrace"
19
"github.com/polydawn/refmt/obj/atlas"
20
"github.com/samber/lo"
···
48
49
// TreeStore exists just to groups methods nicely
50
type TreeStore struct{}
51
52
func (t *TreeStore) ProduceOperationExamples(tx transaction.Read, interval, count int) iter.Seq[[]byte] {
53
return func(yield func([]byte) bool) {
···
280
if err != nil {
281
return stacktrace.Propagate(err, "")
282
}
283
284
+
updated, err := tx.Tree().Set(opKey, markOperationValueNullified(operationValue))
285
if err != nil {
286
return stacktrace.Propagate(err, "")
287
}
···
463
binary.BigEndian.PutUint64(o[16:24], ts)
464
copy(o[24:], opAsBytes)
465
466
+
return o
467
}
468
469
func unmarshalOperationValue(value []byte) (bool, string, time.Time, didplc.OpEnum, error) {
470
nullified := value[0] != 0
471
472
did, err := bytesToDID(value[1:16])
···
485
return nullified, did, createdAt, opEnum, nil
486
}
487
488
+
func markOperationValueNullified(value []byte) []byte {
489
+
v := slices.Clone(value)
490
+
v[0] = 1
491
+
return v
492
}
493
494
func unmarshalLogEntry(operationKey, operationValue []byte) (types.SequencedLogEntry, error) {
store/zstddict/plcops
store/zstddict/plcops
This is a binary file and will not be displayed.