atscan.net
[DEPRECATED] Go implementation of plcbundle
tree.fail
1package didindex
2
3import (
4 "sync"
5 "time"
6
7 "tangled.org/atscan.net/plcbundle/internal/plcclient"
8)
9
10const (
11 DID_INDEX_DIR = ".plcbundle"
12 DID_INDEX_SHARDS = "shards"
13 DID_INDEX_CONFIG = "config.json"
14 DID_SHARD_COUNT = 256
15 DID_PREFIX = "did:plc:"
16 DID_IDENTIFIER_LEN = 24 // Without "did:plc:" prefix
17
18 // Binary format constants
19 DIDINDEX_MAGIC = "PLCD"
20 DIDINDEX_VERSION = 4
21
22 // Format sizes
23 LOCATION_SIZE_V3 = 5 // Old: 2+2+1
24 LOCATION_SIZE_V4 = 4 // New: packed uint32
25
26 BUILD_BATCH_SIZE = 100 // Process 100 bundles at a time
27
28 PREFIX_INDEX_SIZE = 256 // Index first byte of identifier (0x00-0xFF)
29)
30
31// Manager manages sharded DID position indexes with mmap
32type Manager struct {
33 baseDir string
34 indexDir string
35 shardDir string
36 configPath string
37
38 // LRU cache for hot shards
39 shardCache sync.Map
40 maxCache int
41 evictionThreshold int
42 cacheHits int64 // atomic counter
43 cacheMisses int64 // atomic counter
44
45 config *Config
46 logger Logger
47 verbose bool
48
49 indexMu sync.RWMutex
50
51 // Performance tracking
52 totalLookups int64 // Total number of lookups
53 totalLookupTime int64 // Total time in microseconds
54 lookupTimeLock sync.Mutex
55 recentLookups []int64 // Circular buffer for recent lookup times (microseconds)
56 recentLookupIdx int
57 recentLookupSize int
58}
59
60// mmapShard represents a memory-mapped shard file
61type mmapShard struct {
62 shardNum uint8
63 data []byte
64 file interface{} // *os.File (avoid import)
65 lastUsed int64
66 accessCount int64
67 refCount int64
68}
69
70// Config stores index metadata
71type Config struct {
72 Version int `json:"version"`
73 Format string `json:"format"`
74 ShardCount int `json:"shard_count"`
75 TotalDIDs int64 `json:"total_dids"`
76 UpdatedAt time.Time `json:"updated_at"`
77 LastBundle int `json:"last_bundle"`
78}
79
80// ShardBuilder accumulates DID positions for a shard
81type ShardBuilder struct {
82 entries map[string][]OpLocation
83 mu sync.Mutex
84}
85
86// OpLocation represents exact location of an operation
87type OpLocation uint32
88
89// OpLocationWithOperation contains an operation with its bundle/position
90type OpLocationWithOperation struct {
91 Operation plcclient.PLCOperation
92 Bundle int
93 Position int
94}
95
96func NewOpLocation(bundle, position uint16, nullified bool) OpLocation {
97 globalPos := uint32(bundle)*10000 + uint32(position)
98 loc := globalPos << 1
99 if nullified {
100 loc |= 1
101 }
102 return OpLocation(loc)
103}
104
105// Getters
106func (loc OpLocation) GlobalPosition() uint32 {
107 return uint32(loc) >> 1
108}
109
110func (loc OpLocation) Bundle() uint16 {
111 return uint16(loc.GlobalPosition() / 10000)
112}
113
114func (loc OpLocation) Position() uint16 {
115 return uint16(loc.GlobalPosition() % 10000)
116}
117
118func (loc OpLocation) Nullified() bool {
119 return (loc & 1) == 1
120}
121
122func (loc OpLocation) IsAfter(other OpLocation) bool {
123 // Compare global positions directly
124 return loc.GlobalPosition() > other.GlobalPosition()
125}
126
127func (loc OpLocation) IsBefore(other OpLocation) bool {
128 return loc.GlobalPosition() < other.GlobalPosition()
129}
130
131func (loc OpLocation) Equals(other OpLocation) bool {
132 // Compare entire packed value (including nullified bit)
133 return loc == other
134}
135
136func (loc OpLocation) PositionEquals(other OpLocation) bool {
137 // Compare only position (ignore nullified bit)
138 return loc.GlobalPosition() == other.GlobalPosition()
139}
140
141// Convenience conversions
142func (loc OpLocation) BundleInt() int {
143 return int(loc.Bundle())
144}
145
146func (loc OpLocation) PositionInt() int {
147 return int(loc.Position())
148}
149
150// For sorting/comparison
151func (loc OpLocation) Less(other OpLocation) bool {
152 return loc.GlobalPosition() < other.GlobalPosition()
153}