+19 -17

bundle/bundle_test.go internal/bundle/bundle_test.go

··· 5 5 "testing" 6 6 "time" 7 7 8 8 - "tangled.org/atscan.net/plcbundle/bundle" 8 8 + "tangled.org/atscan.net/plcbundle/internal/bundle" 9 9 + "tangled.org/atscan.net/plcbundle/internal/mempool" 9 10 "tangled.org/atscan.net/plcbundle/internal/storage" 11 11 + "tangled.org/atscan.net/plcbundle/internal/types" 10 12 "tangled.org/atscan.net/plcbundle/plcclient" 11 13 ) 12 14 ··· 17 19 if idx == nil { 18 20 t.Fatal("NewIndex returned nil") 19 21 } 20 20 - if idx.Version != bundle.INDEX_VERSION { 21 21 - t.Errorf("expected version %s, got %s", bundle.INDEX_VERSION, idx.Version) 22 22 + if idx.Version != types.INDEX_VERSION { 23 23 + t.Errorf("expected version %s, got %s", types.INDEX_VERSION, idx.Version) 22 24 } 23 25 if idx.Count() != 0 { 24 26 t.Errorf("expected empty index, got count %d", idx.Count()) ··· 31 33 BundleNumber: 1, 32 34 StartTime: time.Now(), 33 35 EndTime: time.Now().Add(time.Hour), 34 34 - OperationCount: bundle.BUNDLE_SIZE, 36 36 + OperationCount: types.BUNDLE_SIZE, 35 37 DIDCount: 1000, 36 38 Hash: "abc123", 37 39 CompressedHash: "def456", ··· 62 64 BundleNumber: 1, 63 65 StartTime: time.Now(), 64 66 EndTime: time.Now().Add(time.Hour), 65 65 - OperationCount: bundle.BUNDLE_SIZE, 67 67 + OperationCount: types.BUNDLE_SIZE, 66 68 Hash: "test123", 67 69 }) 68 70 ··· 88 90 BundleNumber: i, 89 91 StartTime: time.Now(), 90 92 EndTime: time.Now().Add(time.Hour), 91 91 - OperationCount: bundle.BUNDLE_SIZE, 93 93 + OperationCount: types.BUNDLE_SIZE, 92 94 }) 93 95 } 94 96 ··· 109 111 BundleNumber: num, 110 112 StartTime: time.Now(), 111 113 EndTime: time.Now().Add(time.Hour), 112 112 - OperationCount: bundle.BUNDLE_SIZE, 114 114 + OperationCount: types.BUNDLE_SIZE, 113 115 }) 114 116 } 115 117 ··· 137 139 BundleNumber: 1, 138 140 StartTime: time.Now(), 139 141 EndTime: time.Now().Add(time.Hour), 140 140 - Operations: makeTestOperations(bundle.BUNDLE_SIZE), 142 142 + Operations: makeTestOperations(types.BUNDLE_SIZE), 141 143 }, 142 144 wantErr: false, 143 145 }, ··· 145 147 name: "invalid bundle number", 146 148 bundle: &bundle.Bundle{ 147 149 BundleNumber: 0, 148 148 - Operations: makeTestOperations(bundle.BUNDLE_SIZE), 150 150 + Operations: makeTestOperations(types.BUNDLE_SIZE), 149 151 }, 150 152 wantErr: true, 151 153 }, ··· 163 165 BundleNumber: 1, 164 166 StartTime: time.Now().Add(time.Hour), 165 167 EndTime: time.Now(), 166 166 - Operations: makeTestOperations(bundle.BUNDLE_SIZE), 168 168 + Operations: makeTestOperations(types.BUNDLE_SIZE), 167 169 }, 168 170 wantErr: true, 169 171 }, ··· 198 200 199 201 t.Run("CreateAndAdd", func(t *testing.T) { 200 202 minTime := time.Now().Add(-time.Hour) 201 201 - m, err := bundle.NewMempool(tmpDir, 1, minTime, logger) 203 203 + m, err := mempool.NewMempool(tmpDir, 1, minTime, logger) 202 204 if err != nil { 203 205 t.Fatalf("NewMempool failed: %v", err) 204 206 } ··· 218 220 219 221 t.Run("ChronologicalValidation", func(t *testing.T) { 220 222 minTime := time.Now().Add(-time.Hour) 221 221 - m, err := bundle.NewMempool(tmpDir, 2, minTime, logger) 223 223 + m, err := mempool.NewMempool(tmpDir, 2, minTime, logger) 222 224 if err != nil { 223 225 t.Fatalf("NewMempool failed: %v", err) 224 226 } ··· 246 248 247 249 t.Run("TakeOperations", func(t *testing.T) { 248 250 minTime := time.Now().Add(-time.Hour) 249 249 - m, err := bundle.NewMempool(tmpDir, 3, minTime, logger) 251 251 + m, err := mempool.NewMempool(tmpDir, 3, minTime, logger) 250 252 if err != nil { 251 253 t.Fatalf("NewMempool failed: %v", err) 252 254 } ··· 268 270 269 271 t.Run("SaveAndLoad", func(t *testing.T) { 270 272 minTime := time.Now().Add(-time.Hour) 271 271 - m, err := bundle.NewMempool(tmpDir, 4, minTime, logger) 273 273 + m, err := mempool.NewMempool(tmpDir, 4, minTime, logger) 272 274 if err != nil { 273 275 t.Fatalf("NewMempool failed: %v", err) 274 276 } ··· 281 283 } 282 284 283 285 // Create new mempool and load 284 284 - m2, err := bundle.NewMempool(tmpDir, 4, minTime, logger) 286 286 + m2, err := mempool.NewMempool(tmpDir, 4, minTime, logger) 285 287 if err != nil { 286 288 t.Fatalf("NewMempool failed: %v", err) 287 289 } ··· 293 295 294 296 t.Run("Validate", func(t *testing.T) { 295 297 minTime := time.Now().Add(-time.Hour) 296 296 - m, err := bundle.NewMempool(tmpDir, 5, minTime, logger) 298 298 + m, err := mempool.NewMempool(tmpDir, 5, minTime, logger) 297 299 if err != nil { 298 300 t.Fatalf("NewMempool failed: %v", err) 299 301 } ··· 341 343 }) 342 344 343 345 t.Run("SaveAndLoadBundle", func(t *testing.T) { 344 344 - operations := makeTestOperations(bundle.BUNDLE_SIZE) 346 346 + operations := makeTestOperations(types.BUNDLE_SIZE) 345 347 path := filepath.Join(tmpDir, "test_bundle.jsonl.zst") 346 348 347 349 // Save

+3 -2

bundle/bundler.go internal/bundle/bundler.go

··· 3 3 import ( 4 4 "time" 5 5 6 6 + "tangled.org/atscan.net/plcbundle/internal/types" 6 7 "tangled.org/atscan.net/plcbundle/plcclient" 7 8 ) 8 9 9 10 // CreateBundle creates a complete bundle structure from operations 10 11 func (m *Manager) CreateBundle(bundleNumber int, operations []plcclient.PLCOperation, cursor string, parent string) *Bundle { 11 11 - if len(operations) != BUNDLE_SIZE { 12 12 - m.logger.Printf("Warning: bundle has %d operations, expected %d", len(operations), BUNDLE_SIZE) 12 12 + if len(operations) != types.BUNDLE_SIZE { 13 13 + m.logger.Printf("Warning: bundle has %d operations, expected %d", len(operations), types.BUNDLE_SIZE) 13 14 } 14 15 15 16 dids := m.operations.ExtractUniqueDIDs(operations)

bundle/clone.go internal/bundle/clone.go

bundle/index.go internal/bundle/index.go

+18 -16

bundle/manager.go internal/bundle/manager.go

··· 15 15 "time" 16 16 17 17 "tangled.org/atscan.net/plcbundle/internal/didindex" 18 18 + "tangled.org/atscan.net/plcbundle/internal/mempool" 18 19 "tangled.org/atscan.net/plcbundle/internal/storage" 20 20 + "tangled.org/atscan.net/plcbundle/internal/types" 19 21 "tangled.org/atscan.net/plcbundle/plcclient" 20 22 ) 21 23 ··· 37 39 index *Index 38 40 indexPath string 39 41 plcClient *plcclient.Client 40 40 - logger Logger 41 41 - mempool *Mempool 42 42 + logger types.Logger 43 43 + mempool *mempool.Mempool 42 44 didIndex *didindex.Manager // Updated type 43 45 44 46 bundleCache map[int]*Bundle ··· 261 263 minTimestamp = lastBundle.EndTime 262 264 } 263 265 264 264 - mempool, err := NewMempool(config.BundleDir, nextBundleNum, minTimestamp, config.Logger) 266 266 + mempool, err := mempool.NewMempool(config.BundleDir, nextBundleNum, minTimestamp, config.Logger) 265 267 if err != nil { 266 268 return nil, fmt.Errorf("failed to initialize mempool: %w", err) 267 269 } ··· 450 452 nextBundle := bundle.BundleNumber + 1 451 453 minTimestamp := bundle.EndTime 452 454 453 453 - newMempool, err := NewMempool(m.config.BundleDir, nextBundle, minTimestamp, m.logger) 455 455 + newMempool, err := mempool.NewMempool(m.config.BundleDir, nextBundle, minTimestamp, m.logger) 454 456 if err != nil { 455 457 return fmt.Errorf("failed to create new mempool: %w", err) 456 458 } ··· 494 496 m.logger.Printf("Preparing bundle %06d (mempool: %d ops)...", nextBundleNum, m.mempool.Count()) 495 497 } 496 498 497 497 - for m.mempool.Count() < BUNDLE_SIZE { 499 499 + for m.mempool.Count() < types.BUNDLE_SIZE { 498 500 if !quiet { 499 499 - m.logger.Printf("Fetching more operations (have %d/%d)...", m.mempool.Count(), BUNDLE_SIZE) 501 501 + m.logger.Printf("Fetching more operations (have %d/%d)...", m.mempool.Count(), types.BUNDLE_SIZE) 500 502 } 501 503 502 502 - err := m.fetchToMempool(ctx, afterTime, prevBoundaryCIDs, BUNDLE_SIZE-m.mempool.Count(), quiet) 504 504 + err := m.fetchToMempool(ctx, afterTime, prevBoundaryCIDs, types.BUNDLE_SIZE-m.mempool.Count(), quiet) 503 505 if err != nil { 504 504 - if m.mempool.Count() >= BUNDLE_SIZE { 506 506 + if m.mempool.Count() >= types.BUNDLE_SIZE { 505 507 break 506 508 } 507 509 m.mempool.Save() 508 508 - return nil, fmt.Errorf("insufficient operations: have %d, need %d", m.mempool.Count(), BUNDLE_SIZE) 510 510 + return nil, fmt.Errorf("insufficient operations: have %d, need %d", m.mempool.Count(), types.BUNDLE_SIZE) 509 511 } 510 512 511 511 - if m.mempool.Count() < BUNDLE_SIZE { 513 513 + if m.mempool.Count() < types.BUNDLE_SIZE { 512 514 m.mempool.Save() 513 513 - return nil, fmt.Errorf("insufficient operations: have %d, need %d (no more available)", m.mempool.Count(), BUNDLE_SIZE) 515 515 + return nil, fmt.Errorf("insufficient operations: have %d, need %d (no more available)", m.mempool.Count(), types.BUNDLE_SIZE) 514 516 } 515 517 } 516 518 517 519 if !quiet { 518 520 m.logger.Printf("Creating bundle %06d from mempool", nextBundleNum) 519 521 } 520 520 - operations, err := m.mempool.Take(BUNDLE_SIZE) 522 522 + operations, err := m.mempool.Take(types.BUNDLE_SIZE) 521 523 if err != nil { 522 524 m.mempool.Save() 523 525 return nil, fmt.Errorf("failed to take operations from mempool: %w", err) ··· 1285 1287 } 1286 1288 1287 1289 // GetMempool returns the current mempool 1288 1288 - func (m *Manager) GetMempool() *Mempool { 1290 1290 + func (m *Manager) GetMempool() *mempool.Mempool { 1289 1291 return m.mempool 1290 1292 } 1291 1293 ··· 1309 1311 func (m *Manager) GetCurrentCursor() int { 1310 1312 index := m.GetIndex() 1311 1313 bundles := index.GetBundles() 1312 1312 - cursor := len(bundles) * BUNDLE_SIZE 1314 1314 + cursor := len(bundles) * types.BUNDLE_SIZE 1313 1315 1314 1316 // Add mempool operations 1315 1317 mempoolStats := m.GetMempoolStats() ··· 1329 1331 } 1330 1332 1331 1333 // Validate position 1332 1332 - if position < 0 || position >= BUNDLE_SIZE { 1333 1333 - return nil, fmt.Errorf("invalid position: %d (must be 0-%d)", position, BUNDLE_SIZE-1) 1334 1334 + if position < 0 || position >= types.BUNDLE_SIZE { 1335 1335 + return nil, fmt.Errorf("invalid position: %d (must be 0-%d)", position, types.BUNDLE_SIZE-1) 1334 1336 } 1335 1337 1336 1338 // Build file path

-418

bundle/mempool.go

··· 1 1 - package bundle 2 2 - 3 3 - import ( 4 4 - "bufio" 5 5 - "bytes" 6 6 - "fmt" 7 7 - "os" 8 8 - "path/filepath" 9 9 - "sync" 10 10 - "time" 11 11 - 12 12 - "github.com/goccy/go-json" 13 13 - "tangled.org/atscan.net/plcbundle/plcclient" 14 14 - ) 15 15 - 16 16 - const MEMPOOL_FILE_PREFIX = "plc_mempool_" 17 17 - 18 18 - // Mempool stores operations waiting to be bundled 19 19 - // Operations must be strictly chronological 20 20 - type Mempool struct { 21 21 - operations []plcclient.PLCOperation 22 22 - targetBundle int // Which bundle number these operations are for 23 23 - minTimestamp time.Time // Operations must be after this time 24 24 - file string 25 25 - mu sync.RWMutex 26 26 - logger Logger 27 27 - validated bool // Track if we've validated chronological order 28 28 - dirty bool // Track if mempool changed 29 29 - } 30 30 - 31 31 - // NewMempool creates a new mempool for a specific bundle number 32 32 - func NewMempool(bundleDir string, targetBundle int, minTimestamp time.Time, logger Logger) (*Mempool, error) { 33 33 - filename := fmt.Sprintf("%s%06d.jsonl", MEMPOOL_FILE_PREFIX, targetBundle) 34 34 - 35 35 - m := &Mempool{ 36 36 - file: filepath.Join(bundleDir, filename), 37 37 - targetBundle: targetBundle, 38 38 - minTimestamp: minTimestamp, 39 39 - operations: make([]plcclient.PLCOperation, 0), 40 40 - logger: logger, 41 41 - validated: false, 42 42 - } 43 43 - 44 44 - // Load existing mempool from disk if it exists 45 45 - if err := m.Load(); err != nil { 46 46 - // If file doesn't exist, that's OK 47 47 - if !os.IsNotExist(err) { 48 48 - return nil, fmt.Errorf("failed to load mempool: %w", err) 49 49 - } 50 50 - } 51 51 - 52 52 - return m, nil 53 53 - } 54 54 - 55 55 - // Add adds operations to the mempool with strict validation 56 56 - func (m *Mempool) Add(ops []plcclient.PLCOperation) (int, error) { 57 57 - m.mu.Lock() 58 58 - defer m.mu.Unlock() 59 59 - 60 60 - if len(ops) == 0 { 61 61 - return 0, nil 62 62 - } 63 63 - 64 64 - // Build existing CID set 65 65 - existingCIDs := make(map[string]bool) 66 66 - for _, op := range m.operations { 67 67 - existingCIDs[op.CID] = true 68 68 - } 69 69 - 70 70 - // Validate and add operations 71 71 - var newOps []plcclient.PLCOperation 72 72 - var lastTime time.Time 73 73 - 74 74 - // Start from last operation time if we have any 75 75 - if len(m.operations) > 0 { 76 76 - lastTime = m.operations[len(m.operations)-1].CreatedAt 77 77 - } else { 78 78 - lastTime = m.minTimestamp 79 79 - } 80 80 - 81 81 - for _, op := range ops { 82 82 - // Skip duplicates 83 83 - if existingCIDs[op.CID] { 84 84 - continue 85 85 - } 86 86 - 87 87 - // CRITICAL: Validate chronological order 88 88 - if !op.CreatedAt.After(lastTime) && !op.CreatedAt.Equal(lastTime) { 89 89 - return len(newOps), fmt.Errorf( 90 90 - "chronological violation: operation %s at %s is not after %s", 91 91 - op.CID, op.CreatedAt.Format(time.RFC3339Nano), lastTime.Format(time.RFC3339Nano), 92 92 - ) 93 93 - } 94 94 - 95 95 - // Validate operation is after minimum timestamp 96 96 - if op.CreatedAt.Before(m.minTimestamp) { 97 97 - return len(newOps), fmt.Errorf( 98 98 - "operation %s at %s is before minimum timestamp %s (belongs in earlier bundle)", 99 99 - op.CID, op.CreatedAt.Format(time.RFC3339Nano), m.minTimestamp.Format(time.RFC3339Nano), 100 100 - ) 101 101 - } 102 102 - 103 103 - newOps = append(newOps, op) 104 104 - existingCIDs[op.CID] = true 105 105 - lastTime = op.CreatedAt 106 106 - } 107 107 - 108 108 - // Add new operations 109 109 - m.operations = append(m.operations, newOps...) 110 110 - m.validated = true 111 111 - m.dirty = true 112 112 - 113 113 - return len(newOps), nil 114 114 - } 115 115 - 116 116 - // Validate performs a full chronological validation of all operations 117 117 - func (m *Mempool) Validate() error { 118 118 - m.mu.RLock() 119 119 - defer m.mu.RUnlock() 120 120 - 121 121 - if len(m.operations) == 0 { 122 122 - return nil 123 123 - } 124 124 - 125 125 - // Check all operations are after minimum timestamp 126 126 - for i, op := range m.operations { 127 127 - if op.CreatedAt.Before(m.minTimestamp) { 128 128 - return fmt.Errorf( 129 129 - "operation %d (CID: %s) at %s is before minimum timestamp %s", 130 130 - i, op.CID, op.CreatedAt.Format(time.RFC3339Nano), m.minTimestamp.Format(time.RFC3339Nano), 131 131 - ) 132 132 - } 133 133 - } 134 134 - 135 135 - // Check chronological order 136 136 - for i := 1; i < len(m.operations); i++ { 137 137 - prev := m.operations[i-1] 138 138 - curr := m.operations[i] 139 139 - 140 140 - if curr.CreatedAt.Before(prev.CreatedAt) { 141 141 - return fmt.Errorf( 142 142 - "chronological violation at index %d: %s (%s) is before %s (%s)", 143 143 - i, curr.CID, curr.CreatedAt.Format(time.RFC3339Nano), 144 144 - prev.CID, prev.CreatedAt.Format(time.RFC3339Nano), 145 145 - ) 146 146 - } 147 147 - } 148 148 - 149 149 - // Check for duplicate CIDs 150 150 - cidSet := make(map[string]int) 151 151 - for i, op := range m.operations { 152 152 - if prevIdx, exists := cidSet[op.CID]; exists { 153 153 - return fmt.Errorf( 154 154 - "duplicate CID %s at indices %d and %d", 155 155 - op.CID, prevIdx, i, 156 156 - ) 157 157 - } 158 158 - cidSet[op.CID] = i 159 159 - } 160 160 - 161 161 - return nil 162 162 - } 163 163 - 164 164 - // Count returns the number of operations in mempool 165 165 - func (m *Mempool) Count() int { 166 166 - m.mu.RLock() 167 167 - defer m.mu.RUnlock() 168 168 - return len(m.operations) 169 169 - } 170 170 - 171 171 - // Take removes and returns up to n operations from the front 172 172 - func (m *Mempool) Take(n int) ([]plcclient.PLCOperation, error) { 173 173 - m.mu.Lock() 174 174 - defer m.mu.Unlock() 175 175 - 176 176 - // Validate before taking 177 177 - if err := m.validateLocked(); err != nil { 178 178 - return nil, fmt.Errorf("mempool validation failed: %w", err) 179 179 - } 180 180 - 181 181 - if n > len(m.operations) { 182 182 - n = len(m.operations) 183 183 - } 184 184 - 185 185 - result := make([]plcclient.PLCOperation, n) 186 186 - copy(result, m.operations[:n]) 187 187 - 188 188 - // Remove taken operations 189 189 - m.operations = m.operations[n:] 190 190 - 191 191 - return result, nil 192 192 - } 193 193 - 194 194 - // validateLocked performs validation with lock already held 195 195 - func (m *Mempool) validateLocked() error { 196 196 - if m.validated { 197 197 - return nil 198 198 - } 199 199 - 200 200 - if len(m.operations) == 0 { 201 201 - return nil 202 202 - } 203 203 - 204 204 - // Check chronological order 205 205 - lastTime := m.minTimestamp 206 206 - for i, op := range m.operations { 207 207 - if op.CreatedAt.Before(lastTime) { 208 208 - return fmt.Errorf( 209 209 - "chronological violation at index %d: %s is before %s", 210 210 - i, op.CreatedAt.Format(time.RFC3339Nano), lastTime.Format(time.RFC3339Nano), 211 211 - ) 212 212 - } 213 213 - lastTime = op.CreatedAt 214 214 - } 215 215 - 216 216 - m.validated = true 217 217 - return nil 218 218 - } 219 219 - 220 220 - // Peek returns up to n operations without removing them 221 221 - func (m *Mempool) Peek(n int) []plcclient.PLCOperation { 222 222 - m.mu.RLock() 223 223 - defer m.mu.RUnlock() 224 224 - 225 225 - if n > len(m.operations) { 226 226 - n = len(m.operations) 227 227 - } 228 228 - 229 229 - result := make([]plcclient.PLCOperation, n) 230 230 - copy(result, m.operations[:n]) 231 231 - 232 232 - return result 233 233 - } 234 234 - 235 235 - // Clear removes all operations 236 236 - func (m *Mempool) Clear() { 237 237 - m.mu.Lock() 238 238 - defer m.mu.Unlock() 239 239 - m.operations = make([]plcclient.PLCOperation, 0) 240 240 - m.validated = false 241 241 - } 242 242 - 243 243 - // Save persists mempool to disk 244 244 - func (m *Mempool) Save() error { 245 245 - m.mu.Lock() 246 246 - defer m.mu.Unlock() 247 247 - 248 248 - if !m.dirty { 249 249 - return nil 250 250 - } 251 251 - 252 252 - if len(m.operations) == 0 { 253 253 - // Remove file if empty 254 254 - os.Remove(m.file) 255 255 - return nil 256 256 - } 257 257 - 258 258 - // Validate before saving 259 259 - if err := m.validateLocked(); err != nil { 260 260 - return fmt.Errorf("mempool validation failed, refusing to save: %w", err) 261 261 - } 262 262 - 263 263 - // Serialize to JSONL 264 264 - var buf bytes.Buffer 265 265 - for _, op := range m.operations { 266 266 - if len(op.RawJSON) > 0 { 267 267 - buf.Write(op.RawJSON) 268 268 - } else { 269 269 - data, _ := json.Marshal(op) 270 270 - buf.Write(data) 271 271 - } 272 272 - buf.WriteByte('\n') 273 273 - } 274 274 - 275 275 - // Write atomically 276 276 - tempFile := m.file + ".tmp" 277 277 - if err := os.WriteFile(tempFile, buf.Bytes(), 0644); err != nil { 278 278 - return fmt.Errorf("failed to write mempool: %w", err) 279 279 - } 280 280 - 281 281 - if err := os.Rename(tempFile, m.file); err != nil { 282 282 - os.Remove(tempFile) 283 283 - return fmt.Errorf("failed to rename mempool file: %w", err) 284 284 - } 285 285 - 286 286 - m.dirty = false 287 287 - return nil 288 288 - } 289 289 - 290 290 - // Load reads mempool from disk and validates it 291 291 - func (m *Mempool) Load() error { 292 292 - data, err := os.ReadFile(m.file) 293 293 - if err != nil { 294 294 - return err 295 295 - } 296 296 - 297 297 - m.mu.Lock() 298 298 - defer m.mu.Unlock() 299 299 - 300 300 - // Parse JSONL 301 301 - scanner := bufio.NewScanner(bytes.NewReader(data)) 302 302 - buf := make([]byte, 0, 64*1024) 303 303 - scanner.Buffer(buf, 1024*1024) 304 304 - 305 305 - m.operations = make([]plcclient.PLCOperation, 0) 306 306 - 307 307 - for scanner.Scan() { 308 308 - line := scanner.Bytes() 309 309 - if len(line) == 0 { 310 310 - continue 311 311 - } 312 312 - 313 313 - var op plcclient.PLCOperation 314 314 - if err := json.Unmarshal(line, &op); err != nil { 315 315 - return fmt.Errorf("failed to parse mempool operation: %w", err) 316 316 - } 317 317 - 318 318 - op.RawJSON = make([]byte, len(line)) 319 319 - copy(op.RawJSON, line) 320 320 - 321 321 - m.operations = append(m.operations, op) 322 322 - } 323 323 - 324 324 - if err := scanner.Err(); err != nil { 325 325 - return fmt.Errorf("scanner error: %w", err) 326 326 - } 327 327 - 328 328 - // CRITICAL: Validate loaded data 329 329 - if err := m.validateLocked(); err != nil { 330 330 - return fmt.Errorf("loaded mempool failed validation: %w", err) 331 331 - } 332 332 - 333 333 - if len(m.operations) > 0 { 334 334 - m.logger.Printf("Loaded %d operations from mempool for bundle %06d", len(m.operations), m.targetBundle) 335 335 - } 336 336 - 337 337 - return nil 338 338 - } 339 339 - 340 340 - // GetFirstTime returns the created_at of the first operation 341 341 - func (m *Mempool) GetFirstTime() string { 342 342 - m.mu.RLock() 343 343 - defer m.mu.RUnlock() 344 344 - 345 345 - if len(m.operations) == 0 { 346 346 - return "" 347 347 - } 348 348 - 349 349 - return m.operations[0].CreatedAt.Format(time.RFC3339Nano) 350 350 - } 351 351 - 352 352 - // GetLastTime returns the created_at of the last operation 353 353 - func (m *Mempool) GetLastTime() string { 354 354 - m.mu.RLock() 355 355 - defer m.mu.RUnlock() 356 356 - 357 357 - if len(m.operations) == 0 { 358 358 - return "" 359 359 - } 360 360 - 361 361 - return m.operations[len(m.operations)-1].CreatedAt.Format(time.RFC3339Nano) 362 362 - } 363 363 - 364 364 - // GetTargetBundle returns the bundle number this mempool is for 365 365 - func (m *Mempool) GetTargetBundle() int { 366 366 - return m.targetBundle 367 367 - } 368 368 - 369 369 - // GetMinTimestamp returns the minimum timestamp for operations 370 370 - func (m *Mempool) GetMinTimestamp() time.Time { 371 371 - return m.minTimestamp 372 372 - } 373 373 - 374 374 - // Stats returns mempool statistics 375 375 - func (m *Mempool) Stats() map[string]interface{} { 376 376 - m.mu.RLock() 377 377 - defer m.mu.RUnlock() 378 378 - 379 379 - count := len(m.operations) 380 380 - 381 381 - stats := map[string]interface{}{ 382 382 - "count": count, 383 383 - "can_create_bundle": count >= BUNDLE_SIZE, 384 384 - "target_bundle": m.targetBundle, 385 385 - "min_timestamp": m.minTimestamp, 386 386 - "validated": m.validated, 387 387 - } 388 388 - 389 389 - if count > 0 { 390 390 - stats["first_time"] = m.operations[0].CreatedAt 391 391 - stats["last_time"] = m.operations[len(m.operations)-1].CreatedAt 392 392 - 393 393 - // Calculate size and unique DIDs 394 394 - totalSize := 0 395 395 - didSet := make(map[string]bool) 396 396 - for _, op := range m.operations { 397 397 - totalSize += len(op.RawJSON) 398 398 - didSet[op.DID] = true 399 399 - } 400 400 - stats["size_bytes"] = totalSize 401 401 - stats["did_count"] = len(didSet) 402 402 - } 403 403 - 404 404 - return stats 405 405 - } 406 406 - 407 407 - // Delete removes the mempool file 408 408 - func (m *Mempool) Delete() error { 409 409 - if err := os.Remove(m.file); err != nil && !os.IsNotExist(err) { 410 410 - return fmt.Errorf("failed to delete mempool file: %w", err) 411 411 - } 412 412 - return nil 413 413 - } 414 414 - 415 415 - // GetFilename returns the mempool filename 416 416 - func (m *Mempool) GetFilename() string { 417 417 - return filepath.Base(m.file) 418 418 - }

bundle/metadata.go internal/bundle/metadata.go

+6 -16

bundle/types.go internal/bundle/types.go

··· 5 5 "path/filepath" 6 6 "time" 7 7 8 8 + "tangled.org/atscan.net/plcbundle/internal/types" 8 9 "tangled.org/atscan.net/plcbundle/plcclient" 9 9 - ) 10 10 - 11 11 - const ( 12 12 - // BUNDLE_SIZE is the standard number of operations per bundle 13 13 - BUNDLE_SIZE = 10000 14 10 ) 15 11 16 12 // Bundle represents a PLC bundle ··· 49 45 if len(b.Operations) > 0 { 50 46 return len(b.Operations) 51 47 } 52 52 - return BUNDLE_SIZE 48 48 + return types.BUNDLE_SIZE 53 49 } 54 50 55 51 // CompressionRatio returns the compression ratio ··· 65 61 if b.BundleNumber < 1 { 66 62 return fmt.Errorf("invalid bundle number: %d", b.BundleNumber) 67 63 } 68 68 - if len(b.Operations) != BUNDLE_SIZE { 69 69 - return fmt.Errorf("invalid operation count: expected %d, got %d", BUNDLE_SIZE, len(b.Operations)) 64 64 + if len(b.Operations) != types.BUNDLE_SIZE { 65 65 + return fmt.Errorf("invalid operation count: expected %d, got %d", types.BUNDLE_SIZE, len(b.Operations)) 70 66 } 71 67 if b.StartTime.After(b.EndTime) { 72 68 return fmt.Errorf("start_time is after end_time") ··· 163 159 IndexUpdated bool 164 160 } 165 161 166 166 - // Logger interface for bundle operations 167 167 - type Logger interface { 168 168 - Printf(format string, v ...interface{}) 169 169 - Println(v ...interface{}) 170 170 - } 171 171 - 172 162 // Config holds configuration for bundle operations 173 163 type Config struct { 174 164 BundleDir string ··· 176 166 AutoRebuild bool 177 167 RebuildWorkers int // Number of workers for parallel rebuild (0 = auto-detect) 178 168 RebuildProgress func(current, total int) // Progress callback for rebuild 179 179 - Logger Logger 169 169 + Logger types.Logger 180 170 } 181 171 182 172 // DefaultConfig returns default configuration ··· 199 189 ProgressFunc func(downloaded, total int, bytesDownloaded, bytesTotal int64) 200 190 SaveInterval time.Duration 201 191 Verbose bool 202 202 - Logger Logger 192 192 + Logger types.Logger 203 193 } 204 194 205 195 // CloneResult contains cloning results

+1 -2

cmd/plcbundle/compare.go

··· 11 11 "time" 12 12 13 13 "github.com/goccy/go-json" 14 14 - 15 15 - "tangled.org/atscan.net/plcbundle/bundle" 14 14 + "tangled.org/atscan.net/plcbundle/internal/bundle" 16 15 ) 17 16 18 17 // IndexComparison holds comparison results

+7 -6

cmd/plcbundle/info.go

··· 9 9 "strings" 10 10 "time" 11 11 12 12 - "tangled.org/atscan.net/plcbundle/bundle" 12 12 + "tangled.org/atscan.net/plcbundle/internal/bundle" 13 13 + "tangled.org/atscan.net/plcbundle/internal/types" 13 14 ) 14 15 15 16 func showGeneralInfo(mgr *bundle.Manager, dir string, verbose bool, showBundles bool, verify bool, showTimeline bool) { ··· 83 84 // Operations count (exact calculation) 84 85 mempoolStats := mgr.GetMempoolStats() 85 86 mempoolCount := mempoolStats["count"].(int) 86 86 - bundleOpsCount := bundleCount * bundle.BUNDLE_SIZE 87 87 + bundleOpsCount := bundleCount * types.BUNDLE_SIZE 87 88 totalOps := bundleOpsCount + mempoolCount 88 89 89 90 fmt.Printf("🔢 Operations\n") ··· 146 147 if mempoolCount > 0 { 147 148 targetBundle := mempoolStats["target_bundle"].(int) 148 149 canCreate := mempoolStats["can_create_bundle"].(bool) 149 149 - progress := float64(mempoolCount) / float64(bundle.BUNDLE_SIZE) * 100 150 150 + progress := float64(mempoolCount) / float64(types.BUNDLE_SIZE) * 100 150 151 151 152 fmt.Printf("🔄 Mempool (next bundle: %06d)\n", targetBundle) 152 152 - fmt.Printf(" Operations: %s / %s\n", formatNumber(mempoolCount), formatNumber(bundle.BUNDLE_SIZE)) 153 153 + fmt.Printf(" Operations: %s / %s\n", formatNumber(mempoolCount), formatNumber(types.BUNDLE_SIZE)) 153 154 fmt.Printf(" Progress: %.1f%%\n", progress) 154 155 155 156 // Progress bar 156 157 barWidth := 40 157 157 - filled := int(float64(barWidth) * float64(mempoolCount) / float64(bundle.BUNDLE_SIZE)) 158 158 + filled := int(float64(barWidth) * float64(mempoolCount) / float64(types.BUNDLE_SIZE)) 158 159 if filled > barWidth { 159 160 filled = barWidth 160 161 } ··· 164 165 if canCreate { 165 166 fmt.Printf(" ✓ Ready to create bundle\n") 166 167 } else { 167 167 - remaining := bundle.BUNDLE_SIZE - mempoolCount 168 168 + remaining := types.BUNDLE_SIZE - mempoolCount 168 169 fmt.Printf(" Need %s more operations\n", formatNumber(remaining)) 169 170 } 170 171 fmt.Printf("\n")

+6 -5

cmd/plcbundle/main.go

··· 18 18 19 19 "github.com/goccy/go-json" 20 20 21 21 - "tangled.org/atscan.net/plcbundle/bundle" 21 21 + "tangled.org/atscan.net/plcbundle/internal/bundle" 22 22 "tangled.org/atscan.net/plcbundle/internal/didindex" 23 23 + "tangled.org/atscan.net/plcbundle/internal/types" 23 24 "tangled.org/atscan.net/plcbundle/plcclient" 24 25 ) 25 26 ··· 1165 1166 fmt.Printf("Mempool Status:\n") 1166 1167 fmt.Printf(" Target bundle: %06d\n", targetBundle) 1167 1168 fmt.Printf(" Operations: %d\n", count) 1168 1168 - fmt.Printf(" Can create bundle: %v (need %d)\n", canCreate, bundle.BUNDLE_SIZE) 1169 1169 + fmt.Printf(" Can create bundle: %v (need %d)\n", canCreate, types.BUNDLE_SIZE) 1169 1170 fmt.Printf(" Min timestamp: %s\n", minTimestamp.Format("2006-01-02 15:04:05")) 1170 1171 1171 1172 validationIcon := "✓" ··· 1187 1188 fmt.Printf(" Last operation: %s\n", lastTime.Format("2006-01-02 15:04:05")) 1188 1189 } 1189 1190 1190 1190 - progress := float64(count) / float64(bundle.BUNDLE_SIZE) * 100 1191 1191 - fmt.Printf(" Progress: %.1f%% (%d/%d)\n", progress, count, bundle.BUNDLE_SIZE) 1191 1191 + progress := float64(count) / float64(types.BUNDLE_SIZE) * 100 1192 1192 + fmt.Printf(" Progress: %.1f%% (%d/%d)\n", progress, count, types.BUNDLE_SIZE) 1192 1193 1193 1194 // Show progress bar 1194 1195 barWidth := 40 1195 1195 - filled := int(float64(barWidth) * float64(count) / float64(bundle.BUNDLE_SIZE)) 1196 1196 + filled := int(float64(barWidth) * float64(count) / float64(types.BUNDLE_SIZE)) 1196 1197 if filled > barWidth { 1197 1198 filled = barWidth 1198 1199 }

+17 -16

cmd/plcbundle/server.go

··· 15 15 "github.com/goccy/go-json" 16 16 "github.com/gorilla/websocket" 17 17 18 18 - "tangled.org/atscan.net/plcbundle/bundle" 18 18 + "tangled.org/atscan.net/plcbundle/internal/bundle" 19 19 + "tangled.org/atscan.net/plcbundle/internal/types" 19 20 "tangled.org/atscan.net/plcbundle/plcclient" 20 21 ) 21 22 ··· 256 257 sb.WriteString("\nMempool Stats\n") 257 258 sb.WriteString("━━━━━━━━━━━━━\n") 258 259 sb.WriteString(fmt.Sprintf(" Target bundle: %d\n", targetBundle)) 259 259 - sb.WriteString(fmt.Sprintf(" Operations: %d / %d\n", count, bundle.BUNDLE_SIZE)) 260 260 + sb.WriteString(fmt.Sprintf(" Operations: %d / %d\n", count, types.BUNDLE_SIZE)) 260 261 sb.WriteString(fmt.Sprintf(" Can create bundle: %v\n", canCreate)) 261 262 262 263 if count > 0 { 263 263 - progress := float64(count) / float64(bundle.BUNDLE_SIZE) * 100 264 264 + progress := float64(count) / float64(types.BUNDLE_SIZE) * 100 264 265 sb.WriteString(fmt.Sprintf(" Progress: %.1f%%\n", progress)) 265 266 266 267 barWidth := 50 267 267 - filled := int(float64(barWidth) * float64(count) / float64(bundle.BUNDLE_SIZE)) 268 268 + filled := int(float64(barWidth) * float64(count) / float64(types.BUNDLE_SIZE)) 268 269 if filled > barWidth { 269 270 filled = barWidth 270 271 } ··· 355 356 sb.WriteString(" Default: starts from latest (skips all historical data)\n") 356 357 357 358 latestCursor := mgr.GetCurrentCursor() 358 358 - bundledOps := len(index.GetBundles()) * bundle.BUNDLE_SIZE 359 359 + bundledOps := len(index.GetBundles()) * types.BUNDLE_SIZE 359 360 mempoolOps := latestCursor - bundledOps 360 361 361 362 if syncMode && mempoolOps > 0 { ··· 517 518 } 518 519 } 519 520 520 520 - totalOps := bundleCount * bundle.BUNDLE_SIZE 521 521 + totalOps := bundleCount * types.BUNDLE_SIZE 521 522 response.Bundles.TotalOperations = totalOps 522 523 523 524 duration := response.Bundles.EndTime.Sub(response.Bundles.StartTime) ··· 536 537 CanCreateBundle: mempoolStats["can_create_bundle"].(bool), 537 538 MinTimestamp: mempoolStats["min_timestamp"].(time.Time), 538 539 Validated: mempoolStats["validated"].(bool), 539 539 - ProgressPercent: float64(count) / float64(bundle.BUNDLE_SIZE) * 100, 540 540 - BundleSize: bundle.BUNDLE_SIZE, 541 541 - OperationsNeeded: bundle.BUNDLE_SIZE - count, 540 540 + ProgressPercent: float64(count) / float64(types.BUNDLE_SIZE) * 100, 541 541 + BundleSize: types.BUNDLE_SIZE, 542 542 + OperationsNeeded: types.BUNDLE_SIZE - count, 542 543 } 543 544 544 545 if firstTime, ok := mempoolStats["first_time"].(time.Time); ok { ··· 550 551 mempool.LastOpAgeSeconds = int(time.Since(lastTime).Seconds()) 551 552 } 552 553 553 553 - if count > 100 && count < bundle.BUNDLE_SIZE { 554 554 + if count > 100 && count < types.BUNDLE_SIZE { 554 555 if !mempool.FirstTime.IsZero() && !mempool.LastTime.IsZero() { 555 556 timespan := mempool.LastTime.Sub(mempool.FirstTime) 556 557 if timespan.Seconds() > 0 { 557 558 opsPerSec := float64(count) / timespan.Seconds() 558 558 - remaining := bundle.BUNDLE_SIZE - count 559 559 + remaining := types.BUNDLE_SIZE - count 559 560 mempool.EtaNextBundleSeconds = int(float64(remaining) / opsPerSec) 560 561 } 561 562 } ··· 770 771 currentRecord := startCursor 771 772 772 773 if len(bundles) > 0 { 773 773 - startBundleIdx := startCursor / bundle.BUNDLE_SIZE 774 774 - startPosition := startCursor % bundle.BUNDLE_SIZE 774 774 + startBundleIdx := startCursor / types.BUNDLE_SIZE 775 775 + startPosition := startCursor % types.BUNDLE_SIZE 775 776 776 777 if startBundleIdx < len(bundles) { 777 778 for i := startBundleIdx; i < len(bundles); i++ { ··· 790 791 } 791 792 792 793 lastSeenMempoolCount := 0 793 793 - if err := streamMempool(conn, mgr, startCursor, len(bundles)*bundle.BUNDLE_SIZE, &currentRecord, &lastSeenMempoolCount, done); err != nil { 794 794 + if err := streamMempool(conn, mgr, startCursor, len(bundles)*types.BUNDLE_SIZE, &currentRecord, &lastSeenMempoolCount, done); err != nil { 794 795 return err 795 796 } 796 797 ··· 821 822 fmt.Fprintf(os.Stderr, "WebSocket: %d new bundle(s) created (operations already streamed from mempool)\n", newBundleCount) 822 823 } 823 824 824 824 - currentRecord += newBundleCount * bundle.BUNDLE_SIZE 825 825 + currentRecord += newBundleCount * types.BUNDLE_SIZE 825 826 lastBundleCount = len(bundles) 826 827 lastSeenMempoolCount = 0 827 828 } 828 829 829 829 - if err := streamMempool(conn, mgr, startCursor, len(bundles)*bundle.BUNDLE_SIZE, &currentRecord, &lastSeenMempoolCount, done); err != nil { 830 830 + if err := streamMempool(conn, mgr, startCursor, len(bundles)*types.BUNDLE_SIZE, &currentRecord, &lastSeenMempoolCount, done); err != nil { 830 831 return err 831 832 } 832 833

+1 -1

detector/runner.go

··· 7 7 "sync" 8 8 "time" 9 9 10 10 - "tangled.org/atscan.net/plcbundle/bundle" 10 10 + "tangled.org/atscan.net/plcbundle/internal/bundle" 11 11 "tangled.org/atscan.net/plcbundle/plcclient" 12 12 ) 13 13

+418

internal/mempool/mempool.go

··· 1 1 package mempool 2 2 + 3 3 + import ( 4 4 + "bufio" 5 5 + "bytes" 6 6 + "fmt" 7 7 + "os" 8 8 + "path/filepath" 9 9 + "sync" 10 10 + "time" 11 11 + 12 12 + "github.com/goccy/go-json" 13 13 + "tangled.org/atscan.net/plcbundle/internal/types" 14 14 + "tangled.org/atscan.net/plcbundle/plcclient" 15 15 + ) 16 16 + 17 17 + const MEMPOOL_FILE_PREFIX = "plc_mempool_" 18 18 + 19 19 + // Mempool stores operations waiting to be bundled 20 20 + // Operations must be strictly chronological 21 21 + type Mempool struct { 22 22 + operations []plcclient.PLCOperation 23 23 + targetBundle int // Which bundle number these operations are for 24 24 + minTimestamp time.Time // Operations must be after this time 25 25 + file string 26 26 + mu sync.RWMutex 27 27 + logger types.Logger 28 28 + validated bool // Track if we've validated chronological order 29 29 + dirty bool // Track if mempool changed 30 30 + } 31 31 + 32 32 + // NewMempool creates a new mempool for a specific bundle number 33 33 + func NewMempool(bundleDir string, targetBundle int, minTimestamp time.Time, logger types.Logger) (*Mempool, error) { 34 34 + filename := fmt.Sprintf("%s%06d.jsonl", MEMPOOL_FILE_PREFIX, targetBundle) 35 35 + 36 36 + m := &Mempool{ 37 37 + file: filepath.Join(bundleDir, filename), 38 38 + targetBundle: targetBundle, 39 39 + minTimestamp: minTimestamp, 40 40 + operations: make([]plcclient.PLCOperation, 0), 41 41 + logger: logger, 42 42 + validated: false, 43 43 + } 44 44 + 45 45 + // Load existing mempool from disk if it exists 46 46 + if err := m.Load(); err != nil { 47 47 + // If file doesn't exist, that's OK 48 48 + if !os.IsNotExist(err) { 49 49 + return nil, fmt.Errorf("failed to load mempool: %w", err) 50 50 + } 51 51 + } 52 52 + 53 53 + return m, nil 54 54 + } 55 55 + 56 56 + // Add adds operations to the mempool with strict validation 57 57 + func (m *Mempool) Add(ops []plcclient.PLCOperation) (int, error) { 58 58 + m.mu.Lock() 59 59 + defer m.mu.Unlock() 60 60 + 61 61 + if len(ops) == 0 { 62 62 + return 0, nil 63 63 + } 64 64 + 65 65 + // Build existing CID set 66 66 + existingCIDs := make(map[string]bool) 67 67 + for _, op := range m.operations { 68 68 + existingCIDs[op.CID] = true 69 69 + } 70 70 + 71 71 + // Validate and add operations 72 72 + var newOps []plcclient.PLCOperation 73 73 + var lastTime time.Time 74 74 + 75 75 + // Start from last operation time if we have any 76 76 + if len(m.operations) > 0 { 77 77 + lastTime = m.operations[len(m.operations)-1].CreatedAt 78 78 + } else { 79 79 + lastTime = m.minTimestamp 80 80 + } 81 81 + 82 82 + for _, op := range ops { 83 83 + // Skip duplicates 84 84 + if existingCIDs[op.CID] { 85 85 + continue 86 86 + } 87 87 + 88 88 + // CRITICAL: Validate chronological order 89 89 + if !op.CreatedAt.After(lastTime) && !op.CreatedAt.Equal(lastTime) { 90 90 + return len(newOps), fmt.Errorf( 91 91 + "chronological violation: operation %s at %s is not after %s", 92 92 + op.CID, op.CreatedAt.Format(time.RFC3339Nano), lastTime.Format(time.RFC3339Nano), 93 93 + ) 94 94 + } 95 95 + 96 96 + // Validate operation is after minimum timestamp 97 97 + if op.CreatedAt.Before(m.minTimestamp) { 98 98 + return len(newOps), fmt.Errorf( 99 99 + "operation %s at %s is before minimum timestamp %s (belongs in earlier bundle)", 100 100 + op.CID, op.CreatedAt.Format(time.RFC3339Nano), m.minTimestamp.Format(time.RFC3339Nano), 101 101 + ) 102 102 + } 103 103 + 104 104 + newOps = append(newOps, op) 105 105 + existingCIDs[op.CID] = true 106 106 + lastTime = op.CreatedAt 107 107 + } 108 108 + 109 109 + // Add new operations 110 110 + m.operations = append(m.operations, newOps...) 111 111 + m.validated = true 112 112 + m.dirty = true 113 113 + 114 114 + return len(newOps), nil 115 115 + } 116 116 + 117 117 + // Validate performs a full chronological validation of all operations 118 118 + func (m *Mempool) Validate() error { 119 119 + m.mu.RLock() 120 120 + defer m.mu.RUnlock() 121 121 + 122 122 + if len(m.operations) == 0 { 123 123 + return nil 124 124 + } 125 125 + 126 126 + // Check all operations are after minimum timestamp 127 127 + for i, op := range m.operations { 128 128 + if op.CreatedAt.Before(m.minTimestamp) { 129 129 + return fmt.Errorf( 130 130 + "operation %d (CID: %s) at %s is before minimum timestamp %s", 131 131 + i, op.CID, op.CreatedAt.Format(time.RFC3339Nano), m.minTimestamp.Format(time.RFC3339Nano), 132 132 + ) 133 133 + } 134 134 + } 135 135 + 136 136 + // Check chronological order 137 137 + for i := 1; i < len(m.operations); i++ { 138 138 + prev := m.operations[i-1] 139 139 + curr := m.operations[i] 140 140 + 141 141 + if curr.CreatedAt.Before(prev.CreatedAt) { 142 142 + return fmt.Errorf( 143 143 + "chronological violation at index %d: %s (%s) is before %s (%s)", 144 144 + i, curr.CID, curr.CreatedAt.Format(time.RFC3339Nano), 145 145 + prev.CID, prev.CreatedAt.Format(time.RFC3339Nano), 146 146 + ) 147 147 + } 148 148 + } 149 149 + 150 150 + // Check for duplicate CIDs 151 151 + cidSet := make(map[string]int) 152 152 + for i, op := range m.operations { 153 153 + if prevIdx, exists := cidSet[op.CID]; exists { 154 154 + return fmt.Errorf( 155 155 + "duplicate CID %s at indices %d and %d", 156 156 + op.CID, prevIdx, i, 157 157 + ) 158 158 + } 159 159 + cidSet[op.CID] = i 160 160 + } 161 161 + 162 162 + return nil 163 163 + } 164 164 + 165 165 + // Count returns the number of operations in mempool 166 166 + func (m *Mempool) Count() int { 167 167 + m.mu.RLock() 168 168 + defer m.mu.RUnlock() 169 169 + return len(m.operations) 170 170 + } 171 171 + 172 172 + // Take removes and returns up to n operations from the front 173 173 + func (m *Mempool) Take(n int) ([]plcclient.PLCOperation, error) { 174 174 + m.mu.Lock() 175 175 + defer m.mu.Unlock() 176 176 + 177 177 + // Validate before taking 178 178 + if err := m.validateLocked(); err != nil { 179 179 + return nil, fmt.Errorf("mempool validation failed: %w", err) 180 180 + } 181 181 + 182 182 + if n > len(m.operations) { 183 183 + n = len(m.operations) 184 184 + } 185 185 + 186 186 + result := make([]plcclient.PLCOperation, n) 187 187 + copy(result, m.operations[:n]) 188 188 + 189 189 + // Remove taken operations 190 190 + m.operations = m.operations[n:] 191 191 + 192 192 + return result, nil 193 193 + } 194 194 + 195 195 + // validateLocked performs validation with lock already held 196 196 + func (m *Mempool) validateLocked() error { 197 197 + if m.validated { 198 198 + return nil 199 199 + } 200 200 + 201 201 + if len(m.operations) == 0 { 202 202 + return nil 203 203 + } 204 204 + 205 205 + // Check chronological order 206 206 + lastTime := m.minTimestamp 207 207 + for i, op := range m.operations { 208 208 + if op.CreatedAt.Before(lastTime) { 209 209 + return fmt.Errorf( 210 210 + "chronological violation at index %d: %s is before %s", 211 211 + i, op.CreatedAt.Format(time.RFC3339Nano), lastTime.Format(time.RFC3339Nano), 212 212 + ) 213 213 + } 214 214 + lastTime = op.CreatedAt 215 215 + } 216 216 + 217 217 + m.validated = true 218 218 + return nil 219 219 + } 220 220 + 221 221 + // Peek returns up to n operations without removing them 222 222 + func (m *Mempool) Peek(n int) []plcclient.PLCOperation { 223 223 + m.mu.RLock() 224 224 + defer m.mu.RUnlock() 225 225 + 226 226 + if n > len(m.operations) { 227 227 + n = len(m.operations) 228 228 + } 229 229 + 230 230 + result := make([]plcclient.PLCOperation, n) 231 231 + copy(result, m.operations[:n]) 232 232 + 233 233 + return result 234 234 + } 235 235 + 236 236 + // Clear removes all operations 237 237 + func (m *Mempool) Clear() { 238 238 + m.mu.Lock() 239 239 + defer m.mu.Unlock() 240 240 + m.operations = make([]plcclient.PLCOperation, 0) 241 241 + m.validated = false 242 242 + } 243 243 + 244 244 + // Save persists mempool to disk 245 245 + func (m *Mempool) Save() error { 246 246 + m.mu.Lock() 247 247 + defer m.mu.Unlock() 248 248 + 249 249 + if !m.dirty { 250 250 + return nil 251 251 + } 252 252 + 253 253 + if len(m.operations) == 0 { 254 254 + // Remove file if empty 255 255 + os.Remove(m.file) 256 256 + return nil 257 257 + } 258 258 + 259 259 + // Validate before saving 260 260 + if err := m.validateLocked(); err != nil { 261 261 + return fmt.Errorf("mempool validation failed, refusing to save: %w", err) 262 262 + } 263 263 + 264 264 + // Serialize to JSONL 265 265 + var buf bytes.Buffer 266 266 + for _, op := range m.operations { 267 267 + if len(op.RawJSON) > 0 { 268 268 + buf.Write(op.RawJSON) 269 269 + } else { 270 270 + data, _ := json.Marshal(op) 271 271 + buf.Write(data) 272 272 + } 273 273 + buf.WriteByte('\n') 274 274 + } 275 275 + 276 276 + // Write atomically 277 277 + tempFile := m.file + ".tmp" 278 278 + if err := os.WriteFile(tempFile, buf.Bytes(), 0644); err != nil { 279 279 + return fmt.Errorf("failed to write mempool: %w", err) 280 280 + } 281 281 + 282 282 + if err := os.Rename(tempFile, m.file); err != nil { 283 283 + os.Remove(tempFile) 284 284 + return fmt.Errorf("failed to rename mempool file: %w", err) 285 285 + } 286 286 + 287 287 + m.dirty = false 288 288 + return nil 289 289 + } 290 290 + 291 291 + // Load reads mempool from disk and validates it 292 292 + func (m *Mempool) Load() error { 293 293 + data, err := os.ReadFile(m.file) 294 294 + if err != nil { 295 295 + return err 296 296 + } 297 297 + 298 298 + m.mu.Lock() 299 299 + defer m.mu.Unlock() 300 300 + 301 301 + // Parse JSONL 302 302 + scanner := bufio.NewScanner(bytes.NewReader(data)) 303 303 + buf := make([]byte, 0, 64*1024) 304 304 + scanner.Buffer(buf, 1024*1024) 305 305 + 306 306 + m.operations = make([]plcclient.PLCOperation, 0) 307 307 + 308 308 + for scanner.Scan() { 309 309 + line := scanner.Bytes() 310 310 + if len(line) == 0 { 311 311 + continue 312 312 + } 313 313 + 314 314 + var op plcclient.PLCOperation 315 315 + if err := json.Unmarshal(line, &op); err != nil { 316 316 + return fmt.Errorf("failed to parse mempool operation: %w", err) 317 317 + } 318 318 + 319 319 + op.RawJSON = make([]byte, len(line)) 320 320 + copy(op.RawJSON, line) 321 321 + 322 322 + m.operations = append(m.operations, op) 323 323 + } 324 324 + 325 325 + if err := scanner.Err(); err != nil { 326 326 + return fmt.Errorf("scanner error: %w", err) 327 327 + } 328 328 + 329 329 + // CRITICAL: Validate loaded data 330 330 + if err := m.validateLocked(); err != nil { 331 331 + return fmt.Errorf("loaded mempool failed validation: %w", err) 332 332 + } 333 333 + 334 334 + if len(m.operations) > 0 { 335 335 + m.logger.Printf("Loaded %d operations from mempool for bundle %06d", len(m.operations), m.targetBundle) 336 336 + } 337 337 + 338 338 + return nil 339 339 + } 340 340 + 341 341 + // GetFirstTime returns the created_at of the first operation 342 342 + func (m *Mempool) GetFirstTime() string { 343 343 + m.mu.RLock() 344 344 + defer m.mu.RUnlock() 345 345 + 346 346 + if len(m.operations) == 0 { 347 347 + return "" 348 348 + } 349 349 + 350 350 + return m.operations[0].CreatedAt.Format(time.RFC3339Nano) 351 351 + } 352 352 + 353 353 + // GetLastTime returns the created_at of the last operation 354 354 + func (m *Mempool) GetLastTime() string { 355 355 + m.mu.RLock() 356 356 + defer m.mu.RUnlock() 357 357 + 358 358 + if len(m.operations) == 0 { 359 359 + return "" 360 360 + } 361 361 + 362 362 + return m.operations[len(m.operations)-1].CreatedAt.Format(time.RFC3339Nano) 363 363 + } 364 364 + 365 365 + // GetTargetBundle returns the bundle number this mempool is for 366 366 + func (m *Mempool) GetTargetBundle() int { 367 367 + return m.targetBundle 368 368 + } 369 369 + 370 370 + // GetMinTimestamp returns the minimum timestamp for operations 371 371 + func (m *Mempool) GetMinTimestamp() time.Time { 372 372 + return m.minTimestamp 373 373 + } 374 374 + 375 375 + // Stats returns mempool statistics 376 376 + func (m *Mempool) Stats() map[string]interface{} { 377 377 + m.mu.RLock() 378 378 + defer m.mu.RUnlock() 379 379 + 380 380 + count := len(m.operations) 381 381 + 382 382 + stats := map[string]interface{}{ 383 383 + "count": count, 384 384 + "can_create_bundle": count >= types.BUNDLE_SIZE, 385 385 + "target_bundle": m.targetBundle, 386 386 + "min_timestamp": m.minTimestamp, 387 387 + "validated": m.validated, 388 388 + } 389 389 + 390 390 + if count > 0 { 391 391 + stats["first_time"] = m.operations[0].CreatedAt 392 392 + stats["last_time"] = m.operations[len(m.operations)-1].CreatedAt 393 393 + 394 394 + // Calculate size and unique DIDs 395 395 + totalSize := 0 396 396 + didSet := make(map[string]bool) 397 397 + for _, op := range m.operations { 398 398 + totalSize += len(op.RawJSON) 399 399 + didSet[op.DID] = true 400 400 + } 401 401 + stats["size_bytes"] = totalSize 402 402 + stats["did_count"] = len(didSet) 403 403 + } 404 404 + 405 405 + return stats 406 406 + } 407 407 + 408 408 + // Delete removes the mempool file 409 409 + func (m *Mempool) Delete() error { 410 410 + if err := os.Remove(m.file); err != nil && !os.IsNotExist(err) { 411 411 + return fmt.Errorf("failed to delete mempool file: %w", err) 412 412 + } 413 413 + return nil 414 414 + } 415 415 + 416 416 + // GetFilename returns the mempool filename 417 417 + func (m *Mempool) GetFilename() string { 418 418 + return filepath.Base(m.file) 419 419 + }

+18

internal/types/types.go

··· 1 1 + package types 2 2 + 3 3 + // Logger is a simple logging interface used throughout plcbundle 4 4 + type Logger interface { 5 5 + Printf(format string, v ...interface{}) 6 6 + Println(v ...interface{}) 7 7 + } 8 8 + 9 9 + const ( 10 10 + // BUNDLE_SIZE is the standard number of operations per bundle 11 11 + BUNDLE_SIZE = 10000 12 12 + 13 13 + // INDEX_FILE is the default index filename 14 14 + INDEX_FILE = "plc_bundles.json" 15 15 + 16 16 + // INDEX_VERSION is the current index format version 17 17 + INDEX_VERSION = "1.0" 18 18 + )

-148

plcbundle.go

··· 1 1 - package plcbundle 2 2 - 3 3 - import ( 4 4 - "context" 5 5 - "io" 6 6 - "time" 7 7 - 8 8 - "tangled.org/atscan.net/plcbundle/bundle" 9 9 - "tangled.org/atscan.net/plcbundle/plcclient" 10 10 - ) 11 11 - 12 12 - // Re-export commonly used types for convenience 13 13 - type ( 14 14 - Bundle = bundle.Bundle 15 15 - BundleMetadata = bundle.BundleMetadata 16 16 - Index = bundle.Index 17 17 - Manager = bundle.Manager 18 18 - Config = bundle.Config 19 19 - VerificationResult = bundle.VerificationResult 20 20 - ChainVerificationResult = bundle.ChainVerificationResult 21 21 - DirectoryScanResult = bundle.DirectoryScanResult 22 22 - Logger = bundle.Logger 23 23 - 24 24 - PLCOperation = plcclient.PLCOperation 25 25 - PLCClient = plcclient.Client 26 26 - ExportOptions = plcclient.ExportOptions 27 27 - ) 28 28 - 29 29 - // Re-export constants 30 30 - const ( 31 31 - BUNDLE_SIZE = bundle.BUNDLE_SIZE 32 32 - INDEX_FILE = bundle.INDEX_FILE 33 33 - ) 34 34 - 35 35 - // NewManager creates a new bundle manager (convenience wrapper) 36 36 - func NewManager(config *Config, plcClient *PLCClient) (*Manager, error) { 37 37 - return bundle.NewManager(config, plcClient) 38 38 - } 39 39 - 40 40 - // NewPLCClient creates a new PLC client (convenience wrapper) 41 41 - func NewPLCClient(baseURL string, opts ...plcclient.ClientOption) *PLCClient { 42 42 - return plcclient.NewClient(baseURL, opts...) 43 43 - } 44 44 - 45 45 - // DefaultConfig returns default configuration (convenience wrapper) 46 46 - func DefaultConfig(bundleDir string) *Config { 47 47 - return bundle.DefaultConfig(bundleDir) 48 48 - } 49 49 - 50 50 - // NewIndex creates a new empty index (convenience wrapper) 51 51 - func NewIndex(origin string) *Index { 52 52 - return bundle.NewIndex(origin) 53 53 - } 54 54 - 55 55 - // LoadIndex loads an index from a file (convenience wrapper) 56 56 - func LoadIndex(path string) (*Index, error) { 57 57 - return bundle.LoadIndex(path) 58 58 - } 59 59 - 60 60 - // BundleManager provides a high-level API for bundle operations 61 61 - type BundleManager struct { 62 62 - mgr *Manager 63 63 - } 64 64 - 65 65 - // New creates a new BundleManager with default settings 66 66 - func New(bundleDir string, plcURL string) (*BundleManager, error) { 67 67 - config := DefaultConfig(bundleDir) 68 68 - var plcClient *PLCClient 69 69 - if plcURL != "" { 70 70 - plcClient = NewPLCClient(plcURL) 71 71 - } 72 72 - 73 73 - mgr, err := NewManager(config, plcClient) 74 74 - if err != nil { 75 75 - return nil, err 76 76 - } 77 77 - 78 78 - return &BundleManager{mgr: mgr}, nil 79 79 - } 80 80 - 81 81 - // Close closes the manager 82 82 - func (bm *BundleManager) Close() { 83 83 - bm.mgr.Close() 84 84 - } 85 85 - 86 86 - // FetchNext fetches the next bundle from PLC 87 87 - func (bm *BundleManager) FetchNext(ctx context.Context) (*Bundle, error) { 88 88 - b, err := bm.mgr.FetchNextBundle(ctx, false) 89 89 - if err != nil { 90 90 - return nil, err 91 91 - } 92 92 - return b, bm.mgr.SaveBundle(ctx, b, false) 93 93 - } 94 94 - 95 95 - // Load loads a bundle by number 96 96 - func (bm *BundleManager) Load(ctx context.Context, bundleNumber int) (*Bundle, error) { 97 97 - return bm.mgr.LoadBundle(ctx, bundleNumber) 98 98 - } 99 99 - 100 100 - // Verify verifies a bundle 101 101 - func (bm *BundleManager) Verify(ctx context.Context, bundleNumber int) (*VerificationResult, error) { 102 102 - return bm.mgr.VerifyBundle(ctx, bundleNumber) 103 103 - } 104 104 - 105 105 - // VerifyChain verifies the entire chain 106 106 - func (bm *BundleManager) VerifyChain(ctx context.Context) (*ChainVerificationResult, error) { 107 107 - return bm.mgr.VerifyChain(ctx) 108 108 - } 109 109 - 110 110 - // GetIndex returns the index 111 111 - func (bm *BundleManager) GetIndex() *Index { 112 112 - return bm.mgr.GetIndex() 113 113 - } 114 114 - 115 115 - // GetInfo returns manager info 116 116 - func (bm *BundleManager) GetInfo() map[string]interface{} { 117 117 - return bm.mgr.GetInfo() 118 118 - } 119 119 - 120 120 - // Export exports operations from bundles 121 121 - func (bm *BundleManager) Export(ctx context.Context, afterTime time.Time, count int) ([]PLCOperation, error) { 122 122 - return bm.mgr.ExportOperations(ctx, afterTime, count) 123 123 - } 124 124 - 125 125 - // Scan scans the directory and rebuilds the index 126 126 - func (bm *BundleManager) Scan() (*DirectoryScanResult, error) { 127 127 - return bm.mgr.ScanDirectory() 128 128 - } 129 129 - 130 130 - // ScanBundle scans a single bundle file 131 131 - func (bm *BundleManager) ScanBundle(path string, bundleNumber int) (*BundleMetadata, error) { 132 132 - return bm.mgr.ScanBundle(path, bundleNumber) 133 133 - } 134 134 - 135 135 - // IsBundleIndexed checks if a bundle is in the index 136 136 - func (bm *BundleManager) IsBundleIndexed(bundleNumber int) bool { 137 137 - return bm.mgr.IsBundleIndexed(bundleNumber) 138 138 - } 139 139 - 140 140 - // StreamRaw streams raw compressed bundle data 141 141 - func (bm *BundleManager) StreamRaw(ctx context.Context, bundleNumber int) (io.ReadCloser, error) { 142 142 - return bm.mgr.StreamBundleRaw(ctx, bundleNumber) 143 143 - } 144 144 - 145 145 - // StreamDecompressed streams decompressed bundle data 146 146 - func (bm *BundleManager) StreamDecompressed(ctx context.Context, bundleNumber int) (io.ReadCloser, error) { 147 147 - return bm.mgr.StreamBundleDecompressed(ctx, bundleNumber) 148 148 - }