+58 -17

bundle/manager.go

··· 1076 1076 return result, nil 1077 1077 } 1078 1078 1079 1079 - // FetchNextBundle delegates to sync.Fetcher 1079 1079 + // FetchNextBundle fetches operations and creates a bundle, looping until caught up 1080 1080 func (m *Manager) FetchNextBundle(ctx context.Context, quiet bool) (*Bundle, error) { 1081 1081 if m.plcClient == nil { 1082 1082 return nil, fmt.Errorf("PLC client not configured") ··· 1115 1115 m.logger.Printf("Preparing bundle %06d (mempool: %d ops)...", nextBundleNum, m.mempool.Count()) 1116 1116 } 1117 1117 1118 1118 - // ✨ Fetch operations if needed (FetchToMempool loops internally) 1119 1119 - if m.mempool.Count() < types.BUNDLE_SIZE { 1118 1118 + // ✨ NEW: Loop until we have enough OR catch up to latest 1119 1119 + maxAttempts := 50 // Safety limit 1120 1120 + attempt := 0 1121 1121 + caughtUp := false 1122 1122 + 1123 1123 + for m.mempool.Count() < types.BUNDLE_SIZE && attempt < maxAttempts { 1124 1124 + attempt++ 1125 1125 + 1126 1126 + needed := types.BUNDLE_SIZE - m.mempool.Count() 1127 1127 + 1128 1128 + if !quiet && attempt > 1 { 1129 1129 + m.logger.Printf(" Attempt %d: Need %d more ops...", attempt, needed) 1130 1130 + } 1131 1131 + 1120 1132 newOps, err := m.syncer.FetchToMempool( 1121 1133 ctx, 1122 1134 afterTime, 1123 1135 prevBoundaryCIDs, 1124 1124 - types.BUNDLE_SIZE-m.mempool.Count(), 1136 1136 + needed, 1125 1137 quiet, 1126 1138 m.mempool.Count(), 1127 1139 ) ··· 1135 1147 } 1136 1148 1137 1149 if !quiet && added > 0 { 1138 1138 - m.logger.Printf("Added %d new operations (mempool now: %d)", added, m.mempool.Count()) 1150 1150 + m.logger.Printf(" Added %d new operations (mempool now: %d)", added, m.mempool.Count()) 1151 1151 + } 1152 1152 + 1153 1153 + // Update cursor for next fetch 1154 1154 + if len(newOps) > 0 { 1155 1155 + afterTime = newOps[len(newOps)-1].CreatedAt.Format(time.RFC3339Nano) 1156 1156 + } 1157 1157 + } 1158 1158 + 1159 1159 + // Check if we caught up (got incomplete batch) 1160 1160 + if err != nil || len(newOps) == 0 { 1161 1161 + caughtUp = true 1162 1162 + if !quiet { 1163 1163 + m.logger.Printf(" Caught up to latest PLC data") 1164 1164 + } 1165 1165 + break 1166 1166 + } 1167 1167 + 1168 1168 + // If we got a full batch but still need more, continue looping 1169 1169 + if len(newOps) > 0 && m.mempool.Count() < types.BUNDLE_SIZE { 1170 1170 + if !quiet { 1171 1171 + m.logger.Printf(" Got full batch, fetching more...") 1139 1172 } 1173 1173 + continue 1140 1174 } 1141 1175 1142 1142 - // If fetch failed AND we don't have enough, return error 1143 1143 - if err != nil && m.mempool.Count() < types.BUNDLE_SIZE { 1144 1144 - m.mempool.Save() 1145 1145 - return nil, err 1176 1176 + // If we have enough, break 1177 1177 + if m.mempool.Count() >= types.BUNDLE_SIZE { 1178 1178 + break 1146 1179 } 1147 1180 } 1148 1181 1182 1182 + // Save mempool state 1183 1183 + m.mempool.Save() 1184 1184 + 1149 1185 // Check if we have enough for a bundle 1150 1186 if m.mempool.Count() < types.BUNDLE_SIZE { 1151 1151 - m.mempool.Save() 1152 1152 - return nil, fmt.Errorf("insufficient operations: have %d, need %d (no more available)", 1153 1153 - m.mempool.Count(), types.BUNDLE_SIZE) 1187 1187 + if caughtUp { 1188 1188 + return nil, fmt.Errorf("insufficient operations: have %d, need %d (caught up to latest PLC data)", 1189 1189 + m.mempool.Count(), types.BUNDLE_SIZE) 1190 1190 + } else { 1191 1191 + return nil, fmt.Errorf("insufficient operations: have %d, need %d (max attempts reached)", 1192 1192 + m.mempool.Count(), types.BUNDLE_SIZE) 1193 1193 + } 1154 1194 } 1155 1195 1156 1196 // Create bundle 1157 1197 operations, err := m.mempool.Take(types.BUNDLE_SIZE) 1158 1198 if err != nil { 1159 1159 - m.mempool.Save() 1160 1199 return nil, err 1161 1200 } 1162 1201 1163 1163 - // Create bundle structure directly 1202 1202 + // Create bundle structure 1164 1203 syncBundle := internalsync.CreateBundle(nextBundleNum, operations, afterTime, prevBundleHash, m.operations) 1165 1204 1166 1166 - // Convert from sync.Bundle to bundle.Bundle inline 1205 1205 + // Convert to bundle.Bundle 1167 1206 bundle := &Bundle{ 1168 1207 BundleNumber: syncBundle.BundleNumber, 1169 1208 StartTime: syncBundle.StartTime, ··· 1175 1214 BoundaryCIDs: syncBundle.BoundaryCIDs, 1176 1215 Compressed: syncBundle.Compressed, 1177 1216 CreatedAt: syncBundle.CreatedAt, 1178 1178 - // Note: Hash fields are empty here, will be calculated in SaveBundle 1179 1217 } 1180 1218 1181 1181 - m.mempool.Save() 1219 1219 + if !quiet { 1220 1220 + m.logger.Printf("✓ Bundle %06d ready (%d ops, %d DIDs)", 1221 1221 + bundle.BundleNumber, len(bundle.Operations), bundle.DIDCount) 1222 1222 + } 1182 1223 1183 1224 return bundle, nil 1184 1225 }

+9 -33

cmd/plcbundle/commands/server.go

··· 310 310 startBundle = lastBundle.BundleNumber + 1 311 311 } 312 312 313 313 - isInitialSync := (lastBundle == nil || lastBundle.BundleNumber < 10) 314 314 - 315 315 - if isInitialSync && !verbose { 316 316 - fmt.Fprintf(os.Stderr, "[Sync] Initial sync - fast loading mode (bundle %06d → ...)\n", startBundle) 317 317 - } else if verbose { 318 318 - fmt.Fprintf(os.Stderr, "[Sync] Checking for new bundles (current: %06d)...\n", startBundle-1) 319 319 - } 320 320 - 321 313 mempoolBefore := mgr.GetMempoolStats()["count"].(int) 322 314 fetchedCount := 0 323 323 - consecutiveErrors := 0 324 315 316 316 + // ✨ SIMPLIFIED: Just keep calling FetchNextBundle until it fails 325 317 for { 326 326 - currentBundle := startBundle + fetchedCount 327 327 - 328 318 b, err := mgr.FetchNextBundle(ctx, !verbose) 329 319 if err != nil { 330 320 if isEndOfDataError(err) { ··· 334 324 335 325 if fetchedCount > 0 { 336 326 fmt.Fprintf(os.Stderr, "[Sync] ✓ Bundle %06d | Synced: %d | Mempool: %d (+%d) | %dms\n", 337 337 - currentBundle-1, fetchedCount, mempoolAfter, addedOps, duration.Milliseconds()) 338 338 - } else if !isInitialSync { 327 327 + startBundle+fetchedCount-1, fetchedCount, mempoolAfter, addedOps, duration.Milliseconds()) 328 328 + } else { 339 329 fmt.Fprintf(os.Stderr, "[Sync] ✓ Bundle %06d | Up to date | Mempool: %d (+%d) | %dms\n", 340 330 startBundle-1, mempoolAfter, addedOps, duration.Milliseconds()) 341 331 } 342 332 break 343 333 } 344 334 345 345 - consecutiveErrors++ 346 346 - if verbose { 347 347 - fmt.Fprintf(os.Stderr, "[Sync] Error fetching bundle %06d: %v\n", currentBundle, err) 348 348 - } 349 349 - 350 350 - if consecutiveErrors >= 3 { 351 351 - fmt.Fprintf(os.Stderr, "[Sync] Too many errors, stopping\n") 352 352 - break 353 353 - } 354 354 - 355 355 - time.Sleep(5 * time.Second) 356 356 - continue 335 335 + // Real error 336 336 + fmt.Fprintf(os.Stderr, "[Sync] Error: %v\n", err) 337 337 + break 357 338 } 358 339 359 359 - consecutiveErrors = 0 360 360 - 340 340 + // Save bundle 361 341 if err := mgr.SaveBundle(ctx, b, !verbose); err != nil { 362 342 fmt.Fprintf(os.Stderr, "[Sync] Error saving bundle %06d: %v\n", b.BundleNumber, err) 363 343 break ··· 366 346 fetchedCount++ 367 347 368 348 if !verbose { 369 369 - fmt.Fprintf(os.Stderr, "[Sync] ✓ %06d | hash=%s | content=%s | %d ops, %d DIDs\n", 370 370 - b.BundleNumber, 371 371 - b.Hash[:16]+"...", 372 372 - b.ContentHash[:16]+"...", 373 373 - len(b.Operations), 374 374 - b.DIDCount) 349 349 + fmt.Fprintf(os.Stderr, "[Sync] ✓ %06d | %d ops, %d DIDs\n", 350 350 + b.BundleNumber, len(b.Operations), b.DIDCount) 375 351 } 376 352 377 353 time.Sleep(500 * time.Millisecond)

+23 -13

internal/sync/fetcher.go

··· 27 27 } 28 28 29 29 // FetchToMempool fetches operations and returns them 30 30 + // Returns empty slice + nil error when caught up (incomplete batch received) 30 31 func (f *Fetcher) FetchToMempool( 31 32 ctx context.Context, 32 33 afterTime string, ··· 59 60 } 60 61 61 62 if !quiet { 62 62 - f.logger.Printf(" Fetch #%d: requesting %d operations", fetchNum+1, batchSize) 63 63 + f.logger.Printf(" Fetch #%d: requesting %d operations (after: %s)", 64 64 + fetchNum+1, batchSize, currentAfter[:19]) 63 65 } 64 66 65 67 batch, err := f.plcClient.Export(ctx, plcclient.ExportOptions{ 66 68 Count: batchSize, 67 69 After: currentAfter, 68 70 }) 71 71 + 69 72 if err != nil { 70 73 return allNewOps, fmt.Errorf("export failed: %w", err) 71 74 } ··· 74 77 if !quiet { 75 78 f.logger.Printf(" No more operations available from PLC") 76 79 } 77 77 - break 80 80 + // Return what we have (might be incomplete) 81 81 + return allNewOps, nil 78 82 } 79 83 80 80 - // Deduplicate against boundary CIDs only 81 81 - // Mempool will handle deduplication of operations already in mempool 84 84 + // Deduplicate 85 85 + beforeDedup := len(allNewOps) 82 86 for _, op := range batch { 83 87 if !seenCIDs[op.CID] { 84 88 seenCIDs[op.CID] = true ··· 86 90 } 87 91 } 88 92 93 93 + if !quiet && len(batch) > 0 { 94 94 + deduped := len(batch) - (len(allNewOps) - beforeDedup) 95 95 + if deduped > 0 { 96 96 + f.logger.Printf(" Received %d ops (%d duplicates filtered)", len(batch), deduped) 97 97 + } 98 98 + } 99 99 + 89 100 // Update cursor 90 101 if len(batch) > 0 { 91 102 currentAfter = batch[len(batch)-1].CreatedAt.Format(time.RFC3339Nano) 92 103 } 93 104 94 94 - // Stop if we got less than requested (caught up) 105 105 + // ✨ KEY: Stop if we got incomplete batch (caught up!) 95 106 if len(batch) < batchSize { 96 107 if !quiet { 97 97 - f.logger.Printf(" Received incomplete batch (%d/%d), caught up to latest", len(batch), batchSize) 108 108 + f.logger.Printf(" Received incomplete batch (%d/%d) → caught up to latest", 109 109 + len(batch), batchSize) 98 110 } 99 99 - break 111 111 + return allNewOps, nil 100 112 } 101 101 - } 102 113 103 103 - if len(allNewOps) > 0 { 104 104 - if !quiet { 105 105 - f.logger.Printf("✓ Fetch complete: %d operations", len(allNewOps)) 114 114 + // If we have enough, stop 115 115 + if len(allNewOps) >= target { 116 116 + break 106 117 } 107 107 - return allNewOps, nil 108 118 } 109 119 110 110 - return nil, fmt.Errorf("no operations available (reached latest data)") 120 120 + return allNewOps, nil 111 121 }