angrydutchman.peedee.es
forked from
atscan.net/plcbundle
A Transparent and Verifiable Way to Sync the AT Protocol's PLC Directory
compare cmd
tree.fail
+364
cmd/plcbundle/compare.go
+364
cmd/plcbundle/compare.go
···
···
1
+
package main
2
+
3
+
import (
4
+
"encoding/json"
5
+
"fmt"
6
+
"io"
7
+
"net/http"
8
+
"os"
9
+
"path/filepath"
10
+
"sort"
11
+
"strings"
12
+
"time"
13
+
14
+
"github.com/atscan/plcbundle/bundle"
15
+
)
16
+
17
+
// IndexComparison holds comparison results
18
+
type IndexComparison struct {
19
+
LocalCount int
20
+
TargetCount int
21
+
CommonCount int
22
+
MissingBundles []int // In target but not in local
23
+
ExtraBundles []int // In local but not in target
24
+
HashMismatches []HashMismatch
25
+
LocalRange [2]int
26
+
TargetRange [2]int
27
+
LocalTotalSize int64
28
+
TargetTotalSize int64
29
+
LocalUpdated time.Time
30
+
TargetUpdated time.Time
31
+
}
32
+
33
+
type HashMismatch struct {
34
+
BundleNumber int
35
+
LocalHash string
36
+
TargetHash string
37
+
}
38
+
39
+
func (ic *IndexComparison) HasDifferences() bool {
40
+
return len(ic.MissingBundles) > 0 || len(ic.ExtraBundles) > 0 || len(ic.HashMismatches) > 0
41
+
}
42
+
43
+
// loadTargetIndex loads an index from a file or URL
44
+
func loadTargetIndex(target string) (*bundle.Index, error) {
45
+
if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") {
46
+
// Load from URL
47
+
return loadIndexFromURL(target)
48
+
}
49
+
50
+
// Load from file
51
+
return bundle.LoadIndex(target)
52
+
}
53
+
54
+
// loadIndexFromURL downloads and parses an index from a URL
55
+
func loadIndexFromURL(url string) (*bundle.Index, error) {
56
+
client := &http.Client{
57
+
Timeout: 30 * time.Second,
58
+
}
59
+
60
+
resp, err := client.Get(url)
61
+
if err != nil {
62
+
return nil, fmt.Errorf("failed to download: %w", err)
63
+
}
64
+
defer resp.Body.Close()
65
+
66
+
if resp.StatusCode != http.StatusOK {
67
+
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
68
+
}
69
+
70
+
data, err := io.ReadAll(resp.Body)
71
+
if err != nil {
72
+
return nil, fmt.Errorf("failed to read response: %w", err)
73
+
}
74
+
75
+
var idx bundle.Index
76
+
if err := json.Unmarshal(data, &idx); err != nil {
77
+
return nil, fmt.Errorf("failed to parse index: %w", err)
78
+
}
79
+
80
+
return &idx, nil
81
+
}
82
+
83
+
// compareIndexes compares two indexes
84
+
func compareIndexes(local, target *bundle.Index) *IndexComparison {
85
+
localBundles := local.GetBundles()
86
+
targetBundles := target.GetBundles()
87
+
88
+
// Create maps for quick lookup
89
+
localMap := make(map[int]*bundle.BundleMetadata)
90
+
targetMap := make(map[int]*bundle.BundleMetadata)
91
+
92
+
for _, b := range localBundles {
93
+
localMap[b.BundleNumber] = b
94
+
}
95
+
for _, b := range targetBundles {
96
+
targetMap[b.BundleNumber] = b
97
+
}
98
+
99
+
comparison := &IndexComparison{
100
+
LocalCount: len(localBundles),
101
+
TargetCount: len(targetBundles),
102
+
MissingBundles: make([]int, 0),
103
+
ExtraBundles: make([]int, 0),
104
+
HashMismatches: make([]HashMismatch, 0),
105
+
}
106
+
107
+
// Get ranges
108
+
if len(localBundles) > 0 {
109
+
comparison.LocalRange = [2]int{localBundles[0].BundleNumber, localBundles[len(localBundles)-1].BundleNumber}
110
+
comparison.LocalUpdated = local.UpdatedAt
111
+
localStats := local.GetStats()
112
+
comparison.LocalTotalSize = localStats["total_size"].(int64)
113
+
}
114
+
115
+
if len(targetBundles) > 0 {
116
+
comparison.TargetRange = [2]int{targetBundles[0].BundleNumber, targetBundles[len(targetBundles)-1].BundleNumber}
117
+
comparison.TargetUpdated = target.UpdatedAt
118
+
targetStats := target.GetStats()
119
+
comparison.TargetTotalSize = targetStats["total_size"].(int64)
120
+
}
121
+
122
+
// Find missing bundles (in target but not in local)
123
+
for bundleNum := range targetMap {
124
+
if _, exists := localMap[bundleNum]; !exists {
125
+
comparison.MissingBundles = append(comparison.MissingBundles, bundleNum)
126
+
}
127
+
}
128
+
sort.Ints(comparison.MissingBundles)
129
+
130
+
// Find extra bundles (in local but not in target)
131
+
for bundleNum := range localMap {
132
+
if _, exists := targetMap[bundleNum]; !exists {
133
+
comparison.ExtraBundles = append(comparison.ExtraBundles, bundleNum)
134
+
}
135
+
}
136
+
sort.Ints(comparison.ExtraBundles)
137
+
138
+
// Find hash mismatches
139
+
for bundleNum, localMeta := range localMap {
140
+
if targetMeta, exists := targetMap[bundleNum]; exists {
141
+
comparison.CommonCount++
142
+
if localMeta.CompressedHash != targetMeta.CompressedHash {
143
+
comparison.HashMismatches = append(comparison.HashMismatches, HashMismatch{
144
+
BundleNumber: bundleNum,
145
+
LocalHash: localMeta.CompressedHash,
146
+
TargetHash: targetMeta.CompressedHash,
147
+
})
148
+
}
149
+
}
150
+
}
151
+
152
+
return comparison
153
+
}
154
+
155
+
// displayComparison displays the comparison results
156
+
func displayComparison(c *IndexComparison, verbose bool) {
157
+
fmt.Printf("Comparison Results\n")
158
+
fmt.Printf("══════════════════\n\n")
159
+
160
+
// Summary
161
+
fmt.Printf("Summary\n")
162
+
fmt.Printf("───────\n")
163
+
fmt.Printf(" Local bundles: %d\n", c.LocalCount)
164
+
fmt.Printf(" Target bundles: %d\n", c.TargetCount)
165
+
fmt.Printf(" Common bundles: %d\n", c.CommonCount)
166
+
fmt.Printf(" Missing bundles: %d\n", len(c.MissingBundles))
167
+
fmt.Printf(" Extra bundles: %d\n", len(c.ExtraBundles))
168
+
fmt.Printf(" Hash mismatches: %d\n", len(c.HashMismatches))
169
+
170
+
if c.LocalCount > 0 {
171
+
fmt.Printf("\n Local range: %06d - %06d\n", c.LocalRange[0], c.LocalRange[1])
172
+
fmt.Printf(" Local size: %.2f MB\n", float64(c.LocalTotalSize)/(1024*1024))
173
+
fmt.Printf(" Local updated: %s\n", c.LocalUpdated.Format("2006-01-02 15:04:05"))
174
+
}
175
+
176
+
if c.TargetCount > 0 {
177
+
fmt.Printf("\n Target range: %06d - %06d\n", c.TargetRange[0], c.TargetRange[1])
178
+
fmt.Printf(" Target size: %.2f MB\n", float64(c.TargetTotalSize)/(1024*1024))
179
+
fmt.Printf(" Target updated: %s\n", c.TargetUpdated.Format("2006-01-02 15:04:05"))
180
+
}
181
+
182
+
// Missing bundles
183
+
if len(c.MissingBundles) > 0 {
184
+
fmt.Printf("\n")
185
+
fmt.Printf("Missing Bundles (in target but not local)\n")
186
+
fmt.Printf("──────────────────────────────────────────\n")
187
+
188
+
if verbose || len(c.MissingBundles) <= 20 {
189
+
// Show all or up to 20
190
+
displayCount := len(c.MissingBundles)
191
+
if displayCount > 20 && !verbose {
192
+
displayCount = 20
193
+
}
194
+
195
+
for i := 0; i < displayCount; i++ {
196
+
fmt.Printf(" %06d\n", c.MissingBundles[i])
197
+
}
198
+
199
+
if len(c.MissingBundles) > displayCount {
200
+
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.MissingBundles)-displayCount)
201
+
}
202
+
} else {
203
+
// Show ranges
204
+
displayBundleRanges(c.MissingBundles)
205
+
}
206
+
}
207
+
208
+
// Extra bundles
209
+
if len(c.ExtraBundles) > 0 {
210
+
fmt.Printf("\n")
211
+
fmt.Printf("Extra Bundles (in local but not target)\n")
212
+
fmt.Printf("────────────────────────────────────────\n")
213
+
214
+
if verbose || len(c.ExtraBundles) <= 20 {
215
+
displayCount := len(c.ExtraBundles)
216
+
if displayCount > 20 && !verbose {
217
+
displayCount = 20
218
+
}
219
+
220
+
for i := 0; i < displayCount; i++ {
221
+
fmt.Printf(" %06d\n", c.ExtraBundles[i])
222
+
}
223
+
224
+
if len(c.ExtraBundles) > displayCount {
225
+
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.ExtraBundles)-displayCount)
226
+
}
227
+
} else {
228
+
displayBundleRanges(c.ExtraBundles)
229
+
}
230
+
}
231
+
232
+
// Hash mismatches
233
+
if len(c.HashMismatches) > 0 {
234
+
fmt.Printf("\n")
235
+
fmt.Printf("Hash Mismatches\n")
236
+
fmt.Printf("───────────────\n")
237
+
238
+
displayCount := len(c.HashMismatches)
239
+
if displayCount > 10 && !verbose {
240
+
displayCount = 10
241
+
}
242
+
243
+
for i := 0; i < displayCount; i++ {
244
+
m := c.HashMismatches[i]
245
+
fmt.Printf(" Bundle %06d:\n", m.BundleNumber)
246
+
fmt.Printf(" Local: %s\n", m.LocalHash[:16]+"...")
247
+
fmt.Printf(" Target: %s\n", m.TargetHash[:16]+"...")
248
+
}
249
+
250
+
if len(c.HashMismatches) > displayCount {
251
+
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.HashMismatches)-displayCount)
252
+
}
253
+
}
254
+
255
+
// Final status
256
+
fmt.Printf("\n")
257
+
if !c.HasDifferences() {
258
+
fmt.Printf("✓ Indexes are identical\n")
259
+
} else {
260
+
fmt.Printf("✗ Indexes have differences\n")
261
+
}
262
+
}
263
+
264
+
// displayBundleRanges displays bundle numbers as ranges
265
+
func displayBundleRanges(bundles []int) {
266
+
if len(bundles) == 0 {
267
+
return
268
+
}
269
+
270
+
rangeStart := bundles[0]
271
+
rangeEnd := bundles[0]
272
+
273
+
for i := 1; i < len(bundles); i++ {
274
+
if bundles[i] == rangeEnd+1 {
275
+
rangeEnd = bundles[i]
276
+
} else {
277
+
// Print current range
278
+
if rangeStart == rangeEnd {
279
+
fmt.Printf(" %06d\n", rangeStart)
280
+
} else {
281
+
fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd)
282
+
}
283
+
rangeStart = bundles[i]
284
+
rangeEnd = bundles[i]
285
+
}
286
+
}
287
+
288
+
// Print last range
289
+
if rangeStart == rangeEnd {
290
+
fmt.Printf(" %06d\n", rangeStart)
291
+
} else {
292
+
fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd)
293
+
}
294
+
}
295
+
296
+
// fetchMissingBundles downloads missing bundles from target server
297
+
func fetchMissingBundles(mgr *bundle.Manager, baseURL string, missingBundles []int) {
298
+
client := &http.Client{
299
+
Timeout: 60 * time.Second,
300
+
}
301
+
302
+
successCount := 0
303
+
errorCount := 0
304
+
305
+
for _, bundleNum := range missingBundles {
306
+
fmt.Printf("Fetching bundle %06d... ", bundleNum)
307
+
308
+
// Download bundle data
309
+
url := fmt.Sprintf("%s/data/%d", baseURL, bundleNum)
310
+
resp, err := client.Get(url)
311
+
if err != nil {
312
+
fmt.Printf("ERROR: %v\n", err)
313
+
errorCount++
314
+
continue
315
+
}
316
+
317
+
if resp.StatusCode != http.StatusOK {
318
+
fmt.Printf("ERROR: status %d\n", resp.StatusCode)
319
+
resp.Body.Close()
320
+
errorCount++
321
+
continue
322
+
}
323
+
324
+
// Save to file
325
+
filename := fmt.Sprintf("%06d.jsonl.zst", bundleNum)
326
+
filepath := filepath.Join(mgr.GetInfo()["bundle_dir"].(string), filename)
327
+
328
+
outFile, err := os.Create(filepath)
329
+
if err != nil {
330
+
fmt.Printf("ERROR: %v\n", err)
331
+
resp.Body.Close()
332
+
errorCount++
333
+
continue
334
+
}
335
+
336
+
_, err = io.Copy(outFile, resp.Body)
337
+
outFile.Close()
338
+
resp.Body.Close()
339
+
340
+
if err != nil {
341
+
fmt.Printf("ERROR: %v\n", err)
342
+
os.Remove(filepath)
343
+
errorCount++
344
+
continue
345
+
}
346
+
347
+
// Scan and index the bundle
348
+
_, err = mgr.ScanAndIndexBundle(filepath, bundleNum)
349
+
if err != nil {
350
+
fmt.Printf("ERROR: %v\n", err)
351
+
errorCount++
352
+
continue
353
+
}
354
+
355
+
fmt.Printf("✓\n")
356
+
successCount++
357
+
358
+
// Small delay to be nice
359
+
time.Sleep(200 * time.Millisecond)
360
+
}
361
+
362
+
fmt.Printf("\n")
363
+
fmt.Printf("✓ Fetch complete: %d succeeded, %d failed\n", successCount, errorCount)
364
+
}
+69
cmd/plcbundle/main.go
+69
cmd/plcbundle/main.go
···
47
cmdMempool()
48
case "serve":
49
cmdServe()
50
case "version":
51
fmt.Printf("plcbundle version %s\n", version)
52
fmt.Printf(" commit: %s\n", gitCommit)
···
73
backfill Fetch/load all bundles and stream to stdout
74
mempool Show mempool status and operations
75
serve Start HTTP server to serve bundle data
76
version Show version
77
78
The tool works with the current directory.
···
872
os.Exit(1)
873
}
874
}
···
47
cmdMempool()
48
case "serve":
49
cmdServe()
50
+
case "compare":
51
+
cmdCompare()
52
case "version":
53
fmt.Printf("plcbundle version %s\n", version)
54
fmt.Printf(" commit: %s\n", gitCommit)
···
75
backfill Fetch/load all bundles and stream to stdout
76
mempool Show mempool status and operations
77
serve Start HTTP server to serve bundle data
78
+
compare Compare local index with target index
79
version Show version
80
81
The tool works with the current directory.
···
875
os.Exit(1)
876
}
877
}
878
+
879
+
func cmdCompare() {
880
+
fs := flag.NewFlagSet("compare", flag.ExitOnError)
881
+
verbose := fs.Bool("v", false, "verbose output (show all differences)")
882
+
fetchMissing := fs.Bool("fetch-missing", false, "fetch missing bundles from target")
883
+
fs.Parse(os.Args[2:])
884
+
885
+
if fs.NArg() < 1 {
886
+
fmt.Fprintf(os.Stderr, "Usage: plcbundle compare <target>\n")
887
+
fmt.Fprintf(os.Stderr, " target: path to plc_bundles.json or URL\n")
888
+
fmt.Fprintf(os.Stderr, "\nExamples:\n")
889
+
fmt.Fprintf(os.Stderr, " plcbundle compare /path/to/plc_bundles.json\n")
890
+
fmt.Fprintf(os.Stderr, " plcbundle compare https://example.com/index.json\n")
891
+
fmt.Fprintf(os.Stderr, " plcbundle compare https://example.com/index.json --fetch-missing\n")
892
+
os.Exit(1)
893
+
}
894
+
895
+
target := fs.Arg(0)
896
+
897
+
mgr, dir, err := getManager("")
898
+
if err != nil {
899
+
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
900
+
os.Exit(1)
901
+
}
902
+
defer mgr.Close()
903
+
904
+
fmt.Printf("Comparing: %s\n", dir)
905
+
fmt.Printf(" Against: %s\n\n", target)
906
+
907
+
// Load local index
908
+
localIndex := mgr.GetIndex()
909
+
910
+
// Load target index
911
+
fmt.Printf("Loading target index...\n")
912
+
targetIndex, err := loadTargetIndex(target)
913
+
if err != nil {
914
+
fmt.Fprintf(os.Stderr, "Error loading target index: %v\n", err)
915
+
os.Exit(1)
916
+
}
917
+
918
+
// Perform comparison
919
+
comparison := compareIndexes(localIndex, targetIndex)
920
+
921
+
// Display results
922
+
displayComparison(comparison, *verbose)
923
+
924
+
// Fetch missing bundles if requested
925
+
if *fetchMissing && len(comparison.MissingBundles) > 0 {
926
+
fmt.Printf("\n")
927
+
if !strings.HasPrefix(target, "http://") && !strings.HasPrefix(target, "https://") {
928
+
fmt.Fprintf(os.Stderr, "Error: --fetch-missing only works with remote URLs\n")
929
+
os.Exit(1)
930
+
}
931
+
932
+
baseURL := strings.TrimSuffix(target, "/index.json")
933
+
baseURL = strings.TrimSuffix(baseURL, "/plc_bundles.json")
934
+
935
+
fmt.Printf("Fetching %d missing bundles...\n\n", len(comparison.MissingBundles))
936
+
fetchMissingBundles(mgr, baseURL, comparison.MissingBundles)
937
+
}
938
+
939
+
// Exit with error if there are differences
940
+
if comparison.HasDifferences() {
941
+
os.Exit(1)
942
+
}
943
+
}