angrydutchman.peedee.es
forked from
atscan.net/plcbundle
A Transparent and Verifiable Way to Sync the AT Protocol's PLC Directory
compare cmd
tree.fail
+364
cmd/plcbundle/compare.go
+364
cmd/plcbundle/compare.go
···
1
+
package main
2
+
3
+
import (
4
+
"encoding/json"
5
+
"fmt"
6
+
"io"
7
+
"net/http"
8
+
"os"
9
+
"path/filepath"
10
+
"sort"
11
+
"strings"
12
+
"time"
13
+
14
+
"github.com/atscan/plcbundle/bundle"
15
+
)
16
+
17
+
// IndexComparison holds comparison results
18
+
type IndexComparison struct {
19
+
LocalCount int
20
+
TargetCount int
21
+
CommonCount int
22
+
MissingBundles []int // In target but not in local
23
+
ExtraBundles []int // In local but not in target
24
+
HashMismatches []HashMismatch
25
+
LocalRange [2]int
26
+
TargetRange [2]int
27
+
LocalTotalSize int64
28
+
TargetTotalSize int64
29
+
LocalUpdated time.Time
30
+
TargetUpdated time.Time
31
+
}
32
+
33
+
type HashMismatch struct {
34
+
BundleNumber int
35
+
LocalHash string
36
+
TargetHash string
37
+
}
38
+
39
+
func (ic *IndexComparison) HasDifferences() bool {
40
+
return len(ic.MissingBundles) > 0 || len(ic.ExtraBundles) > 0 || len(ic.HashMismatches) > 0
41
+
}
42
+
43
+
// loadTargetIndex loads an index from a file or URL
44
+
func loadTargetIndex(target string) (*bundle.Index, error) {
45
+
if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") {
46
+
// Load from URL
47
+
return loadIndexFromURL(target)
48
+
}
49
+
50
+
// Load from file
51
+
return bundle.LoadIndex(target)
52
+
}
53
+
54
+
// loadIndexFromURL downloads and parses an index from a URL
55
+
func loadIndexFromURL(url string) (*bundle.Index, error) {
56
+
client := &http.Client{
57
+
Timeout: 30 * time.Second,
58
+
}
59
+
60
+
resp, err := client.Get(url)
61
+
if err != nil {
62
+
return nil, fmt.Errorf("failed to download: %w", err)
63
+
}
64
+
defer resp.Body.Close()
65
+
66
+
if resp.StatusCode != http.StatusOK {
67
+
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
68
+
}
69
+
70
+
data, err := io.ReadAll(resp.Body)
71
+
if err != nil {
72
+
return nil, fmt.Errorf("failed to read response: %w", err)
73
+
}
74
+
75
+
var idx bundle.Index
76
+
if err := json.Unmarshal(data, &idx); err != nil {
77
+
return nil, fmt.Errorf("failed to parse index: %w", err)
78
+
}
79
+
80
+
return &idx, nil
81
+
}
82
+
83
+
// compareIndexes compares two indexes
84
+
func compareIndexes(local, target *bundle.Index) *IndexComparison {
85
+
localBundles := local.GetBundles()
86
+
targetBundles := target.GetBundles()
87
+
88
+
// Create maps for quick lookup
89
+
localMap := make(map[int]*bundle.BundleMetadata)
90
+
targetMap := make(map[int]*bundle.BundleMetadata)
91
+
92
+
for _, b := range localBundles {
93
+
localMap[b.BundleNumber] = b
94
+
}
95
+
for _, b := range targetBundles {
96
+
targetMap[b.BundleNumber] = b
97
+
}
98
+
99
+
comparison := &IndexComparison{
100
+
LocalCount: len(localBundles),
101
+
TargetCount: len(targetBundles),
102
+
MissingBundles: make([]int, 0),
103
+
ExtraBundles: make([]int, 0),
104
+
HashMismatches: make([]HashMismatch, 0),
105
+
}
106
+
107
+
// Get ranges
108
+
if len(localBundles) > 0 {
109
+
comparison.LocalRange = [2]int{localBundles[0].BundleNumber, localBundles[len(localBundles)-1].BundleNumber}
110
+
comparison.LocalUpdated = local.UpdatedAt
111
+
localStats := local.GetStats()
112
+
comparison.LocalTotalSize = localStats["total_size"].(int64)
113
+
}
114
+
115
+
if len(targetBundles) > 0 {
116
+
comparison.TargetRange = [2]int{targetBundles[0].BundleNumber, targetBundles[len(targetBundles)-1].BundleNumber}
117
+
comparison.TargetUpdated = target.UpdatedAt
118
+
targetStats := target.GetStats()
119
+
comparison.TargetTotalSize = targetStats["total_size"].(int64)
120
+
}
121
+
122
+
// Find missing bundles (in target but not in local)
123
+
for bundleNum := range targetMap {
124
+
if _, exists := localMap[bundleNum]; !exists {
125
+
comparison.MissingBundles = append(comparison.MissingBundles, bundleNum)
126
+
}
127
+
}
128
+
sort.Ints(comparison.MissingBundles)
129
+
130
+
// Find extra bundles (in local but not in target)
131
+
for bundleNum := range localMap {
132
+
if _, exists := targetMap[bundleNum]; !exists {
133
+
comparison.ExtraBundles = append(comparison.ExtraBundles, bundleNum)
134
+
}
135
+
}
136
+
sort.Ints(comparison.ExtraBundles)
137
+
138
+
// Find hash mismatches
139
+
for bundleNum, localMeta := range localMap {
140
+
if targetMeta, exists := targetMap[bundleNum]; exists {
141
+
comparison.CommonCount++
142
+
if localMeta.CompressedHash != targetMeta.CompressedHash {
143
+
comparison.HashMismatches = append(comparison.HashMismatches, HashMismatch{
144
+
BundleNumber: bundleNum,
145
+
LocalHash: localMeta.CompressedHash,
146
+
TargetHash: targetMeta.CompressedHash,
147
+
})
148
+
}
149
+
}
150
+
}
151
+
152
+
return comparison
153
+
}
154
+
155
+
// displayComparison displays the comparison results
156
+
func displayComparison(c *IndexComparison, verbose bool) {
157
+
fmt.Printf("Comparison Results\n")
158
+
fmt.Printf("══════════════════\n\n")
159
+
160
+
// Summary
161
+
fmt.Printf("Summary\n")
162
+
fmt.Printf("───────\n")
163
+
fmt.Printf(" Local bundles: %d\n", c.LocalCount)
164
+
fmt.Printf(" Target bundles: %d\n", c.TargetCount)
165
+
fmt.Printf(" Common bundles: %d\n", c.CommonCount)
166
+
fmt.Printf(" Missing bundles: %d\n", len(c.MissingBundles))
167
+
fmt.Printf(" Extra bundles: %d\n", len(c.ExtraBundles))
168
+
fmt.Printf(" Hash mismatches: %d\n", len(c.HashMismatches))
169
+
170
+
if c.LocalCount > 0 {
171
+
fmt.Printf("\n Local range: %06d - %06d\n", c.LocalRange[0], c.LocalRange[1])
172
+
fmt.Printf(" Local size: %.2f MB\n", float64(c.LocalTotalSize)/(1024*1024))
173
+
fmt.Printf(" Local updated: %s\n", c.LocalUpdated.Format("2006-01-02 15:04:05"))
174
+
}
175
+
176
+
if c.TargetCount > 0 {
177
+
fmt.Printf("\n Target range: %06d - %06d\n", c.TargetRange[0], c.TargetRange[1])
178
+
fmt.Printf(" Target size: %.2f MB\n", float64(c.TargetTotalSize)/(1024*1024))
179
+
fmt.Printf(" Target updated: %s\n", c.TargetUpdated.Format("2006-01-02 15:04:05"))
180
+
}
181
+
182
+
// Missing bundles
183
+
if len(c.MissingBundles) > 0 {
184
+
fmt.Printf("\n")
185
+
fmt.Printf("Missing Bundles (in target but not local)\n")
186
+
fmt.Printf("──────────────────────────────────────────\n")
187
+
188
+
if verbose || len(c.MissingBundles) <= 20 {
189
+
// Show all or up to 20
190
+
displayCount := len(c.MissingBundles)
191
+
if displayCount > 20 && !verbose {
192
+
displayCount = 20
193
+
}
194
+
195
+
for i := 0; i < displayCount; i++ {
196
+
fmt.Printf(" %06d\n", c.MissingBundles[i])
197
+
}
198
+
199
+
if len(c.MissingBundles) > displayCount {
200
+
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.MissingBundles)-displayCount)
201
+
}
202
+
} else {
203
+
// Show ranges
204
+
displayBundleRanges(c.MissingBundles)
205
+
}
206
+
}
207
+
208
+
// Extra bundles
209
+
if len(c.ExtraBundles) > 0 {
210
+
fmt.Printf("\n")
211
+
fmt.Printf("Extra Bundles (in local but not target)\n")
212
+
fmt.Printf("────────────────────────────────────────\n")
213
+
214
+
if verbose || len(c.ExtraBundles) <= 20 {
215
+
displayCount := len(c.ExtraBundles)
216
+
if displayCount > 20 && !verbose {
217
+
displayCount = 20
218
+
}
219
+
220
+
for i := 0; i < displayCount; i++ {
221
+
fmt.Printf(" %06d\n", c.ExtraBundles[i])
222
+
}
223
+
224
+
if len(c.ExtraBundles) > displayCount {
225
+
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.ExtraBundles)-displayCount)
226
+
}
227
+
} else {
228
+
displayBundleRanges(c.ExtraBundles)
229
+
}
230
+
}
231
+
232
+
// Hash mismatches
233
+
if len(c.HashMismatches) > 0 {
234
+
fmt.Printf("\n")
235
+
fmt.Printf("Hash Mismatches\n")
236
+
fmt.Printf("───────────────\n")
237
+
238
+
displayCount := len(c.HashMismatches)
239
+
if displayCount > 10 && !verbose {
240
+
displayCount = 10
241
+
}
242
+
243
+
for i := 0; i < displayCount; i++ {
244
+
m := c.HashMismatches[i]
245
+
fmt.Printf(" Bundle %06d:\n", m.BundleNumber)
246
+
fmt.Printf(" Local: %s\n", m.LocalHash[:16]+"...")
247
+
fmt.Printf(" Target: %s\n", m.TargetHash[:16]+"...")
248
+
}
249
+
250
+
if len(c.HashMismatches) > displayCount {
251
+
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.HashMismatches)-displayCount)
252
+
}
253
+
}
254
+
255
+
// Final status
256
+
fmt.Printf("\n")
257
+
if !c.HasDifferences() {
258
+
fmt.Printf("✓ Indexes are identical\n")
259
+
} else {
260
+
fmt.Printf("✗ Indexes have differences\n")
261
+
}
262
+
}
263
+
264
+
// displayBundleRanges displays bundle numbers as ranges
265
+
func displayBundleRanges(bundles []int) {
266
+
if len(bundles) == 0 {
267
+
return
268
+
}
269
+
270
+
rangeStart := bundles[0]
271
+
rangeEnd := bundles[0]
272
+
273
+
for i := 1; i < len(bundles); i++ {
274
+
if bundles[i] == rangeEnd+1 {
275
+
rangeEnd = bundles[i]
276
+
} else {
277
+
// Print current range
278
+
if rangeStart == rangeEnd {
279
+
fmt.Printf(" %06d\n", rangeStart)
280
+
} else {
281
+
fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd)
282
+
}
283
+
rangeStart = bundles[i]
284
+
rangeEnd = bundles[i]
285
+
}
286
+
}
287
+
288
+
// Print last range
289
+
if rangeStart == rangeEnd {
290
+
fmt.Printf(" %06d\n", rangeStart)
291
+
} else {
292
+
fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd)
293
+
}
294
+
}
295
+
296
+
// fetchMissingBundles downloads missing bundles from target server
297
+
func fetchMissingBundles(mgr *bundle.Manager, baseURL string, missingBundles []int) {
298
+
client := &http.Client{
299
+
Timeout: 60 * time.Second,
300
+
}
301
+
302
+
successCount := 0
303
+
errorCount := 0
304
+
305
+
for _, bundleNum := range missingBundles {
306
+
fmt.Printf("Fetching bundle %06d... ", bundleNum)
307
+
308
+
// Download bundle data
309
+
url := fmt.Sprintf("%s/data/%d", baseURL, bundleNum)
310
+
resp, err := client.Get(url)
311
+
if err != nil {
312
+
fmt.Printf("ERROR: %v\n", err)
313
+
errorCount++
314
+
continue
315
+
}
316
+
317
+
if resp.StatusCode != http.StatusOK {
318
+
fmt.Printf("ERROR: status %d\n", resp.StatusCode)
319
+
resp.Body.Close()
320
+
errorCount++
321
+
continue
322
+
}
323
+
324
+
// Save to file
325
+
filename := fmt.Sprintf("%06d.jsonl.zst", bundleNum)
326
+
filepath := filepath.Join(mgr.GetInfo()["bundle_dir"].(string), filename)
327
+
328
+
outFile, err := os.Create(filepath)
329
+
if err != nil {
330
+
fmt.Printf("ERROR: %v\n", err)
331
+
resp.Body.Close()
332
+
errorCount++
333
+
continue
334
+
}
335
+
336
+
_, err = io.Copy(outFile, resp.Body)
337
+
outFile.Close()
338
+
resp.Body.Close()
339
+
340
+
if err != nil {
341
+
fmt.Printf("ERROR: %v\n", err)
342
+
os.Remove(filepath)
343
+
errorCount++
344
+
continue
345
+
}
346
+
347
+
// Scan and index the bundle
348
+
_, err = mgr.ScanAndIndexBundle(filepath, bundleNum)
349
+
if err != nil {
350
+
fmt.Printf("ERROR: %v\n", err)
351
+
errorCount++
352
+
continue
353
+
}
354
+
355
+
fmt.Printf("✓\n")
356
+
successCount++
357
+
358
+
// Small delay to be nice
359
+
time.Sleep(200 * time.Millisecond)
360
+
}
361
+
362
+
fmt.Printf("\n")
363
+
fmt.Printf("✓ Fetch complete: %d succeeded, %d failed\n", successCount, errorCount)
364
+
}
+69
cmd/plcbundle/main.go
+69
cmd/plcbundle/main.go
···
47
47
cmdMempool()
48
48
case "serve":
49
49
cmdServe()
50
+
case "compare":
51
+
cmdCompare()
50
52
case "version":
51
53
fmt.Printf("plcbundle version %s\n", version)
52
54
fmt.Printf(" commit: %s\n", gitCommit)
···
73
75
backfill Fetch/load all bundles and stream to stdout
74
76
mempool Show mempool status and operations
75
77
serve Start HTTP server to serve bundle data
78
+
compare Compare local index with target index
76
79
version Show version
77
80
78
81
The tool works with the current directory.
···
872
875
os.Exit(1)
873
876
}
874
877
}
878
+
879
+
func cmdCompare() {
880
+
fs := flag.NewFlagSet("compare", flag.ExitOnError)
881
+
verbose := fs.Bool("v", false, "verbose output (show all differences)")
882
+
fetchMissing := fs.Bool("fetch-missing", false, "fetch missing bundles from target")
883
+
fs.Parse(os.Args[2:])
884
+
885
+
if fs.NArg() < 1 {
886
+
fmt.Fprintf(os.Stderr, "Usage: plcbundle compare <target>\n")
887
+
fmt.Fprintf(os.Stderr, " target: path to plc_bundles.json or URL\n")
888
+
fmt.Fprintf(os.Stderr, "\nExamples:\n")
889
+
fmt.Fprintf(os.Stderr, " plcbundle compare /path/to/plc_bundles.json\n")
890
+
fmt.Fprintf(os.Stderr, " plcbundle compare https://example.com/index.json\n")
891
+
fmt.Fprintf(os.Stderr, " plcbundle compare https://example.com/index.json --fetch-missing\n")
892
+
os.Exit(1)
893
+
}
894
+
895
+
target := fs.Arg(0)
896
+
897
+
mgr, dir, err := getManager("")
898
+
if err != nil {
899
+
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
900
+
os.Exit(1)
901
+
}
902
+
defer mgr.Close()
903
+
904
+
fmt.Printf("Comparing: %s\n", dir)
905
+
fmt.Printf(" Against: %s\n\n", target)
906
+
907
+
// Load local index
908
+
localIndex := mgr.GetIndex()
909
+
910
+
// Load target index
911
+
fmt.Printf("Loading target index...\n")
912
+
targetIndex, err := loadTargetIndex(target)
913
+
if err != nil {
914
+
fmt.Fprintf(os.Stderr, "Error loading target index: %v\n", err)
915
+
os.Exit(1)
916
+
}
917
+
918
+
// Perform comparison
919
+
comparison := compareIndexes(localIndex, targetIndex)
920
+
921
+
// Display results
922
+
displayComparison(comparison, *verbose)
923
+
924
+
// Fetch missing bundles if requested
925
+
if *fetchMissing && len(comparison.MissingBundles) > 0 {
926
+
fmt.Printf("\n")
927
+
if !strings.HasPrefix(target, "http://") && !strings.HasPrefix(target, "https://") {
928
+
fmt.Fprintf(os.Stderr, "Error: --fetch-missing only works with remote URLs\n")
929
+
os.Exit(1)
930
+
}
931
+
932
+
baseURL := strings.TrimSuffix(target, "/index.json")
933
+
baseURL = strings.TrimSuffix(baseURL, "/plc_bundles.json")
934
+
935
+
fmt.Printf("Fetching %d missing bundles...\n\n", len(comparison.MissingBundles))
936
+
fetchMissingBundles(mgr, baseURL, comparison.MissingBundles)
937
+
}
938
+
939
+
// Exit with error if there are differences
940
+
if comparison.HasDifferences() {
941
+
os.Exit(1)
942
+
}
943
+
}