atscan.net
[DEPRECATED] Go implementation of plcbundle
update structure (5)
tree.fail
+57
bundle.go
+57
bundle.go
···
···
1
+
package plcbundle
2
+
3
+
import (
4
+
"context"
5
+
"io"
6
+
7
+
"tangled.org/atscan.net/plcbundle/internal/bundle"
8
+
)
9
+
10
+
// Manager is the main entry point for plcbundle operations
11
+
type Manager struct {
12
+
internal *bundle.Manager
13
+
}
14
+
15
+
// New creates a new plcbundle manager
16
+
func New(opts ...Option) (*Manager, error) {
17
+
config := defaultConfig()
18
+
for _, opt := range opts {
19
+
opt(config)
20
+
}
21
+
22
+
mgr, err := bundle.NewManager(config.bundleConfig, config.plcClient)
23
+
if err != nil {
24
+
return nil, err
25
+
}
26
+
27
+
return &Manager{internal: mgr}, nil
28
+
}
29
+
30
+
// LoadBundle loads a bundle by number
31
+
func (m *Manager) LoadBundle(ctx context.Context, bundleNumber int) (*Bundle, error) {
32
+
b, err := m.internal.LoadBundle(ctx, bundleNumber)
33
+
if err != nil {
34
+
return nil, err
35
+
}
36
+
return toBundlePublic(b), nil
37
+
}
38
+
39
+
// StreamBundleRaw streams raw compressed bundle data
40
+
func (m *Manager) StreamBundleRaw(ctx context.Context, bundleNumber int) (io.ReadCloser, error) {
41
+
return m.internal.StreamBundleRaw(ctx, bundleNumber)
42
+
}
43
+
44
+
// Close closes the manager and releases resources
45
+
func (m *Manager) Close() error {
46
+
m.internal.Close()
47
+
return nil
48
+
}
49
+
50
+
// FetchNextBundle fetches the next bundle from PLC directory
51
+
func (m *Manager) FetchNextBundle(ctx context.Context) (*Bundle, error) {
52
+
b, err := m.internal.FetchNextBundle(ctx, false)
53
+
if err != nil {
54
+
return nil, err
55
+
}
56
+
return toBundlePublic(b), nil
57
+
}
+110
cmd/plcbundle/commands/backfill.go
+110
cmd/plcbundle/commands/backfill.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"flag"
6
+
"fmt"
7
+
"os"
8
+
)
9
+
10
+
// BackfillCommand handles the backfill subcommand
11
+
func BackfillCommand(args []string) error {
12
+
fs := flag.NewFlagSet("backfill", flag.ExitOnError)
13
+
plcURL := fs.String("plc", "https://plc.directory", "PLC directory URL")
14
+
startFrom := fs.Int("start", 1, "bundle number to start from")
15
+
endAt := fs.Int("end", 0, "bundle number to end at (0 = until caught up)")
16
+
verbose := fs.Bool("verbose", false, "verbose sync logging")
17
+
18
+
if err := fs.Parse(args); err != nil {
19
+
return err
20
+
}
21
+
22
+
mgr, dir, err := getManager(*plcURL)
23
+
if err != nil {
24
+
return err
25
+
}
26
+
defer mgr.Close()
27
+
28
+
fmt.Fprintf(os.Stderr, "Starting backfill from: %s\n", dir)
29
+
fmt.Fprintf(os.Stderr, "Starting from bundle: %06d\n", *startFrom)
30
+
if *endAt > 0 {
31
+
fmt.Fprintf(os.Stderr, "Ending at bundle: %06d\n", *endAt)
32
+
} else {
33
+
fmt.Fprintf(os.Stderr, "Ending: when caught up\n")
34
+
}
35
+
fmt.Fprintf(os.Stderr, "\n")
36
+
37
+
ctx := context.Background()
38
+
39
+
currentBundle := *startFrom
40
+
processedCount := 0
41
+
fetchedCount := 0
42
+
loadedCount := 0
43
+
operationCount := 0
44
+
45
+
for {
46
+
if *endAt > 0 && currentBundle > *endAt {
47
+
break
48
+
}
49
+
50
+
fmt.Fprintf(os.Stderr, "Processing bundle %06d... ", currentBundle)
51
+
52
+
// Try to load from disk first
53
+
bundle, err := mgr.LoadBundle(ctx, currentBundle)
54
+
55
+
if err != nil {
56
+
// Bundle doesn't exist, fetch it
57
+
fmt.Fprintf(os.Stderr, "fetching... ")
58
+
59
+
bundle, err = mgr.FetchNextBundle(ctx, !*verbose)
60
+
if err != nil {
61
+
if isEndOfDataError(err) {
62
+
fmt.Fprintf(os.Stderr, "\n✓ Caught up! No more complete bundles available.\n")
63
+
break
64
+
}
65
+
fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
66
+
break
67
+
}
68
+
69
+
if err := mgr.SaveBundle(ctx, bundle, !*verbose); err != nil {
70
+
return fmt.Errorf("error saving: %w", err)
71
+
}
72
+
73
+
fetchedCount++
74
+
fmt.Fprintf(os.Stderr, "saved... ")
75
+
} else {
76
+
loadedCount++
77
+
}
78
+
79
+
// Output operations to stdout (JSONL)
80
+
for _, op := range bundle.Operations {
81
+
if len(op.RawJSON) > 0 {
82
+
fmt.Println(string(op.RawJSON))
83
+
}
84
+
}
85
+
86
+
operationCount += len(bundle.Operations)
87
+
processedCount++
88
+
89
+
fmt.Fprintf(os.Stderr, "✓ (%d ops, %d DIDs)\n", len(bundle.Operations), bundle.DIDCount)
90
+
91
+
currentBundle++
92
+
93
+
// Progress summary every 100 bundles
94
+
if processedCount%100 == 0 {
95
+
fmt.Fprintf(os.Stderr, "\n--- Progress: %d bundles processed (%d fetched, %d loaded) ---\n",
96
+
processedCount, fetchedCount, loadedCount)
97
+
fmt.Fprintf(os.Stderr, " Total operations: %d\n\n", operationCount)
98
+
}
99
+
}
100
+
101
+
// Final summary
102
+
fmt.Fprintf(os.Stderr, "\n✓ Backfill complete\n")
103
+
fmt.Fprintf(os.Stderr, " Bundles processed: %d\n", processedCount)
104
+
fmt.Fprintf(os.Stderr, " Newly fetched: %d\n", fetchedCount)
105
+
fmt.Fprintf(os.Stderr, " Loaded from disk: %d\n", loadedCount)
106
+
fmt.Fprintf(os.Stderr, " Total operations: %d\n", operationCount)
107
+
fmt.Fprintf(os.Stderr, " Range: %06d - %06d\n", *startFrom, currentBundle-1)
108
+
109
+
return nil
110
+
}
+157
cmd/plcbundle/commands/clone.go
+157
cmd/plcbundle/commands/clone.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"flag"
6
+
"fmt"
7
+
"os"
8
+
"os/signal"
9
+
"strings"
10
+
"sync"
11
+
"syscall"
12
+
"time"
13
+
14
+
"tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui"
15
+
internalsync "tangled.org/atscan.net/plcbundle/internal/sync"
16
+
)
17
+
18
+
// CloneCommand handles the clone subcommand
19
+
func CloneCommand(args []string) error {
20
+
fs := flag.NewFlagSet("clone", flag.ExitOnError)
21
+
workers := fs.Int("workers", 4, "number of concurrent download workers")
22
+
verbose := fs.Bool("v", false, "verbose output")
23
+
skipExisting := fs.Bool("skip-existing", true, "skip bundles that already exist locally")
24
+
saveInterval := fs.Duration("save-interval", 5*time.Second, "interval to save index during download")
25
+
26
+
if err := fs.Parse(args); err != nil {
27
+
return err
28
+
}
29
+
30
+
if fs.NArg() < 1 {
31
+
return fmt.Errorf("usage: plcbundle clone <remote-url> [options]\n\n" +
32
+
"Clone bundles from a remote plcbundle HTTP endpoint\n\n" +
33
+
"Example:\n" +
34
+
" plcbundle clone https://plc.example.com")
35
+
}
36
+
37
+
remoteURL := strings.TrimSuffix(fs.Arg(0), "/")
38
+
39
+
// Create manager
40
+
mgr, dir, err := getManager("")
41
+
if err != nil {
42
+
return err
43
+
}
44
+
defer mgr.Close()
45
+
46
+
fmt.Printf("Cloning from: %s\n", remoteURL)
47
+
fmt.Printf("Target directory: %s\n", dir)
48
+
fmt.Printf("Workers: %d\n", *workers)
49
+
fmt.Printf("(Press Ctrl+C to safely interrupt - progress will be saved)\n\n")
50
+
51
+
// Set up signal handling
52
+
ctx, cancel := context.WithCancel(context.Background())
53
+
defer cancel()
54
+
55
+
sigChan := make(chan os.Signal, 1)
56
+
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
57
+
58
+
// Set up progress bar
59
+
var progress *ui.ProgressBar
60
+
var progressMu sync.Mutex
61
+
progressActive := true
62
+
63
+
go func() {
64
+
<-sigChan
65
+
progressMu.Lock()
66
+
progressActive = false
67
+
if progress != nil {
68
+
fmt.Println()
69
+
}
70
+
progressMu.Unlock()
71
+
72
+
fmt.Printf("\n⚠️ Interrupt received! Finishing current downloads and saving progress...\n")
73
+
cancel()
74
+
}()
75
+
76
+
// Clone with library
77
+
result, err := mgr.CloneFromRemote(ctx, internalsync.CloneOptions{
78
+
RemoteURL: remoteURL,
79
+
Workers: *workers,
80
+
SkipExisting: *skipExisting,
81
+
SaveInterval: *saveInterval,
82
+
Verbose: *verbose,
83
+
ProgressFunc: func(downloaded, total int, bytesDownloaded, bytesTotal int64) {
84
+
progressMu.Lock()
85
+
defer progressMu.Unlock()
86
+
87
+
if !progressActive {
88
+
return
89
+
}
90
+
91
+
if progress == nil {
92
+
progress = ui.NewProgressBarWithBytes(total, bytesTotal)
93
+
}
94
+
progress.SetWithBytes(downloaded, bytesDownloaded)
95
+
},
96
+
})
97
+
98
+
// Ensure progress is stopped
99
+
progressMu.Lock()
100
+
progressActive = false
101
+
if progress != nil {
102
+
progress.Finish()
103
+
}
104
+
progressMu.Unlock()
105
+
106
+
if err != nil {
107
+
return fmt.Errorf("clone failed: %w", err)
108
+
}
109
+
110
+
// Display results
111
+
if result.Interrupted {
112
+
fmt.Printf("⚠️ Download interrupted by user\n")
113
+
} else {
114
+
fmt.Printf("\n✓ Clone complete in %s\n", result.Duration.Round(time.Millisecond))
115
+
}
116
+
117
+
fmt.Printf("\nResults:\n")
118
+
fmt.Printf(" Remote bundles: %d\n", result.RemoteBundles)
119
+
if result.Skipped > 0 {
120
+
fmt.Printf(" Skipped (existing): %d\n", result.Skipped)
121
+
}
122
+
fmt.Printf(" Downloaded: %d\n", result.Downloaded)
123
+
if result.Failed > 0 {
124
+
fmt.Printf(" Failed: %d\n", result.Failed)
125
+
}
126
+
fmt.Printf(" Total size: %s\n", formatBytes(result.TotalBytes))
127
+
128
+
if result.Duration.Seconds() > 0 && result.Downloaded > 0 {
129
+
mbPerSec := float64(result.TotalBytes) / result.Duration.Seconds() / (1024 * 1024)
130
+
bundlesPerSec := float64(result.Downloaded) / result.Duration.Seconds()
131
+
fmt.Printf(" Average speed: %.1f MB/s (%.1f bundles/s)\n", mbPerSec, bundlesPerSec)
132
+
}
133
+
134
+
if result.Failed > 0 {
135
+
fmt.Printf("\n⚠️ Failed bundles: ")
136
+
for i, num := range result.FailedBundles {
137
+
if i > 0 {
138
+
fmt.Printf(", ")
139
+
}
140
+
if i > 10 {
141
+
fmt.Printf("... and %d more", len(result.FailedBundles)-10)
142
+
break
143
+
}
144
+
fmt.Printf("%06d", num)
145
+
}
146
+
fmt.Printf("\nRe-run the clone command to retry failed bundles.\n")
147
+
return fmt.Errorf("clone completed with errors")
148
+
}
149
+
150
+
if result.Interrupted {
151
+
fmt.Printf("\n✓ Progress saved. Re-run the clone command to resume.\n")
152
+
return fmt.Errorf("clone interrupted")
153
+
}
154
+
155
+
fmt.Printf("\n✓ Clone complete!\n")
156
+
return nil
157
+
}
+149
cmd/plcbundle/commands/common.go
+149
cmd/plcbundle/commands/common.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"os"
7
+
"strings"
8
+
"time"
9
+
10
+
"tangled.org/atscan.net/plcbundle/internal/bundle"
11
+
"tangled.org/atscan.net/plcbundle/internal/bundleindex"
12
+
"tangled.org/atscan.net/plcbundle/internal/didindex"
13
+
internalsync "tangled.org/atscan.net/plcbundle/internal/sync"
14
+
"tangled.org/atscan.net/plcbundle/plcclient"
15
+
)
16
+
17
+
// BundleManager interface (for testing/mocking)
18
+
type BundleManager interface {
19
+
Close()
20
+
GetIndex() *bundleindex.Index
21
+
LoadBundle(ctx context.Context, bundleNumber int) (*bundle.Bundle, error)
22
+
VerifyBundle(ctx context.Context, bundleNumber int) (*bundle.VerificationResult, error)
23
+
VerifyChain(ctx context.Context) (*bundle.ChainVerificationResult, error)
24
+
GetInfo() map[string]interface{}
25
+
GetMempoolStats() map[string]interface{}
26
+
GetMempoolOperations() ([]plcclient.PLCOperation, error)
27
+
ValidateMempool() error
28
+
RefreshMempool() error
29
+
ClearMempool() error
30
+
FetchNextBundle(ctx context.Context, quiet bool) (*bundle.Bundle, error)
31
+
SaveBundle(ctx context.Context, b *bundle.Bundle, quiet bool) error
32
+
GetDIDIndexStats() map[string]interface{}
33
+
GetDIDIndex() *didindex.Manager
34
+
BuildDIDIndex(ctx context.Context, progress func(int, int)) error
35
+
GetDIDOperationsWithLocations(ctx context.Context, did string, verbose bool) ([]bundle.PLCOperationWithLocation, error)
36
+
GetDIDOperationsFromMempool(did string) ([]plcclient.PLCOperation, error)
37
+
GetLatestDIDOperation(ctx context.Context, did string) (*plcclient.PLCOperation, error)
38
+
LoadOperation(ctx context.Context, bundleNum, position int) (*plcclient.PLCOperation, error)
39
+
CloneFromRemote(ctx context.Context, opts internalsync.CloneOptions) (*internalsync.CloneResult, error)
40
+
}
41
+
42
+
// PLCOperationWithLocation wraps operation with location info
43
+
type PLCOperationWithLocation = bundle.PLCOperationWithLocation
44
+
45
+
// getManager creates or opens a bundle manager
46
+
func getManager(plcURL string) (*bundle.Manager, string, error) {
47
+
dir, err := os.Getwd()
48
+
if err != nil {
49
+
return nil, "", err
50
+
}
51
+
52
+
if err := os.MkdirAll(dir, 0755); err != nil {
53
+
return nil, "", fmt.Errorf("failed to create directory: %w", err)
54
+
}
55
+
56
+
config := bundle.DefaultConfig(dir)
57
+
58
+
var client *plcclient.Client
59
+
if plcURL != "" {
60
+
client = plcclient.NewClient(plcURL)
61
+
}
62
+
63
+
mgr, err := bundle.NewManager(config, client)
64
+
if err != nil {
65
+
return nil, "", err
66
+
}
67
+
68
+
return mgr, dir, nil
69
+
}
70
+
71
+
// parseBundleRange parses bundle range string
72
+
func parseBundleRange(rangeStr string) (start, end int, err error) {
73
+
if !strings.Contains(rangeStr, "-") {
74
+
var num int
75
+
_, err = fmt.Sscanf(rangeStr, "%d", &num)
76
+
if err != nil {
77
+
return 0, 0, fmt.Errorf("invalid bundle number: %w", err)
78
+
}
79
+
return num, num, nil
80
+
}
81
+
82
+
parts := strings.Split(rangeStr, "-")
83
+
if len(parts) != 2 {
84
+
return 0, 0, fmt.Errorf("invalid range format (expected: N or start-end)")
85
+
}
86
+
87
+
_, err = fmt.Sscanf(parts[0], "%d", &start)
88
+
if err != nil {
89
+
return 0, 0, fmt.Errorf("invalid start: %w", err)
90
+
}
91
+
92
+
_, err = fmt.Sscanf(parts[1], "%d", &end)
93
+
if err != nil {
94
+
return 0, 0, fmt.Errorf("invalid end: %w", err)
95
+
}
96
+
97
+
if start > end {
98
+
return 0, 0, fmt.Errorf("start must be <= end")
99
+
}
100
+
101
+
return start, end, nil
102
+
}
103
+
104
+
// Formatting helpers
105
+
106
+
func formatBytes(bytes int64) string {
107
+
const unit = 1000
108
+
if bytes < unit {
109
+
return fmt.Sprintf("%d B", bytes)
110
+
}
111
+
div, exp := int64(unit), 0
112
+
for n := bytes / unit; n >= unit; n /= unit {
113
+
div *= unit
114
+
exp++
115
+
}
116
+
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
117
+
}
118
+
119
+
func formatDuration(d time.Duration) string {
120
+
if d < time.Minute {
121
+
return fmt.Sprintf("%.0f seconds", d.Seconds())
122
+
}
123
+
if d < time.Hour {
124
+
return fmt.Sprintf("%.1f minutes", d.Minutes())
125
+
}
126
+
if d < 24*time.Hour {
127
+
return fmt.Sprintf("%.1f hours", d.Hours())
128
+
}
129
+
days := d.Hours() / 24
130
+
if days < 30 {
131
+
return fmt.Sprintf("%.1f days", days)
132
+
}
133
+
if days < 365 {
134
+
return fmt.Sprintf("%.1f months", days/30)
135
+
}
136
+
return fmt.Sprintf("%.1f years", days/365)
137
+
}
138
+
139
+
func formatNumber(n int) string {
140
+
s := fmt.Sprintf("%d", n)
141
+
var result []byte
142
+
for i, c := range s {
143
+
if i > 0 && (len(s)-i)%3 == 0 {
144
+
result = append(result, ',')
145
+
}
146
+
result = append(result, byte(c))
147
+
}
148
+
return string(result)
149
+
}
+466
cmd/plcbundle/commands/compare.go
+466
cmd/plcbundle/commands/compare.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"flag"
5
+
"fmt"
6
+
"io"
7
+
"net/http"
8
+
"os"
9
+
"path/filepath"
10
+
"sort"
11
+
"strings"
12
+
"time"
13
+
14
+
"github.com/goccy/go-json"
15
+
"tangled.org/atscan.net/plcbundle/internal/bundleindex"
16
+
)
17
+
18
+
// CompareCommand handles the compare subcommand
19
+
func CompareCommand(args []string) error {
20
+
fs := flag.NewFlagSet("compare", flag.ExitOnError)
21
+
verbose := fs.Bool("v", false, "verbose output (show all differences)")
22
+
fetchMissing := fs.Bool("fetch-missing", false, "fetch missing bundles from target")
23
+
24
+
if err := fs.Parse(args); err != nil {
25
+
return err
26
+
}
27
+
28
+
if fs.NArg() < 1 {
29
+
return fmt.Errorf("usage: plcbundle compare <target> [options]\n" +
30
+
" target: URL or path to remote plcbundle server/index\n\n" +
31
+
"Examples:\n" +
32
+
" plcbundle compare https://plc.example.com\n" +
33
+
" plcbundle compare https://plc.example.com/index.json\n" +
34
+
" plcbundle compare /path/to/plc_bundles.json\n" +
35
+
" plcbundle compare https://plc.example.com --fetch-missing")
36
+
}
37
+
38
+
target := fs.Arg(0)
39
+
40
+
mgr, dir, err := getManager("")
41
+
if err != nil {
42
+
return err
43
+
}
44
+
defer mgr.Close()
45
+
46
+
fmt.Printf("Comparing: %s\n", dir)
47
+
fmt.Printf(" Against: %s\n\n", target)
48
+
49
+
// Load local index
50
+
localIndex := mgr.GetIndex()
51
+
52
+
// Load target index
53
+
fmt.Printf("Loading target index...\n")
54
+
targetIndex, err := loadTargetIndex(target)
55
+
if err != nil {
56
+
return fmt.Errorf("error loading target index: %w", err)
57
+
}
58
+
59
+
// Perform comparison
60
+
comparison := compareIndexes(localIndex, targetIndex)
61
+
62
+
// Display results
63
+
displayComparison(comparison, *verbose)
64
+
65
+
// Fetch missing bundles if requested
66
+
if *fetchMissing && len(comparison.MissingBundles) > 0 {
67
+
fmt.Printf("\n")
68
+
if !strings.HasPrefix(target, "http://") && !strings.HasPrefix(target, "https://") {
69
+
return fmt.Errorf("--fetch-missing only works with remote URLs")
70
+
}
71
+
72
+
baseURL := strings.TrimSuffix(target, "/index.json")
73
+
baseURL = strings.TrimSuffix(baseURL, "/plc_bundles.json")
74
+
75
+
fmt.Printf("Fetching %d missing bundles...\n\n", len(comparison.MissingBundles))
76
+
if err := fetchMissingBundles(mgr, baseURL, comparison.MissingBundles); err != nil {
77
+
return err
78
+
}
79
+
}
80
+
81
+
if comparison.HasDifferences() {
82
+
return fmt.Errorf("indexes have differences")
83
+
}
84
+
85
+
return nil
86
+
}
87
+
88
+
func loadTargetIndex(target string) (*bundleindex.Index, error) {
89
+
if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") {
90
+
return loadIndexFromURL(target)
91
+
}
92
+
return bundleindex.LoadIndex(target)
93
+
}
94
+
95
+
func loadIndexFromURL(url string) (*bundleindex.Index, error) {
96
+
if !strings.HasSuffix(url, ".json") {
97
+
url = strings.TrimSuffix(url, "/") + "/index.json"
98
+
}
99
+
100
+
client := &http.Client{Timeout: 30 * time.Second}
101
+
102
+
resp, err := client.Get(url)
103
+
if err != nil {
104
+
return nil, fmt.Errorf("failed to download: %w", err)
105
+
}
106
+
defer resp.Body.Close()
107
+
108
+
if resp.StatusCode != http.StatusOK {
109
+
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
110
+
}
111
+
112
+
data, err := io.ReadAll(resp.Body)
113
+
if err != nil {
114
+
return nil, fmt.Errorf("failed to read response: %w", err)
115
+
}
116
+
117
+
var idx bundleindex.Index
118
+
if err := json.Unmarshal(data, &idx); err != nil {
119
+
return nil, fmt.Errorf("failed to parse index: %w", err)
120
+
}
121
+
122
+
return &idx, nil
123
+
}
124
+
125
+
func compareIndexes(local, target *bundleindex.Index) *IndexComparison {
126
+
localBundles := local.GetBundles()
127
+
targetBundles := target.GetBundles()
128
+
129
+
localMap := make(map[int]*bundleindex.BundleMetadata)
130
+
targetMap := make(map[int]*bundleindex.BundleMetadata)
131
+
132
+
for _, b := range localBundles {
133
+
localMap[b.BundleNumber] = b
134
+
}
135
+
for _, b := range targetBundles {
136
+
targetMap[b.BundleNumber] = b
137
+
}
138
+
139
+
comparison := &IndexComparison{
140
+
LocalCount: len(localBundles),
141
+
TargetCount: len(targetBundles),
142
+
MissingBundles: make([]int, 0),
143
+
ExtraBundles: make([]int, 0),
144
+
HashMismatches: make([]HashMismatch, 0),
145
+
ContentMismatches: make([]HashMismatch, 0),
146
+
}
147
+
148
+
// Get ranges
149
+
if len(localBundles) > 0 {
150
+
comparison.LocalRange = [2]int{localBundles[0].BundleNumber, localBundles[len(localBundles)-1].BundleNumber}
151
+
comparison.LocalUpdated = local.UpdatedAt
152
+
comparison.LocalTotalSize = local.TotalSize
153
+
}
154
+
155
+
if len(targetBundles) > 0 {
156
+
comparison.TargetRange = [2]int{targetBundles[0].BundleNumber, targetBundles[len(targetBundles)-1].BundleNumber}
157
+
comparison.TargetUpdated = target.UpdatedAt
158
+
comparison.TargetTotalSize = target.TotalSize
159
+
}
160
+
161
+
// Find missing bundles
162
+
for bundleNum := range targetMap {
163
+
if _, exists := localMap[bundleNum]; !exists {
164
+
comparison.MissingBundles = append(comparison.MissingBundles, bundleNum)
165
+
}
166
+
}
167
+
sort.Ints(comparison.MissingBundles)
168
+
169
+
// Find extra bundles
170
+
for bundleNum := range localMap {
171
+
if _, exists := targetMap[bundleNum]; !exists {
172
+
comparison.ExtraBundles = append(comparison.ExtraBundles, bundleNum)
173
+
}
174
+
}
175
+
sort.Ints(comparison.ExtraBundles)
176
+
177
+
// Compare hashes
178
+
for bundleNum, localMeta := range localMap {
179
+
if targetMeta, exists := targetMap[bundleNum]; exists {
180
+
comparison.CommonCount++
181
+
182
+
chainMismatch := localMeta.Hash != targetMeta.Hash
183
+
contentMismatch := localMeta.ContentHash != targetMeta.ContentHash
184
+
185
+
if chainMismatch || contentMismatch {
186
+
mismatch := HashMismatch{
187
+
BundleNumber: bundleNum,
188
+
LocalHash: localMeta.Hash,
189
+
TargetHash: targetMeta.Hash,
190
+
LocalContentHash: localMeta.ContentHash,
191
+
TargetContentHash: targetMeta.ContentHash,
192
+
}
193
+
194
+
if chainMismatch {
195
+
comparison.HashMismatches = append(comparison.HashMismatches, mismatch)
196
+
}
197
+
if contentMismatch && !chainMismatch {
198
+
comparison.ContentMismatches = append(comparison.ContentMismatches, mismatch)
199
+
}
200
+
}
201
+
}
202
+
}
203
+
204
+
return comparison
205
+
}
206
+
207
+
func displayComparison(c *IndexComparison, verbose bool) {
208
+
fmt.Printf("Comparison Results\n")
209
+
fmt.Printf("══════════════════\n\n")
210
+
211
+
fmt.Printf("Summary\n───────\n")
212
+
fmt.Printf(" Local bundles: %d\n", c.LocalCount)
213
+
fmt.Printf(" Target bundles: %d\n", c.TargetCount)
214
+
fmt.Printf(" Common bundles: %d\n", c.CommonCount)
215
+
fmt.Printf(" Missing bundles: %s\n", formatCount(len(c.MissingBundles)))
216
+
fmt.Printf(" Extra bundles: %s\n", formatCount(len(c.ExtraBundles)))
217
+
fmt.Printf(" Hash mismatches: %s\n", formatCountCritical(len(c.HashMismatches)))
218
+
fmt.Printf(" Content mismatches: %s\n", formatCount(len(c.ContentMismatches)))
219
+
220
+
if c.LocalCount > 0 {
221
+
fmt.Printf("\n Local range: %06d - %06d\n", c.LocalRange[0], c.LocalRange[1])
222
+
fmt.Printf(" Local size: %.2f MB\n", float64(c.LocalTotalSize)/(1024*1024))
223
+
fmt.Printf(" Local updated: %s\n", c.LocalUpdated.Format("2006-01-02 15:04:05"))
224
+
}
225
+
226
+
if c.TargetCount > 0 {
227
+
fmt.Printf("\n Target range: %06d - %06d\n", c.TargetRange[0], c.TargetRange[1])
228
+
fmt.Printf(" Target size: %.2f MB\n", float64(c.TargetTotalSize)/(1024*1024))
229
+
fmt.Printf(" Target updated: %s\n", c.TargetUpdated.Format("2006-01-02 15:04:05"))
230
+
}
231
+
232
+
// Show differences
233
+
if len(c.HashMismatches) > 0 {
234
+
showHashMismatches(c.HashMismatches, verbose)
235
+
}
236
+
237
+
if len(c.MissingBundles) > 0 {
238
+
showMissingBundles(c.MissingBundles, verbose)
239
+
}
240
+
241
+
if len(c.ExtraBundles) > 0 {
242
+
showExtraBundles(c.ExtraBundles, verbose)
243
+
}
244
+
245
+
// Final status
246
+
fmt.Printf("\n")
247
+
if !c.HasDifferences() {
248
+
fmt.Printf("✓ Indexes are identical\n")
249
+
} else {
250
+
fmt.Printf("✗ Indexes have differences\n")
251
+
if len(c.HashMismatches) > 0 {
252
+
fmt.Printf("\n⚠️ WARNING: Chain hash mismatches detected!\n")
253
+
fmt.Printf("This indicates different bundle content or chain integrity issues.\n")
254
+
}
255
+
}
256
+
}
257
+
258
+
func showHashMismatches(mismatches []HashMismatch, verbose bool) {
259
+
fmt.Printf("\n⚠️ CHAIN HASH MISMATCHES (CRITICAL)\n")
260
+
fmt.Printf("════════════════════════════════════\n\n")
261
+
262
+
displayCount := len(mismatches)
263
+
if displayCount > 10 && !verbose {
264
+
displayCount = 10
265
+
}
266
+
267
+
for i := 0; i < displayCount; i++ {
268
+
m := mismatches[i]
269
+
fmt.Printf(" Bundle %06d:\n", m.BundleNumber)
270
+
fmt.Printf(" Chain Hash:\n")
271
+
fmt.Printf(" Local: %s\n", m.LocalHash)
272
+
fmt.Printf(" Target: %s\n", m.TargetHash)
273
+
274
+
if m.LocalContentHash != m.TargetContentHash {
275
+
fmt.Printf(" Content Hash (also differs):\n")
276
+
fmt.Printf(" Local: %s\n", m.LocalContentHash)
277
+
fmt.Printf(" Target: %s\n", m.TargetContentHash)
278
+
}
279
+
fmt.Printf("\n")
280
+
}
281
+
282
+
if len(mismatches) > displayCount {
283
+
fmt.Printf(" ... and %d more (use -v to show all)\n\n", len(mismatches)-displayCount)
284
+
}
285
+
}
286
+
287
+
func showMissingBundles(bundles []int, verbose bool) {
288
+
fmt.Printf("\nMissing Bundles (in target but not local)\n")
289
+
fmt.Printf("──────────────────────────────────────────\n")
290
+
291
+
if verbose || len(bundles) <= 20 {
292
+
displayCount := len(bundles)
293
+
if displayCount > 20 && !verbose {
294
+
displayCount = 20
295
+
}
296
+
297
+
for i := 0; i < displayCount; i++ {
298
+
fmt.Printf(" %06d\n", bundles[i])
299
+
}
300
+
301
+
if len(bundles) > displayCount {
302
+
fmt.Printf(" ... and %d more (use -v to show all)\n", len(bundles)-displayCount)
303
+
}
304
+
} else {
305
+
displayBundleRanges(bundles)
306
+
}
307
+
}
308
+
309
+
func showExtraBundles(bundles []int, verbose bool) {
310
+
fmt.Printf("\nExtra Bundles (in local but not target)\n")
311
+
fmt.Printf("────────────────────────────────────────\n")
312
+
313
+
if verbose || len(bundles) <= 20 {
314
+
displayCount := len(bundles)
315
+
if displayCount > 20 && !verbose {
316
+
displayCount = 20
317
+
}
318
+
319
+
for i := 0; i < displayCount; i++ {
320
+
fmt.Printf(" %06d\n", bundles[i])
321
+
}
322
+
323
+
if len(bundles) > displayCount {
324
+
fmt.Printf(" ... and %d more (use -v to show all)\n", len(bundles)-displayCount)
325
+
}
326
+
} else {
327
+
displayBundleRanges(bundles)
328
+
}
329
+
}
330
+
331
+
func displayBundleRanges(bundles []int) {
332
+
if len(bundles) == 0 {
333
+
return
334
+
}
335
+
336
+
rangeStart := bundles[0]
337
+
rangeEnd := bundles[0]
338
+
339
+
for i := 1; i < len(bundles); i++ {
340
+
if bundles[i] == rangeEnd+1 {
341
+
rangeEnd = bundles[i]
342
+
} else {
343
+
if rangeStart == rangeEnd {
344
+
fmt.Printf(" %06d\n", rangeStart)
345
+
} else {
346
+
fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd)
347
+
}
348
+
rangeStart = bundles[i]
349
+
rangeEnd = bundles[i]
350
+
}
351
+
}
352
+
353
+
if rangeStart == rangeEnd {
354
+
fmt.Printf(" %06d\n", rangeStart)
355
+
} else {
356
+
fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd)
357
+
}
358
+
}
359
+
360
+
func fetchMissingBundles(mgr BundleManager, baseURL string, missingBundles []int) error {
361
+
client := &http.Client{Timeout: 60 * time.Second}
362
+
363
+
successCount := 0
364
+
errorCount := 0
365
+
366
+
info := mgr.GetInfo()
367
+
bundleDir := info["bundle_dir"].(string)
368
+
369
+
for _, bundleNum := range missingBundles {
370
+
fmt.Printf("Fetching bundle %06d... ", bundleNum)
371
+
372
+
url := fmt.Sprintf("%s/data/%d", baseURL, bundleNum)
373
+
resp, err := client.Get(url)
374
+
if err != nil {
375
+
fmt.Printf("ERROR: %v\n", err)
376
+
errorCount++
377
+
continue
378
+
}
379
+
380
+
if resp.StatusCode != http.StatusOK {
381
+
fmt.Printf("ERROR: status %d\n", resp.StatusCode)
382
+
resp.Body.Close()
383
+
errorCount++
384
+
continue
385
+
}
386
+
387
+
filename := fmt.Sprintf("%06d.jsonl.zst", bundleNum)
388
+
filepath := filepath.Join(bundleDir, filename)
389
+
390
+
outFile, err := os.Create(filepath)
391
+
if err != nil {
392
+
fmt.Printf("ERROR: %v\n", err)
393
+
resp.Body.Close()
394
+
errorCount++
395
+
continue
396
+
}
397
+
398
+
_, err = io.Copy(outFile, resp.Body)
399
+
outFile.Close()
400
+
resp.Body.Close()
401
+
402
+
if err != nil {
403
+
fmt.Printf("ERROR: %v\n", err)
404
+
os.Remove(filepath)
405
+
errorCount++
406
+
continue
407
+
}
408
+
409
+
fmt.Printf("✓\n")
410
+
successCount++
411
+
time.Sleep(200 * time.Millisecond)
412
+
}
413
+
414
+
fmt.Printf("\n✓ Fetch complete: %d succeeded, %d failed\n", successCount, errorCount)
415
+
416
+
if errorCount > 0 {
417
+
return fmt.Errorf("some bundles failed to download")
418
+
}
419
+
420
+
return nil
421
+
}
422
+
423
+
// Types
424
+
425
+
type IndexComparison struct {
426
+
LocalCount int
427
+
TargetCount int
428
+
CommonCount int
429
+
MissingBundles []int
430
+
ExtraBundles []int
431
+
HashMismatches []HashMismatch
432
+
ContentMismatches []HashMismatch
433
+
LocalRange [2]int
434
+
TargetRange [2]int
435
+
LocalTotalSize int64
436
+
TargetTotalSize int64
437
+
LocalUpdated time.Time
438
+
TargetUpdated time.Time
439
+
}
440
+
441
+
type HashMismatch struct {
442
+
BundleNumber int
443
+
LocalHash string
444
+
TargetHash string
445
+
LocalContentHash string
446
+
TargetContentHash string
447
+
}
448
+
449
+
func (ic *IndexComparison) HasDifferences() bool {
450
+
return len(ic.MissingBundles) > 0 || len(ic.ExtraBundles) > 0 ||
451
+
len(ic.HashMismatches) > 0 || len(ic.ContentMismatches) > 0
452
+
}
453
+
454
+
func formatCount(count int) string {
455
+
if count == 0 {
456
+
return "\033[32m0 ✓\033[0m"
457
+
}
458
+
return fmt.Sprintf("\033[33m%d ⚠️\033[0m", count)
459
+
}
460
+
461
+
func formatCountCritical(count int) string {
462
+
if count == 0 {
463
+
return "\033[32m0 ✓\033[0m"
464
+
}
465
+
return fmt.Sprintf("\033[31m%d ✗\033[0m", count)
466
+
}
+122
cmd/plcbundle/commands/export.go
+122
cmd/plcbundle/commands/export.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"flag"
6
+
"fmt"
7
+
"os"
8
+
"time"
9
+
10
+
"github.com/goccy/go-json"
11
+
)
12
+
13
+
// ExportCommand handles the export subcommand
14
+
func ExportCommand(args []string) error {
15
+
fs := flag.NewFlagSet("export", flag.ExitOnError)
16
+
bundles := fs.String("bundles", "", "bundle number or range (e.g., '42' or '1-100')")
17
+
all := fs.Bool("all", false, "export all bundles")
18
+
count := fs.Int("count", 0, "limit number of operations (0 = all)")
19
+
after := fs.String("after", "", "timestamp to start after (RFC3339)")
20
+
21
+
if err := fs.Parse(args); err != nil {
22
+
return err
23
+
}
24
+
25
+
if !*all && *bundles == "" {
26
+
return fmt.Errorf("usage: plcbundle export --bundles <number|range> [options]\n" +
27
+
" or: plcbundle export --all [options]\n\n" +
28
+
"Examples:\n" +
29
+
" plcbundle export --bundles 42\n" +
30
+
" plcbundle export --bundles 1-100\n" +
31
+
" plcbundle export --all\n" +
32
+
" plcbundle export --all --count 50000\n" +
33
+
" plcbundle export --bundles 42 | jq .")
34
+
}
35
+
36
+
mgr, _, err := getManager("")
37
+
if err != nil {
38
+
return err
39
+
}
40
+
defer mgr.Close()
41
+
42
+
// Determine bundle range
43
+
var start, end int
44
+
if *all {
45
+
index := mgr.GetIndex()
46
+
bundleList := index.GetBundles()
47
+
if len(bundleList) == 0 {
48
+
return fmt.Errorf("no bundles available")
49
+
}
50
+
start = bundleList[0].BundleNumber
51
+
end = bundleList[len(bundleList)-1].BundleNumber
52
+
53
+
fmt.Fprintf(os.Stderr, "Exporting all bundles (%d-%d)\n", start, end)
54
+
} else {
55
+
var err error
56
+
start, end, err = parseBundleRange(*bundles)
57
+
if err != nil {
58
+
return err
59
+
}
60
+
fmt.Fprintf(os.Stderr, "Exporting bundles %d-%d\n", start, end)
61
+
}
62
+
63
+
if *count > 0 {
64
+
fmt.Fprintf(os.Stderr, "Limit: %d operations\n", *count)
65
+
}
66
+
if *after != "" {
67
+
fmt.Fprintf(os.Stderr, "After: %s\n", *after)
68
+
}
69
+
fmt.Fprintf(os.Stderr, "\n")
70
+
71
+
// Parse after time
72
+
var afterTime time.Time
73
+
if *after != "" {
74
+
afterTime, err = time.Parse(time.RFC3339, *after)
75
+
if err != nil {
76
+
return fmt.Errorf("invalid after time: %w", err)
77
+
}
78
+
}
79
+
80
+
ctx := context.Background()
81
+
exported := 0
82
+
83
+
// Export operations
84
+
for bundleNum := start; bundleNum <= end; bundleNum++ {
85
+
if *count > 0 && exported >= *count {
86
+
break
87
+
}
88
+
89
+
fmt.Fprintf(os.Stderr, "Processing bundle %d...\r", bundleNum)
90
+
91
+
bundle, err := mgr.LoadBundle(ctx, bundleNum)
92
+
if err != nil {
93
+
fmt.Fprintf(os.Stderr, "\nWarning: failed to load bundle %d: %v\n", bundleNum, err)
94
+
continue
95
+
}
96
+
97
+
for _, op := range bundle.Operations {
98
+
if !afterTime.IsZero() && op.CreatedAt.Before(afterTime) {
99
+
continue
100
+
}
101
+
102
+
if *count > 0 && exported >= *count {
103
+
break
104
+
}
105
+
106
+
// Output as JSONL
107
+
if len(op.RawJSON) > 0 {
108
+
fmt.Println(string(op.RawJSON))
109
+
} else {
110
+
data, _ := json.Marshal(op)
111
+
fmt.Println(string(data))
112
+
}
113
+
114
+
exported++
115
+
}
116
+
}
117
+
118
+
fmt.Fprintf(os.Stderr, "\n\n✓ Export complete\n")
119
+
fmt.Fprintf(os.Stderr, " Exported: %d operations\n", exported)
120
+
121
+
return nil
122
+
}
+108
cmd/plcbundle/commands/fetch.go
+108
cmd/plcbundle/commands/fetch.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"flag"
6
+
"fmt"
7
+
"os"
8
+
"time"
9
+
)
10
+
11
+
// FetchCommand handles the fetch subcommand
12
+
func FetchCommand(args []string) error {
13
+
fs := flag.NewFlagSet("fetch", flag.ExitOnError)
14
+
plcURL := fs.String("plc", "https://plc.directory", "PLC directory URL")
15
+
count := fs.Int("count", 0, "number of bundles to fetch (0 = fetch all available)")
16
+
verbose := fs.Bool("verbose", false, "verbose sync logging")
17
+
18
+
if err := fs.Parse(args); err != nil {
19
+
return err
20
+
}
21
+
22
+
mgr, dir, err := getManager(*plcURL)
23
+
if err != nil {
24
+
return err
25
+
}
26
+
defer mgr.Close()
27
+
28
+
fmt.Printf("Working in: %s\n", dir)
29
+
30
+
ctx := context.Background()
31
+
32
+
// Get starting bundle info
33
+
index := mgr.GetIndex()
34
+
lastBundle := index.GetLastBundle()
35
+
startBundle := 1
36
+
if lastBundle != nil {
37
+
startBundle = lastBundle.BundleNumber + 1
38
+
}
39
+
40
+
fmt.Printf("Starting from bundle %06d\n", startBundle)
41
+
42
+
if *count > 0 {
43
+
fmt.Printf("Fetching %d bundles...\n", *count)
44
+
} else {
45
+
fmt.Printf("Fetching all available bundles...\n")
46
+
}
47
+
48
+
fetchedCount := 0
49
+
consecutiveErrors := 0
50
+
maxConsecutiveErrors := 3
51
+
52
+
for {
53
+
// Check if we've reached the requested count
54
+
if *count > 0 && fetchedCount >= *count {
55
+
break
56
+
}
57
+
58
+
currentBundle := startBundle + fetchedCount
59
+
60
+
if *count > 0 {
61
+
fmt.Printf("Fetching bundle %d/%d (bundle %06d)...\n", fetchedCount+1, *count, currentBundle)
62
+
} else {
63
+
fmt.Printf("Fetching bundle %06d...\n", currentBundle)
64
+
}
65
+
66
+
b, err := mgr.FetchNextBundle(ctx, !*verbose)
67
+
if err != nil {
68
+
// Check if we've reached the end
69
+
if isEndOfDataError(err) {
70
+
fmt.Printf("\n✓ Caught up! No more complete bundles available.\n")
71
+
fmt.Printf(" Last bundle: %06d\n", currentBundle-1)
72
+
break
73
+
}
74
+
75
+
// Handle other errors
76
+
consecutiveErrors++
77
+
fmt.Fprintf(os.Stderr, "Error fetching bundle %06d: %v\n", currentBundle, err)
78
+
79
+
if consecutiveErrors >= maxConsecutiveErrors {
80
+
return fmt.Errorf("too many consecutive errors, stopping")
81
+
}
82
+
83
+
fmt.Printf("Waiting 5 seconds before retry...\n")
84
+
time.Sleep(5 * time.Second)
85
+
continue
86
+
}
87
+
88
+
// Reset error counter on success
89
+
consecutiveErrors = 0
90
+
91
+
if err := mgr.SaveBundle(ctx, b, !*verbose); err != nil {
92
+
return fmt.Errorf("error saving bundle %06d: %w", b.BundleNumber, err)
93
+
}
94
+
95
+
fetchedCount++
96
+
fmt.Printf("✓ Saved bundle %06d (%d operations, %d DIDs)\n",
97
+
b.BundleNumber, len(b.Operations), b.DIDCount)
98
+
}
99
+
100
+
if fetchedCount > 0 {
101
+
fmt.Printf("\n✓ Fetch complete: %d bundles retrieved\n", fetchedCount)
102
+
fmt.Printf(" Current range: %06d - %06d\n", startBundle, startBundle+fetchedCount-1)
103
+
} else {
104
+
fmt.Printf("\n✓ Already up to date!\n")
105
+
}
106
+
107
+
return nil
108
+
}
+49
cmd/plcbundle/commands/getop.go
+49
cmd/plcbundle/commands/getop.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"strconv"
7
+
8
+
"github.com/goccy/go-json"
9
+
)
10
+
11
+
// GetOpCommand handles the get-op subcommand
12
+
func GetOpCommand(args []string) error {
13
+
if len(args) < 2 {
14
+
return fmt.Errorf("usage: plcbundle get-op <bundle> <position>\n" +
15
+
"Example: plcbundle get-op 42 1337")
16
+
}
17
+
18
+
bundleNum, err := strconv.Atoi(args[0])
19
+
if err != nil {
20
+
return fmt.Errorf("invalid bundle number")
21
+
}
22
+
23
+
position, err := strconv.Atoi(args[1])
24
+
if err != nil {
25
+
return fmt.Errorf("invalid position")
26
+
}
27
+
28
+
mgr, _, err := getManager("")
29
+
if err != nil {
30
+
return err
31
+
}
32
+
defer mgr.Close()
33
+
34
+
ctx := context.Background()
35
+
op, err := mgr.LoadOperation(ctx, bundleNum, position)
36
+
if err != nil {
37
+
return err
38
+
}
39
+
40
+
// Output JSON
41
+
if len(op.RawJSON) > 0 {
42
+
fmt.Println(string(op.RawJSON))
43
+
} else {
44
+
data, _ := json.Marshal(op)
45
+
fmt.Println(string(data))
46
+
}
47
+
48
+
return nil
49
+
}
+431
cmd/plcbundle/commands/index.go
+431
cmd/plcbundle/commands/index.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"flag"
6
+
"fmt"
7
+
"os"
8
+
"strings"
9
+
"time"
10
+
11
+
"github.com/goccy/go-json"
12
+
"tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui"
13
+
"tangled.org/atscan.net/plcbundle/plcclient"
14
+
)
15
+
16
+
// IndexCommand handles the index subcommand
17
+
func IndexCommand(args []string) error {
18
+
if len(args) < 1 {
19
+
printIndexUsage()
20
+
return fmt.Errorf("subcommand required")
21
+
}
22
+
23
+
subcommand := args[0]
24
+
25
+
switch subcommand {
26
+
case "build":
27
+
return indexBuild(args[1:])
28
+
case "stats":
29
+
return indexStats(args[1:])
30
+
case "lookup":
31
+
return indexLookup(args[1:])
32
+
case "resolve":
33
+
return indexResolve(args[1:])
34
+
default:
35
+
printIndexUsage()
36
+
return fmt.Errorf("unknown index subcommand: %s", subcommand)
37
+
}
38
+
}
39
+
40
+
func printIndexUsage() {
41
+
fmt.Printf(`Usage: plcbundle index <command> [options]
42
+
43
+
Commands:
44
+
build Build DID index from bundles
45
+
stats Show index statistics
46
+
lookup Lookup a specific DID
47
+
resolve Resolve DID to current document
48
+
49
+
Examples:
50
+
plcbundle index build
51
+
plcbundle index stats
52
+
plcbundle index lookup did:plc:524tuhdhh3m7li5gycdn6boe
53
+
plcbundle index resolve did:plc:524tuhdhh3m7li5gycdn6boe
54
+
`)
55
+
}
56
+
57
+
func indexBuild(args []string) error {
58
+
fs := flag.NewFlagSet("index build", flag.ExitOnError)
59
+
force := fs.Bool("force", false, "rebuild even if index exists")
60
+
61
+
if err := fs.Parse(args); err != nil {
62
+
return err
63
+
}
64
+
65
+
mgr, dir, err := getManager("")
66
+
if err != nil {
67
+
return err
68
+
}
69
+
defer mgr.Close()
70
+
71
+
stats := mgr.GetDIDIndexStats()
72
+
if stats["exists"].(bool) && !*force {
73
+
fmt.Printf("DID index already exists (use --force to rebuild)\n")
74
+
fmt.Printf("Directory: %s\n", dir)
75
+
fmt.Printf("Total DIDs: %d\n", stats["total_dids"])
76
+
return nil
77
+
}
78
+
79
+
fmt.Printf("Building DID index in: %s\n", dir)
80
+
81
+
index := mgr.GetIndex()
82
+
bundleCount := index.Count()
83
+
84
+
if bundleCount == 0 {
85
+
fmt.Printf("No bundles to index\n")
86
+
return nil
87
+
}
88
+
89
+
fmt.Printf("Indexing %d bundles...\n\n", bundleCount)
90
+
91
+
progress := ui.NewProgressBar(bundleCount)
92
+
start := time.Now()
93
+
ctx := context.Background()
94
+
95
+
err = mgr.BuildDIDIndex(ctx, func(current, total int) {
96
+
progress.Set(current)
97
+
})
98
+
99
+
progress.Finish()
100
+
101
+
if err != nil {
102
+
return fmt.Errorf("error building index: %w", err)
103
+
}
104
+
105
+
elapsed := time.Since(start)
106
+
stats = mgr.GetDIDIndexStats()
107
+
108
+
fmt.Printf("\n✓ DID index built in %s\n", elapsed.Round(time.Millisecond))
109
+
fmt.Printf(" Total DIDs: %s\n", formatNumber(int(stats["total_dids"].(int64))))
110
+
fmt.Printf(" Shards: %d\n", stats["shard_count"])
111
+
fmt.Printf(" Location: %s/.plcbundle/\n", dir)
112
+
113
+
return nil
114
+
}
115
+
116
+
func indexStats(args []string) error {
117
+
mgr, dir, err := getManager("")
118
+
if err != nil {
119
+
return err
120
+
}
121
+
defer mgr.Close()
122
+
123
+
stats := mgr.GetDIDIndexStats()
124
+
125
+
if !stats["exists"].(bool) {
126
+
fmt.Printf("DID index does not exist\n")
127
+
fmt.Printf("Run: plcbundle index build\n")
128
+
return nil
129
+
}
130
+
131
+
indexedDIDs := stats["indexed_dids"].(int64)
132
+
mempoolDIDs := stats["mempool_dids"].(int64)
133
+
totalDIDs := stats["total_dids"].(int64)
134
+
135
+
fmt.Printf("\nDID Index Statistics\n")
136
+
fmt.Printf("════════════════════\n\n")
137
+
fmt.Printf(" Location: %s/.plcbundle/\n", dir)
138
+
139
+
if mempoolDIDs > 0 {
140
+
fmt.Printf(" Indexed DIDs: %s (in bundles)\n", formatNumber(int(indexedDIDs)))
141
+
fmt.Printf(" Mempool DIDs: %s (not yet bundled)\n", formatNumber(int(mempoolDIDs)))
142
+
fmt.Printf(" Total DIDs: %s\n", formatNumber(int(totalDIDs)))
143
+
} else {
144
+
fmt.Printf(" Total DIDs: %s\n", formatNumber(int(totalDIDs)))
145
+
}
146
+
147
+
fmt.Printf(" Shard count: %d\n", stats["shard_count"])
148
+
fmt.Printf(" Last bundle: %06d\n", stats["last_bundle"])
149
+
fmt.Printf(" Updated: %s\n\n", stats["updated_at"].(time.Time).Format("2006-01-02 15:04:05"))
150
+
fmt.Printf(" Cached shards: %d / %d\n", stats["cached_shards"], stats["cache_limit"])
151
+
152
+
if cachedList, ok := stats["cache_order"].([]int); ok && len(cachedList) > 0 {
153
+
fmt.Printf(" Hot shards: ")
154
+
for i, shard := range cachedList {
155
+
if i > 0 {
156
+
fmt.Printf(", ")
157
+
}
158
+
if i >= 10 {
159
+
fmt.Printf("... (+%d more)", len(cachedList)-10)
160
+
break
161
+
}
162
+
fmt.Printf("%02x", shard)
163
+
}
164
+
fmt.Printf("\n")
165
+
}
166
+
167
+
fmt.Printf("\n")
168
+
return nil
169
+
}
170
+
171
+
func indexLookup(args []string) error {
172
+
fs := flag.NewFlagSet("index lookup", flag.ExitOnError)
173
+
verbose := fs.Bool("v", false, "verbose debug output")
174
+
showJSON := fs.Bool("json", false, "output as JSON")
175
+
176
+
if err := fs.Parse(args); err != nil {
177
+
return err
178
+
}
179
+
180
+
if fs.NArg() < 1 {
181
+
return fmt.Errorf("usage: plcbundle index lookup <did> [-v] [--json]")
182
+
}
183
+
184
+
did := fs.Arg(0)
185
+
186
+
mgr, _, err := getManager("")
187
+
if err != nil {
188
+
return err
189
+
}
190
+
defer mgr.Close()
191
+
192
+
stats := mgr.GetDIDIndexStats()
193
+
if !stats["exists"].(bool) {
194
+
fmt.Fprintf(os.Stderr, "⚠️ DID index does not exist. Run: plcbundle index build\n")
195
+
fmt.Fprintf(os.Stderr, " Falling back to full scan (this will be slow)...\n\n")
196
+
}
197
+
198
+
if !*showJSON {
199
+
fmt.Printf("Looking up: %s\n", did)
200
+
if *verbose {
201
+
fmt.Printf("Verbose mode: enabled\n")
202
+
}
203
+
fmt.Printf("\n")
204
+
}
205
+
206
+
totalStart := time.Now()
207
+
ctx := context.Background()
208
+
209
+
// Lookup operations
210
+
lookupStart := time.Now()
211
+
opsWithLoc, err := mgr.GetDIDOperationsWithLocations(ctx, did, *verbose)
212
+
if err != nil {
213
+
return err
214
+
}
215
+
lookupElapsed := time.Since(lookupStart)
216
+
217
+
// Check mempool
218
+
mempoolStart := time.Now()
219
+
mempoolOps, err := mgr.GetDIDOperationsFromMempool(did)
220
+
if err != nil {
221
+
return fmt.Errorf("error checking mempool: %w", err)
222
+
}
223
+
mempoolElapsed := time.Since(mempoolStart)
224
+
225
+
totalElapsed := time.Since(totalStart)
226
+
227
+
if len(opsWithLoc) == 0 && len(mempoolOps) == 0 {
228
+
if *showJSON {
229
+
fmt.Println("{\"found\": false, \"operations\": []}")
230
+
} else {
231
+
fmt.Printf("DID not found (searched in %s)\n", totalElapsed)
232
+
}
233
+
return nil
234
+
}
235
+
236
+
if *showJSON {
237
+
return outputLookupJSON(did, opsWithLoc, mempoolOps, totalElapsed, lookupElapsed, mempoolElapsed)
238
+
}
239
+
240
+
return displayLookupResults(did, opsWithLoc, mempoolOps, totalElapsed, lookupElapsed, mempoolElapsed, *verbose, stats)
241
+
}
242
+
243
+
func indexResolve(args []string) error {
244
+
fs := flag.NewFlagSet("index resolve", flag.ExitOnError)
245
+
246
+
if err := fs.Parse(args); err != nil {
247
+
return err
248
+
}
249
+
250
+
if fs.NArg() < 1 {
251
+
return fmt.Errorf("usage: plcbundle index resolve <did>")
252
+
}
253
+
254
+
did := fs.Arg(0)
255
+
256
+
mgr, _, err := getManager("")
257
+
if err != nil {
258
+
return err
259
+
}
260
+
defer mgr.Close()
261
+
262
+
ctx := context.Background()
263
+
fmt.Fprintf(os.Stderr, "Resolving: %s\n", did)
264
+
265
+
start := time.Now()
266
+
267
+
// Check mempool first
268
+
mempoolOps, _ := mgr.GetDIDOperationsFromMempool(did)
269
+
if len(mempoolOps) > 0 {
270
+
for i := len(mempoolOps) - 1; i >= 0; i-- {
271
+
if !mempoolOps[i].IsNullified() {
272
+
doc, err := plcclient.ResolveDIDDocument(did, []plcclient.PLCOperation{mempoolOps[i]})
273
+
if err != nil {
274
+
return fmt.Errorf("resolution failed: %w", err)
275
+
}
276
+
277
+
totalTime := time.Since(start)
278
+
fmt.Fprintf(os.Stderr, "Total: %s (resolved from mempool)\n\n", totalTime)
279
+
280
+
data, _ := json.MarshalIndent(doc, "", " ")
281
+
fmt.Println(string(data))
282
+
return nil
283
+
}
284
+
}
285
+
}
286
+
287
+
// Use index
288
+
op, err := mgr.GetLatestDIDOperation(ctx, did)
289
+
if err != nil {
290
+
return fmt.Errorf("failed to get latest operation: %w", err)
291
+
}
292
+
293
+
doc, err := plcclient.ResolveDIDDocument(did, []plcclient.PLCOperation{*op})
294
+
if err != nil {
295
+
return fmt.Errorf("resolution failed: %w", err)
296
+
}
297
+
298
+
totalTime := time.Since(start)
299
+
fmt.Fprintf(os.Stderr, "Total: %s\n\n", totalTime)
300
+
301
+
data, _ := json.MarshalIndent(doc, "", " ")
302
+
fmt.Println(string(data))
303
+
304
+
return nil
305
+
}
306
+
307
+
func outputLookupJSON(did string, opsWithLoc []PLCOperationWithLocation, mempoolOps []plcclient.PLCOperation, totalElapsed, lookupElapsed, mempoolElapsed time.Duration) error {
308
+
output := map[string]interface{}{
309
+
"found": true,
310
+
"did": did,
311
+
"timing": map[string]interface{}{
312
+
"total_ms": totalElapsed.Milliseconds(),
313
+
"lookup_ms": lookupElapsed.Milliseconds(),
314
+
"mempool_ms": mempoolElapsed.Milliseconds(),
315
+
},
316
+
"bundled": make([]map[string]interface{}, 0),
317
+
"mempool": make([]map[string]interface{}, 0),
318
+
}
319
+
320
+
for _, owl := range opsWithLoc {
321
+
output["bundled"] = append(output["bundled"].([]map[string]interface{}), map[string]interface{}{
322
+
"bundle": owl.Bundle,
323
+
"position": owl.Position,
324
+
"cid": owl.Operation.CID,
325
+
"nullified": owl.Operation.IsNullified(),
326
+
"created_at": owl.Operation.CreatedAt.Format(time.RFC3339Nano),
327
+
})
328
+
}
329
+
330
+
for _, op := range mempoolOps {
331
+
output["mempool"] = append(output["mempool"].([]map[string]interface{}), map[string]interface{}{
332
+
"cid": op.CID,
333
+
"nullified": op.IsNullified(),
334
+
"created_at": op.CreatedAt.Format(time.RFC3339Nano),
335
+
})
336
+
}
337
+
338
+
data, _ := json.MarshalIndent(output, "", " ")
339
+
fmt.Println(string(data))
340
+
341
+
return nil
342
+
}
343
+
344
+
func displayLookupResults(did string, opsWithLoc []PLCOperationWithLocation, mempoolOps []plcclient.PLCOperation, totalElapsed, lookupElapsed, mempoolElapsed time.Duration, verbose bool, stats map[string]interface{}) error {
345
+
nullifiedCount := 0
346
+
for _, owl := range opsWithLoc {
347
+
if owl.Operation.IsNullified() {
348
+
nullifiedCount++
349
+
}
350
+
}
351
+
352
+
totalOps := len(opsWithLoc) + len(mempoolOps)
353
+
activeOps := len(opsWithLoc) - nullifiedCount + len(mempoolOps)
354
+
355
+
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
356
+
fmt.Printf(" DID Lookup Results\n")
357
+
fmt.Printf("═══════════════════════════════════════════════════════════════\n\n")
358
+
fmt.Printf("DID: %s\n\n", did)
359
+
360
+
fmt.Printf("Summary\n───────\n")
361
+
fmt.Printf(" Total operations: %d\n", totalOps)
362
+
fmt.Printf(" Active operations: %d\n", activeOps)
363
+
if nullifiedCount > 0 {
364
+
fmt.Printf(" Nullified: %d\n", nullifiedCount)
365
+
}
366
+
if len(opsWithLoc) > 0 {
367
+
fmt.Printf(" Bundled: %d\n", len(opsWithLoc))
368
+
}
369
+
if len(mempoolOps) > 0 {
370
+
fmt.Printf(" Mempool: %d\n", len(mempoolOps))
371
+
}
372
+
fmt.Printf("\n")
373
+
374
+
fmt.Printf("Performance\n───────────\n")
375
+
fmt.Printf(" Index lookup: %s\n", lookupElapsed)
376
+
fmt.Printf(" Mempool check: %s\n", mempoolElapsed)
377
+
fmt.Printf(" Total time: %s\n\n", totalElapsed)
378
+
379
+
// Show operations
380
+
if len(opsWithLoc) > 0 {
381
+
fmt.Printf("Bundled Operations (%d total)\n", len(opsWithLoc))
382
+
fmt.Printf("══════════════════════════════════════════════════════════════\n\n")
383
+
384
+
for i, owl := range opsWithLoc {
385
+
op := owl.Operation
386
+
status := "✓ Active"
387
+
if op.IsNullified() {
388
+
status = "✗ Nullified"
389
+
}
390
+
391
+
fmt.Printf("Operation %d [Bundle %06d, Position %04d]\n", i+1, owl.Bundle, owl.Position)
392
+
fmt.Printf(" CID: %s\n", op.CID)
393
+
fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000 MST"))
394
+
fmt.Printf(" Status: %s\n", status)
395
+
396
+
if verbose && !op.IsNullified() {
397
+
showOperationDetails(&op)
398
+
}
399
+
400
+
fmt.Printf("\n")
401
+
}
402
+
}
403
+
404
+
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
405
+
fmt.Printf("✓ Lookup complete in %s\n", totalElapsed)
406
+
if stats["exists"].(bool) {
407
+
fmt.Printf(" Method: DID index (fast)\n")
408
+
} else {
409
+
fmt.Printf(" Method: Full scan (slow)\n")
410
+
}
411
+
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
412
+
413
+
return nil
414
+
}
415
+
416
+
func showOperationDetails(op *plcclient.PLCOperation) {
417
+
if opData, err := op.GetOperationData(); err == nil && opData != nil {
418
+
if opType, ok := opData["type"].(string); ok {
419
+
fmt.Printf(" Type: %s\n", opType)
420
+
}
421
+
422
+
if handle, ok := opData["handle"].(string); ok {
423
+
fmt.Printf(" Handle: %s\n", handle)
424
+
} else if aka, ok := opData["alsoKnownAs"].([]interface{}); ok && len(aka) > 0 {
425
+
if akaStr, ok := aka[0].(string); ok {
426
+
handle := strings.TrimPrefix(akaStr, "at://")
427
+
fmt.Printf(" Handle: %s\n", handle)
428
+
}
429
+
}
430
+
}
431
+
}
+194
cmd/plcbundle/commands/mempool.go
+194
cmd/plcbundle/commands/mempool.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"flag"
5
+
"fmt"
6
+
"os"
7
+
"path/filepath"
8
+
"strings"
9
+
"time"
10
+
11
+
"tangled.org/atscan.net/plcbundle/internal/types"
12
+
)
13
+
14
+
// MempoolCommand handles the mempool subcommand
15
+
func MempoolCommand(args []string) error {
16
+
fs := flag.NewFlagSet("mempool", flag.ExitOnError)
17
+
clear := fs.Bool("clear", false, "clear the mempool")
18
+
export := fs.Bool("export", false, "export mempool operations as JSONL to stdout")
19
+
refresh := fs.Bool("refresh", false, "reload mempool from disk")
20
+
validate := fs.Bool("validate", false, "validate chronological order")
21
+
verbose := fs.Bool("v", false, "verbose output")
22
+
23
+
if err := fs.Parse(args); err != nil {
24
+
return err
25
+
}
26
+
27
+
mgr, dir, err := getManager("")
28
+
if err != nil {
29
+
return err
30
+
}
31
+
defer mgr.Close()
32
+
33
+
fmt.Printf("Working in: %s\n\n", dir)
34
+
35
+
// Handle validate
36
+
if *validate {
37
+
fmt.Printf("Validating mempool chronological order...\n")
38
+
if err := mgr.ValidateMempool(); err != nil {
39
+
return fmt.Errorf("validation failed: %w", err)
40
+
}
41
+
fmt.Printf("✓ Mempool validation passed\n")
42
+
return nil
43
+
}
44
+
45
+
// Handle refresh
46
+
if *refresh {
47
+
fmt.Printf("Refreshing mempool from disk...\n")
48
+
if err := mgr.RefreshMempool(); err != nil {
49
+
return fmt.Errorf("refresh failed: %w", err)
50
+
}
51
+
52
+
if err := mgr.ValidateMempool(); err != nil {
53
+
fmt.Fprintf(os.Stderr, "⚠️ Warning: mempool validation failed after refresh: %v\n", err)
54
+
} else {
55
+
fmt.Printf("✓ Mempool refreshed and validated\n\n")
56
+
}
57
+
}
58
+
59
+
// Handle clear
60
+
if *clear {
61
+
stats := mgr.GetMempoolStats()
62
+
count := stats["count"].(int)
63
+
64
+
if count == 0 {
65
+
fmt.Println("Mempool is already empty")
66
+
return nil
67
+
}
68
+
69
+
fmt.Printf("⚠️ This will clear %d operations from the mempool.\n", count)
70
+
fmt.Printf("Are you sure? [y/N]: ")
71
+
var response string
72
+
fmt.Scanln(&response)
73
+
if strings.ToLower(strings.TrimSpace(response)) != "y" {
74
+
fmt.Println("Cancelled")
75
+
return nil
76
+
}
77
+
78
+
if err := mgr.ClearMempool(); err != nil {
79
+
return fmt.Errorf("clear failed: %w", err)
80
+
}
81
+
82
+
fmt.Printf("✓ Mempool cleared (%d operations removed)\n", count)
83
+
return nil
84
+
}
85
+
86
+
// Handle export
87
+
if *export {
88
+
ops, err := mgr.GetMempoolOperations()
89
+
if err != nil {
90
+
return fmt.Errorf("failed to get mempool operations: %w", err)
91
+
}
92
+
93
+
if len(ops) == 0 {
94
+
fmt.Fprintf(os.Stderr, "Mempool is empty\n")
95
+
return nil
96
+
}
97
+
98
+
for _, op := range ops {
99
+
if len(op.RawJSON) > 0 {
100
+
fmt.Println(string(op.RawJSON))
101
+
}
102
+
}
103
+
104
+
fmt.Fprintf(os.Stderr, "Exported %d operations from mempool\n", len(ops))
105
+
return nil
106
+
}
107
+
108
+
// Default: Show mempool stats
109
+
return showMempoolStats(mgr, dir, *verbose)
110
+
}
111
+
112
+
func showMempoolStats(mgr BundleManager, dir string, verbose bool) error {
113
+
stats := mgr.GetMempoolStats()
114
+
count := stats["count"].(int)
115
+
canCreate := stats["can_create_bundle"].(bool)
116
+
targetBundle := stats["target_bundle"].(int)
117
+
minTimestamp := stats["min_timestamp"].(time.Time)
118
+
validated := stats["validated"].(bool)
119
+
120
+
fmt.Printf("Mempool Status:\n")
121
+
fmt.Printf(" Target bundle: %06d\n", targetBundle)
122
+
fmt.Printf(" Operations: %d\n", count)
123
+
fmt.Printf(" Can create bundle: %v (need %d)\n", canCreate, types.BUNDLE_SIZE)
124
+
fmt.Printf(" Min timestamp: %s\n", minTimestamp.Format("2006-01-02 15:04:05"))
125
+
126
+
validationIcon := "✓"
127
+
if !validated {
128
+
validationIcon = "⚠️"
129
+
}
130
+
fmt.Printf(" Validated: %s %v\n", validationIcon, validated)
131
+
132
+
if count > 0 {
133
+
if sizeBytes, ok := stats["size_bytes"].(int); ok {
134
+
fmt.Printf(" Size: %.2f KB\n", float64(sizeBytes)/1024)
135
+
}
136
+
137
+
if firstTime, ok := stats["first_time"].(time.Time); ok {
138
+
fmt.Printf(" First operation: %s\n", firstTime.Format("2006-01-02 15:04:05"))
139
+
}
140
+
141
+
if lastTime, ok := stats["last_time"].(time.Time); ok {
142
+
fmt.Printf(" Last operation: %s\n", lastTime.Format("2006-01-02 15:04:05"))
143
+
}
144
+
145
+
progress := float64(count) / float64(types.BUNDLE_SIZE) * 100
146
+
fmt.Printf(" Progress: %.1f%% (%d/%d)\n", progress, count, types.BUNDLE_SIZE)
147
+
148
+
// Progress bar
149
+
barWidth := 40
150
+
filled := int(float64(barWidth) * float64(count) / float64(types.BUNDLE_SIZE))
151
+
if filled > barWidth {
152
+
filled = barWidth
153
+
}
154
+
bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled)
155
+
fmt.Printf(" [%s]\n", bar)
156
+
} else {
157
+
fmt.Printf(" (empty)\n")
158
+
}
159
+
160
+
// Verbose: Show sample operations
161
+
if verbose && count > 0 {
162
+
fmt.Println()
163
+
fmt.Printf("Sample operations (showing up to 10):\n")
164
+
165
+
ops, err := mgr.GetMempoolOperations()
166
+
if err != nil {
167
+
return fmt.Errorf("error getting operations: %w", err)
168
+
}
169
+
170
+
showCount := 10
171
+
if len(ops) < showCount {
172
+
showCount = len(ops)
173
+
}
174
+
175
+
for i := 0; i < showCount; i++ {
176
+
op := ops[i]
177
+
fmt.Printf(" %d. DID: %s\n", i+1, op.DID)
178
+
fmt.Printf(" CID: %s\n", op.CID)
179
+
fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000"))
180
+
}
181
+
182
+
if len(ops) > showCount {
183
+
fmt.Printf(" ... and %d more\n", len(ops)-showCount)
184
+
}
185
+
}
186
+
187
+
fmt.Println()
188
+
189
+
// Show mempool file
190
+
mempoolFilename := fmt.Sprintf("plc_mempool_%06d.jsonl", targetBundle)
191
+
fmt.Printf("File: %s\n", filepath.Join(dir, mempoolFilename))
192
+
193
+
return nil
194
+
}
+165
cmd/plcbundle/commands/rebuild.go
+165
cmd/plcbundle/commands/rebuild.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"flag"
6
+
"fmt"
7
+
"os"
8
+
"path/filepath"
9
+
"runtime"
10
+
"time"
11
+
12
+
"tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui"
13
+
"tangled.org/atscan.net/plcbundle/internal/bundle"
14
+
"tangled.org/atscan.net/plcbundle/internal/bundleindex"
15
+
)
16
+
17
+
// RebuildCommand handles the rebuild subcommand
18
+
func RebuildCommand(args []string) error {
19
+
fs := flag.NewFlagSet("rebuild", flag.ExitOnError)
20
+
verbose := fs.Bool("v", false, "verbose output")
21
+
workers := fs.Int("workers", 0, "number of parallel workers (0 = CPU count)")
22
+
noProgress := fs.Bool("no-progress", false, "disable progress bar")
23
+
24
+
if err := fs.Parse(args); err != nil {
25
+
return err
26
+
}
27
+
28
+
// Auto-detect CPU count
29
+
if *workers == 0 {
30
+
*workers = runtime.NumCPU()
31
+
}
32
+
33
+
// Get working directory
34
+
dir, err := os.Getwd()
35
+
if err != nil {
36
+
return err
37
+
}
38
+
39
+
if err := os.MkdirAll(dir, 0755); err != nil {
40
+
return err
41
+
}
42
+
43
+
// Create manager WITHOUT auto-rebuild
44
+
config := bundle.DefaultConfig(dir)
45
+
config.AutoRebuild = false
46
+
config.RebuildWorkers = *workers
47
+
48
+
mgr, err := bundle.NewManager(config, nil)
49
+
if err != nil {
50
+
return err
51
+
}
52
+
defer mgr.Close()
53
+
54
+
fmt.Printf("Rebuilding index from: %s\n", dir)
55
+
fmt.Printf("Using %d workers\n", *workers)
56
+
57
+
// Find all bundle files
58
+
files, err := filepath.Glob(filepath.Join(dir, "*.jsonl.zst"))
59
+
if err != nil {
60
+
return fmt.Errorf("error scanning directory: %w", err)
61
+
}
62
+
63
+
// Filter out hidden/temp files
64
+
files = filterBundleFiles(files)
65
+
66
+
if len(files) == 0 {
67
+
fmt.Println("No bundle files found")
68
+
return nil
69
+
}
70
+
71
+
fmt.Printf("Found %d bundle files\n\n", len(files))
72
+
73
+
start := time.Now()
74
+
75
+
// Create progress bar
76
+
var progress *ui.ProgressBar
77
+
var progressCallback func(int, int, int64)
78
+
79
+
if !*noProgress {
80
+
fmt.Println("Processing bundles:")
81
+
progress = ui.NewProgressBar(len(files))
82
+
83
+
progressCallback = func(current, total int, bytesProcessed int64) {
84
+
progress.SetWithBytes(current, bytesProcessed)
85
+
}
86
+
}
87
+
88
+
// Use parallel scan
89
+
result, err := mgr.ScanDirectoryParallel(*workers, progressCallback)
90
+
91
+
if err != nil {
92
+
if progress != nil {
93
+
progress.Finish()
94
+
}
95
+
return fmt.Errorf("rebuild failed: %w", err)
96
+
}
97
+
98
+
if progress != nil {
99
+
progress.Finish()
100
+
}
101
+
102
+
elapsed := time.Since(start)
103
+
104
+
fmt.Printf("\n✓ Index rebuilt in %s\n", elapsed.Round(time.Millisecond))
105
+
fmt.Printf(" Total bundles: %d\n", result.BundleCount)
106
+
fmt.Printf(" Compressed size: %s\n", formatBytes(result.TotalSize))
107
+
fmt.Printf(" Uncompressed size: %s\n", formatBytes(result.TotalUncompressed))
108
+
109
+
if result.TotalUncompressed > 0 {
110
+
ratio := float64(result.TotalUncompressed) / float64(result.TotalSize)
111
+
fmt.Printf(" Compression ratio: %.2fx\n", ratio)
112
+
}
113
+
114
+
fmt.Printf(" Average speed: %.1f bundles/sec\n", float64(result.BundleCount)/elapsed.Seconds())
115
+
116
+
if elapsed.Seconds() > 0 {
117
+
compressedThroughput := float64(result.TotalSize) / elapsed.Seconds() / (1000 * 1000)
118
+
uncompressedThroughput := float64(result.TotalUncompressed) / elapsed.Seconds() / (1000 * 1000)
119
+
fmt.Printf(" Throughput (compressed): %.1f MB/s\n", compressedThroughput)
120
+
fmt.Printf(" Throughput (uncompressed): %.1f MB/s\n", uncompressedThroughput)
121
+
}
122
+
123
+
fmt.Printf(" Index file: %s\n", filepath.Join(dir, bundleindex.INDEX_FILE))
124
+
125
+
if len(result.MissingGaps) > 0 {
126
+
fmt.Printf(" ⚠️ Missing gaps: %d bundles\n", len(result.MissingGaps))
127
+
}
128
+
129
+
// Verify chain if verbose
130
+
if *verbose {
131
+
fmt.Printf("\nVerifying chain integrity...\n")
132
+
133
+
ctx := context.Background()
134
+
verifyResult, err := mgr.VerifyChain(ctx)
135
+
if err != nil {
136
+
fmt.Printf(" ⚠️ Verification error: %v\n", err)
137
+
} else if verifyResult.Valid {
138
+
fmt.Printf(" ✓ Chain is valid (%d bundles verified)\n", len(verifyResult.VerifiedBundles))
139
+
140
+
// Show head hash
141
+
index := mgr.GetIndex()
142
+
if lastMeta := index.GetLastBundle(); lastMeta != nil {
143
+
fmt.Printf(" Chain head: %s...\n", lastMeta.Hash[:16])
144
+
}
145
+
} else {
146
+
fmt.Printf(" ✗ Chain verification failed\n")
147
+
fmt.Printf(" Broken at: bundle %06d\n", verifyResult.BrokenAt)
148
+
fmt.Printf(" Error: %s\n", verifyResult.Error)
149
+
}
150
+
}
151
+
152
+
return nil
153
+
}
154
+
155
+
func filterBundleFiles(files []string) []string {
156
+
filtered := make([]string, 0, len(files))
157
+
for _, file := range files {
158
+
basename := filepath.Base(file)
159
+
if len(basename) > 0 && (basename[0] == '.' || basename[0] == '_') {
160
+
continue
161
+
}
162
+
filtered = append(filtered, file)
163
+
}
164
+
return filtered
165
+
}
+416
cmd/plcbundle/commands/server.go
+416
cmd/plcbundle/commands/server.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"flag"
6
+
"fmt"
7
+
"os"
8
+
"os/signal"
9
+
"runtime"
10
+
"syscall"
11
+
"time"
12
+
13
+
"tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui"
14
+
"tangled.org/atscan.net/plcbundle/internal/bundle"
15
+
"tangled.org/atscan.net/plcbundle/internal/didindex"
16
+
"tangled.org/atscan.net/plcbundle/plcclient"
17
+
"tangled.org/atscan.net/plcbundle/server"
18
+
)
19
+
20
+
// ServerCommand handles the serve subcommand
21
+
func ServerCommand(args []string) error {
22
+
fs := flag.NewFlagSet("serve", flag.ExitOnError)
23
+
port := fs.String("port", "8080", "HTTP server port")
24
+
host := fs.String("host", "127.0.0.1", "HTTP server host")
25
+
syncMode := fs.Bool("sync", false, "enable sync mode (auto-sync from PLC)")
26
+
plcURL := fs.String("plc", "https://plc.directory", "PLC directory URL (for sync mode)")
27
+
syncInterval := fs.Duration("sync-interval", 1*time.Minute, "sync interval for sync mode")
28
+
enableWebSocket := fs.Bool("websocket", false, "enable WebSocket endpoint for streaming")
29
+
enableResolver := fs.Bool("resolver", false, "enable DID resolution endpoints")
30
+
workers := fs.Int("workers", 0, "number of workers for auto-rebuild (0 = CPU count)")
31
+
verbose := fs.Bool("verbose", false, "verbose sync logging")
32
+
33
+
if err := fs.Parse(args); err != nil {
34
+
return err
35
+
}
36
+
37
+
// Auto-detect CPU count
38
+
if *workers == 0 {
39
+
*workers = runtime.NumCPU()
40
+
}
41
+
42
+
// Get working directory
43
+
dir, err := os.Getwd()
44
+
if err != nil {
45
+
return fmt.Errorf("failed to get working directory: %w", err)
46
+
}
47
+
48
+
if err := os.MkdirAll(dir, 0755); err != nil {
49
+
return fmt.Errorf("failed to create directory: %w", err)
50
+
}
51
+
52
+
// Create manager config
53
+
config := bundle.DefaultConfig(dir)
54
+
config.RebuildWorkers = *workers
55
+
config.RebuildProgress = func(current, total int) {
56
+
if current%100 == 0 || current == total {
57
+
fmt.Printf(" Rebuild progress: %d/%d bundles (%.1f%%) \r",
58
+
current, total, float64(current)/float64(total)*100)
59
+
if current == total {
60
+
fmt.Println()
61
+
}
62
+
}
63
+
}
64
+
65
+
// Create PLC client if sync mode enabled
66
+
var client *plcclient.Client
67
+
if *syncMode {
68
+
client = plcclient.NewClient(*plcURL)
69
+
}
70
+
71
+
fmt.Printf("Starting plcbundle HTTP server...\n")
72
+
fmt.Printf(" Directory: %s\n", dir)
73
+
74
+
// Create manager
75
+
mgr, err := bundle.NewManager(config, client)
76
+
if err != nil {
77
+
return fmt.Errorf("failed to create manager: %w", err)
78
+
}
79
+
defer mgr.Close()
80
+
81
+
// Build/verify DID index if resolver enabled
82
+
if *enableResolver {
83
+
if err := ensureDIDIndex(mgr, *verbose); err != nil {
84
+
fmt.Fprintf(os.Stderr, "⚠️ DID index warning: %v\n\n", err)
85
+
}
86
+
}
87
+
88
+
addr := fmt.Sprintf("%s:%s", *host, *port)
89
+
90
+
// Display server info
91
+
displayServerInfo(mgr, addr, *syncMode, *enableWebSocket, *enableResolver, *plcURL, *syncInterval)
92
+
93
+
// Setup graceful shutdown
94
+
ctx, cancel := setupGracefulShutdown(mgr)
95
+
defer cancel()
96
+
97
+
// Start sync loop if enabled
98
+
if *syncMode {
99
+
go runSyncLoop(ctx, mgr, *syncInterval, *verbose, *enableResolver)
100
+
}
101
+
102
+
// Create and start HTTP server
103
+
serverConfig := &server.Config{
104
+
Addr: addr,
105
+
SyncMode: *syncMode,
106
+
SyncInterval: *syncInterval,
107
+
EnableWebSocket: *enableWebSocket,
108
+
EnableResolver: *enableResolver,
109
+
Version: GetVersion(), // Pass version
110
+
}
111
+
112
+
srv := server.New(mgr, serverConfig)
113
+
114
+
if err := srv.ListenAndServe(); err != nil {
115
+
return fmt.Errorf("server error: %w", err)
116
+
}
117
+
118
+
return nil
119
+
}
120
+
121
+
// ensureDIDIndex builds DID index if needed
122
+
func ensureDIDIndex(mgr *bundle.Manager, verbose bool) error {
123
+
index := mgr.GetIndex()
124
+
bundleCount := index.Count()
125
+
didStats := mgr.GetDIDIndexStats()
126
+
127
+
if bundleCount == 0 {
128
+
return nil
129
+
}
130
+
131
+
needsBuild := false
132
+
reason := ""
133
+
134
+
if !didStats["exists"].(bool) {
135
+
needsBuild = true
136
+
reason = "index does not exist"
137
+
} else {
138
+
// Check version
139
+
didIndex := mgr.GetDIDIndex()
140
+
if didIndex != nil {
141
+
config := didIndex.GetConfig()
142
+
if config.Version != didindex.DIDINDEX_VERSION {
143
+
needsBuild = true
144
+
reason = fmt.Sprintf("index version outdated (v%d, need v%d)",
145
+
config.Version, didindex.DIDINDEX_VERSION)
146
+
} else {
147
+
// Check if index is behind bundles
148
+
lastBundle := index.GetLastBundle()
149
+
if lastBundle != nil && config.LastBundle < lastBundle.BundleNumber {
150
+
needsBuild = true
151
+
reason = fmt.Sprintf("index is behind (bundle %d, need %d)",
152
+
config.LastBundle, lastBundle.BundleNumber)
153
+
}
154
+
}
155
+
}
156
+
}
157
+
158
+
if needsBuild {
159
+
fmt.Printf(" DID Index: BUILDING (%s)\n", reason)
160
+
fmt.Printf(" This may take several minutes...\n\n")
161
+
162
+
buildStart := time.Now()
163
+
ctx := context.Background()
164
+
165
+
progress := ui.NewProgressBar(bundleCount)
166
+
err := mgr.BuildDIDIndex(ctx, func(current, total int) {
167
+
progress.Set(current)
168
+
})
169
+
progress.Finish()
170
+
171
+
if err != nil {
172
+
return fmt.Errorf("failed to build DID index: %w", err)
173
+
}
174
+
175
+
buildTime := time.Since(buildStart)
176
+
updatedStats := mgr.GetDIDIndexStats()
177
+
fmt.Printf("\n✓ DID index built in %s\n", buildTime.Round(time.Millisecond))
178
+
fmt.Printf(" Total DIDs: %s\n\n", formatNumber(int(updatedStats["total_dids"].(int64))))
179
+
} else {
180
+
fmt.Printf(" DID Index: ready (%s DIDs)\n",
181
+
formatNumber(int(didStats["total_dids"].(int64))))
182
+
}
183
+
184
+
// Verify index consistency
185
+
if didStats["exists"].(bool) {
186
+
fmt.Printf(" Verifying index consistency...\n")
187
+
188
+
ctx := context.Background()
189
+
if err := mgr.GetDIDIndex().VerifyAndRepairIndex(ctx, mgr); err != nil {
190
+
return fmt.Errorf("index verification/repair failed: %w", err)
191
+
}
192
+
fmt.Printf(" ✓ Index verified\n")
193
+
}
194
+
195
+
return nil
196
+
}
197
+
198
+
// setupGracefulShutdown sets up signal handling for graceful shutdown
199
+
func setupGracefulShutdown(mgr *bundle.Manager) (context.Context, context.CancelFunc) {
200
+
ctx, cancel := context.WithCancel(context.Background())
201
+
202
+
sigChan := make(chan os.Signal, 1)
203
+
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
204
+
205
+
go func() {
206
+
<-sigChan
207
+
fmt.Fprintf(os.Stderr, "\n\n⚠️ Shutdown signal received...\n")
208
+
fmt.Fprintf(os.Stderr, " Saving mempool...\n")
209
+
210
+
if err := mgr.SaveMempool(); err != nil {
211
+
fmt.Fprintf(os.Stderr, " ✗ Failed to save mempool: %v\n", err)
212
+
} else {
213
+
fmt.Fprintf(os.Stderr, " ✓ Mempool saved\n")
214
+
}
215
+
216
+
fmt.Fprintf(os.Stderr, " Closing DID index...\n")
217
+
if err := mgr.GetDIDIndex().Close(); err != nil {
218
+
fmt.Fprintf(os.Stderr, " ✗ Failed to close index: %v\n", err)
219
+
} else {
220
+
fmt.Fprintf(os.Stderr, " ✓ Index closed\n")
221
+
}
222
+
223
+
fmt.Fprintf(os.Stderr, " ✓ Shutdown complete\n")
224
+
225
+
cancel()
226
+
os.Exit(0)
227
+
}()
228
+
229
+
return ctx, cancel
230
+
}
231
+
232
+
// displayServerInfo shows server configuration
233
+
func displayServerInfo(mgr *bundle.Manager, addr string, syncMode, wsEnabled, resolverEnabled bool, plcURL string, syncInterval time.Duration) {
234
+
fmt.Printf(" Listening: http://%s\n", addr)
235
+
236
+
if syncMode {
237
+
fmt.Printf(" Sync mode: ENABLED\n")
238
+
fmt.Printf(" PLC URL: %s\n", plcURL)
239
+
fmt.Printf(" Sync interval: %s\n", syncInterval)
240
+
} else {
241
+
fmt.Printf(" Sync mode: disabled\n")
242
+
}
243
+
244
+
if wsEnabled {
245
+
wsScheme := "ws"
246
+
fmt.Printf(" WebSocket: ENABLED (%s://%s/ws)\n", wsScheme, addr)
247
+
} else {
248
+
fmt.Printf(" WebSocket: disabled (use --websocket to enable)\n")
249
+
}
250
+
251
+
if resolverEnabled {
252
+
fmt.Printf(" Resolver: ENABLED (/<did> endpoints)\n")
253
+
} else {
254
+
fmt.Printf(" Resolver: disabled (use --resolver to enable)\n")
255
+
}
256
+
257
+
bundleCount := mgr.GetIndex().Count()
258
+
if bundleCount > 0 {
259
+
fmt.Printf(" Bundles available: %d\n", bundleCount)
260
+
} else {
261
+
fmt.Printf(" Bundles available: 0\n")
262
+
}
263
+
264
+
fmt.Printf("\nPress Ctrl+C to stop\n\n")
265
+
}
266
+
267
+
// runSyncLoop runs the background sync loop
268
+
func runSyncLoop(ctx context.Context, mgr *bundle.Manager, interval time.Duration, verbose bool, resolverEnabled bool) {
269
+
// Initial sync
270
+
syncBundles(ctx, mgr, verbose, resolverEnabled)
271
+
272
+
fmt.Fprintf(os.Stderr, "[Sync] Starting sync loop (interval: %s)\n", interval)
273
+
274
+
ticker := time.NewTicker(interval)
275
+
defer ticker.Stop()
276
+
277
+
saveTicker := time.NewTicker(5 * time.Minute)
278
+
defer saveTicker.Stop()
279
+
280
+
for {
281
+
select {
282
+
case <-ctx.Done():
283
+
if err := mgr.SaveMempool(); err != nil {
284
+
fmt.Fprintf(os.Stderr, "[Sync] Failed to save mempool: %v\n", err)
285
+
}
286
+
fmt.Fprintf(os.Stderr, "[Sync] Stopped\n")
287
+
return
288
+
289
+
case <-ticker.C:
290
+
syncBundles(ctx, mgr, verbose, resolverEnabled)
291
+
292
+
case <-saveTicker.C:
293
+
stats := mgr.GetMempoolStats()
294
+
if stats["count"].(int) > 0 && verbose {
295
+
fmt.Fprintf(os.Stderr, "[Sync] Saving mempool (%d ops)\n", stats["count"])
296
+
mgr.SaveMempool()
297
+
}
298
+
}
299
+
}
300
+
}
301
+
302
+
// syncBundles performs a sync cycle
303
+
func syncBundles(ctx context.Context, mgr *bundle.Manager, verbose bool, resolverEnabled bool) {
304
+
cycleStart := time.Now()
305
+
306
+
index := mgr.GetIndex()
307
+
lastBundle := index.GetLastBundle()
308
+
startBundle := 1
309
+
if lastBundle != nil {
310
+
startBundle = lastBundle.BundleNumber + 1
311
+
}
312
+
313
+
isInitialSync := (lastBundle == nil || lastBundle.BundleNumber < 10)
314
+
315
+
if isInitialSync && !verbose {
316
+
fmt.Fprintf(os.Stderr, "[Sync] Initial sync - fast loading mode (bundle %06d → ...)\n", startBundle)
317
+
} else if verbose {
318
+
fmt.Fprintf(os.Stderr, "[Sync] Checking for new bundles (current: %06d)...\n", startBundle-1)
319
+
}
320
+
321
+
mempoolBefore := mgr.GetMempoolStats()["count"].(int)
322
+
fetchedCount := 0
323
+
consecutiveErrors := 0
324
+
325
+
for {
326
+
currentBundle := startBundle + fetchedCount
327
+
328
+
b, err := mgr.FetchNextBundle(ctx, !verbose)
329
+
if err != nil {
330
+
if isEndOfDataError(err) {
331
+
mempoolAfter := mgr.GetMempoolStats()["count"].(int)
332
+
addedOps := mempoolAfter - mempoolBefore
333
+
duration := time.Since(cycleStart)
334
+
335
+
if fetchedCount > 0 {
336
+
fmt.Fprintf(os.Stderr, "[Sync] ✓ Bundle %06d | Synced: %d | Mempool: %d (+%d) | %dms\n",
337
+
currentBundle-1, fetchedCount, mempoolAfter, addedOps, duration.Milliseconds())
338
+
} else if !isInitialSync {
339
+
fmt.Fprintf(os.Stderr, "[Sync] ✓ Bundle %06d | Up to date | Mempool: %d (+%d) | %dms\n",
340
+
startBundle-1, mempoolAfter, addedOps, duration.Milliseconds())
341
+
}
342
+
break
343
+
}
344
+
345
+
consecutiveErrors++
346
+
if verbose {
347
+
fmt.Fprintf(os.Stderr, "[Sync] Error fetching bundle %06d: %v\n", currentBundle, err)
348
+
}
349
+
350
+
if consecutiveErrors >= 3 {
351
+
fmt.Fprintf(os.Stderr, "[Sync] Too many errors, stopping\n")
352
+
break
353
+
}
354
+
355
+
time.Sleep(5 * time.Second)
356
+
continue
357
+
}
358
+
359
+
consecutiveErrors = 0
360
+
361
+
if err := mgr.SaveBundle(ctx, b, !verbose); err != nil {
362
+
fmt.Fprintf(os.Stderr, "[Sync] Error saving bundle %06d: %v\n", b.BundleNumber, err)
363
+
break
364
+
}
365
+
366
+
fetchedCount++
367
+
368
+
if !verbose {
369
+
fmt.Fprintf(os.Stderr, "[Sync] ✓ %06d | hash=%s | content=%s | %d ops, %d DIDs\n",
370
+
b.BundleNumber,
371
+
b.Hash[:16]+"...",
372
+
b.ContentHash[:16]+"...",
373
+
len(b.Operations),
374
+
b.DIDCount)
375
+
}
376
+
377
+
time.Sleep(500 * time.Millisecond)
378
+
}
379
+
}
380
+
381
+
// isEndOfDataError checks if error indicates end of available data
382
+
func isEndOfDataError(err error) bool {
383
+
if err == nil {
384
+
return false
385
+
}
386
+
387
+
errMsg := err.Error()
388
+
return containsAny(errMsg,
389
+
"insufficient operations",
390
+
"no more operations available",
391
+
"reached latest data")
392
+
}
393
+
394
+
// Helper functions
395
+
396
+
func containsAny(s string, substrs ...string) bool {
397
+
for _, substr := range substrs {
398
+
if contains(s, substr) {
399
+
return true
400
+
}
401
+
}
402
+
return false
403
+
}
404
+
405
+
func contains(s, substr string) bool {
406
+
return len(s) >= len(substr) && indexOf(s, substr) >= 0
407
+
}
408
+
409
+
func indexOf(s, substr string) int {
410
+
for i := 0; i <= len(s)-len(substr); i++ {
411
+
if s[i:i+len(substr)] == substr {
412
+
return i
413
+
}
414
+
}
415
+
return -1
416
+
}
+153
cmd/plcbundle/commands/verify.go
+153
cmd/plcbundle/commands/verify.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"context"
5
+
"flag"
6
+
"fmt"
7
+
8
+
"tangled.org/atscan.net/plcbundle/internal/bundle"
9
+
)
10
+
11
+
// VerifyCommand handles the verify subcommand
12
+
func VerifyCommand(args []string) error {
13
+
fs := flag.NewFlagSet("verify", flag.ExitOnError)
14
+
bundleNum := fs.Int("bundle", 0, "specific bundle to verify (0 = verify chain)")
15
+
verbose := fs.Bool("v", false, "verbose output")
16
+
17
+
if err := fs.Parse(args); err != nil {
18
+
return err
19
+
}
20
+
21
+
mgr, dir, err := getManager("")
22
+
if err != nil {
23
+
return err
24
+
}
25
+
defer mgr.Close()
26
+
27
+
fmt.Printf("Working in: %s\n", dir)
28
+
29
+
ctx := context.Background()
30
+
31
+
if *bundleNum > 0 {
32
+
return verifySingleBundle(ctx, mgr, *bundleNum, *verbose)
33
+
}
34
+
35
+
return verifyChain(ctx, mgr, *verbose)
36
+
}
37
+
38
+
func verifySingleBundle(ctx context.Context, mgr *bundle.Manager, bundleNum int, verbose bool) error {
39
+
fmt.Printf("Verifying bundle %06d...\n", bundleNum)
40
+
41
+
result, err := mgr.VerifyBundle(ctx, bundleNum)
42
+
if err != nil {
43
+
return fmt.Errorf("verification failed: %w", err)
44
+
}
45
+
46
+
if result.Valid {
47
+
fmt.Printf("✓ Bundle %06d is valid\n", bundleNum)
48
+
if verbose {
49
+
fmt.Printf(" File exists: %v\n", result.FileExists)
50
+
fmt.Printf(" Hash match: %v\n", result.HashMatch)
51
+
fmt.Printf(" Hash: %s...\n", result.LocalHash[:16])
52
+
}
53
+
return nil
54
+
}
55
+
56
+
fmt.Printf("✗ Bundle %06d is invalid\n", bundleNum)
57
+
if result.Error != nil {
58
+
fmt.Printf(" Error: %v\n", result.Error)
59
+
}
60
+
if !result.FileExists {
61
+
fmt.Printf(" File not found\n")
62
+
}
63
+
if !result.HashMatch && result.FileExists {
64
+
fmt.Printf(" Expected hash: %s...\n", result.ExpectedHash[:16])
65
+
fmt.Printf(" Actual hash: %s...\n", result.LocalHash[:16])
66
+
}
67
+
return fmt.Errorf("bundle verification failed")
68
+
}
69
+
70
+
func verifyChain(ctx context.Context, mgr *bundle.Manager, verbose bool) error {
71
+
index := mgr.GetIndex()
72
+
bundles := index.GetBundles()
73
+
74
+
if len(bundles) == 0 {
75
+
fmt.Println("No bundles to verify")
76
+
return nil
77
+
}
78
+
79
+
fmt.Printf("Verifying chain of %d bundles...\n\n", len(bundles))
80
+
81
+
verifiedCount := 0
82
+
errorCount := 0
83
+
lastPercent := -1
84
+
85
+
for i, meta := range bundles {
86
+
bundleNum := meta.BundleNumber
87
+
88
+
percent := (i * 100) / len(bundles)
89
+
if percent != lastPercent || verbose {
90
+
if verbose {
91
+
fmt.Printf(" [%3d%%] Verifying bundle %06d...", percent, bundleNum)
92
+
} else if percent%10 == 0 && percent != lastPercent {
93
+
fmt.Printf(" [%3d%%] Verified %d/%d bundles...\n", percent, i, len(bundles))
94
+
}
95
+
lastPercent = percent
96
+
}
97
+
98
+
result, err := mgr.VerifyBundle(ctx, bundleNum)
99
+
if err != nil {
100
+
if verbose {
101
+
fmt.Printf(" ERROR\n")
102
+
}
103
+
fmt.Printf("\n✗ Failed to verify bundle %06d: %v\n", bundleNum, err)
104
+
errorCount++
105
+
continue
106
+
}
107
+
108
+
if !result.Valid {
109
+
if verbose {
110
+
fmt.Printf(" INVALID\n")
111
+
}
112
+
fmt.Printf("\n✗ Bundle %06d hash verification failed\n", bundleNum)
113
+
if result.Error != nil {
114
+
fmt.Printf(" Error: %v\n", result.Error)
115
+
}
116
+
errorCount++
117
+
continue
118
+
}
119
+
120
+
if i > 0 {
121
+
prevMeta := bundles[i-1]
122
+
if meta.Parent != prevMeta.Hash {
123
+
if verbose {
124
+
fmt.Printf(" CHAIN BROKEN\n")
125
+
}
126
+
fmt.Printf("\n✗ Chain broken at bundle %06d\n", bundleNum)
127
+
fmt.Printf(" Expected parent: %s...\n", prevMeta.Hash[:16])
128
+
fmt.Printf(" Actual parent: %s...\n", meta.Parent[:16])
129
+
errorCount++
130
+
continue
131
+
}
132
+
}
133
+
134
+
if verbose {
135
+
fmt.Printf(" ✓\n")
136
+
}
137
+
verifiedCount++
138
+
}
139
+
140
+
fmt.Println()
141
+
if errorCount == 0 {
142
+
fmt.Printf("✓ Chain is valid (%d bundles verified)\n", verifiedCount)
143
+
fmt.Printf(" First bundle: %06d\n", bundles[0].BundleNumber)
144
+
fmt.Printf(" Last bundle: %06d\n", bundles[len(bundles)-1].BundleNumber)
145
+
fmt.Printf(" Chain head: %s...\n", bundles[len(bundles)-1].Hash[:16])
146
+
return nil
147
+
}
148
+
149
+
fmt.Printf("✗ Chain verification failed\n")
150
+
fmt.Printf(" Verified: %d/%d bundles\n", verifiedCount, len(bundles))
151
+
fmt.Printf(" Errors: %d\n", errorCount)
152
+
return fmt.Errorf("chain verification failed")
153
+
}
+49
cmd/plcbundle/commands/version.go
+49
cmd/plcbundle/commands/version.go
···
···
1
+
package commands
2
+
3
+
import (
4
+
"fmt"
5
+
"runtime/debug"
6
+
)
7
+
8
+
var (
9
+
version = "dev"
10
+
gitCommit = "unknown"
11
+
buildDate = "unknown"
12
+
)
13
+
14
+
func init() {
15
+
if info, ok := debug.ReadBuildInfo(); ok {
16
+
if info.Main.Version != "" && info.Main.Version != "(devel)" {
17
+
version = info.Main.Version
18
+
}
19
+
20
+
for _, setting := range info.Settings {
21
+
switch setting.Key {
22
+
case "vcs.revision":
23
+
if setting.Value != "" {
24
+
gitCommit = setting.Value
25
+
if len(gitCommit) > 7 {
26
+
gitCommit = gitCommit[:7]
27
+
}
28
+
}
29
+
case "vcs.time":
30
+
if setting.Value != "" {
31
+
buildDate = setting.Value
32
+
}
33
+
}
34
+
}
35
+
}
36
+
}
37
+
38
+
// VersionCommand handles the version subcommand
39
+
func VersionCommand(args []string) error {
40
+
fmt.Printf("plcbundle version %s\n", version)
41
+
fmt.Printf(" commit: %s\n", gitCommit)
42
+
fmt.Printf(" built: %s\n", buildDate)
43
+
return nil
44
+
}
45
+
46
+
// GetVersion returns the version string
47
+
func GetVersion() string {
48
+
return version
49
+
}
-460
cmd/plcbundle/compare.go
-460
cmd/plcbundle/compare.go
···
1
-
package main
2
-
3
-
import (
4
-
"fmt"
5
-
"io"
6
-
"net/http"
7
-
"os"
8
-
"path/filepath"
9
-
"sort"
10
-
"strings"
11
-
"time"
12
-
13
-
"github.com/goccy/go-json"
14
-
"tangled.org/atscan.net/plcbundle/internal/bundle"
15
-
"tangled.org/atscan.net/plcbundle/internal/bundleindex"
16
-
)
17
-
18
-
// IndexComparison holds comparison results
19
-
type IndexComparison struct {
20
-
LocalCount int
21
-
TargetCount int
22
-
CommonCount int
23
-
MissingBundles []int // In target but not in local
24
-
ExtraBundles []int // In local but not in target
25
-
HashMismatches []HashMismatch
26
-
ContentMismatches []HashMismatch
27
-
LocalRange [2]int
28
-
TargetRange [2]int
29
-
LocalTotalSize int64
30
-
TargetTotalSize int64
31
-
LocalUpdated time.Time
32
-
TargetUpdated time.Time
33
-
}
34
-
35
-
type HashMismatch struct {
36
-
BundleNumber int
37
-
LocalHash string // Chain hash
38
-
TargetHash string // Chain hash
39
-
LocalContentHash string // Content hash
40
-
TargetContentHash string // Content hash
41
-
}
42
-
43
-
func (ic *IndexComparison) HasDifferences() bool {
44
-
return len(ic.MissingBundles) > 0 || len(ic.ExtraBundles) > 0 ||
45
-
len(ic.HashMismatches) > 0 || len(ic.ContentMismatches) > 0
46
-
}
47
-
48
-
// loadTargetIndex loads an index from a file or URL
49
-
func loadTargetIndex(target string) (*bundleindex.Index, error) {
50
-
if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") {
51
-
// Load from URL
52
-
return loadIndexFromURL(target)
53
-
}
54
-
55
-
// Load from file
56
-
return bundleindex.LoadIndex(target)
57
-
}
58
-
59
-
// loadIndexFromURL downloads and parses an index from a URL
60
-
func loadIndexFromURL(url string) (*bundleindex.Index, error) {
61
-
// Smart URL handling - if it doesn't end with .json, append /index.json
62
-
if !strings.HasSuffix(url, ".json") {
63
-
url = strings.TrimSuffix(url, "/") + "/index.json"
64
-
}
65
-
66
-
client := &http.Client{
67
-
Timeout: 30 * time.Second,
68
-
}
69
-
70
-
resp, err := client.Get(url)
71
-
if err != nil {
72
-
return nil, fmt.Errorf("failed to download: %w", err)
73
-
}
74
-
defer resp.Body.Close()
75
-
76
-
if resp.StatusCode != http.StatusOK {
77
-
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
78
-
}
79
-
80
-
data, err := io.ReadAll(resp.Body)
81
-
if err != nil {
82
-
return nil, fmt.Errorf("failed to read response: %w", err)
83
-
}
84
-
85
-
var idx bundleindex.Index
86
-
if err := json.Unmarshal(data, &idx); err != nil {
87
-
return nil, fmt.Errorf("failed to parse index: %w", err)
88
-
}
89
-
90
-
return &idx, nil
91
-
}
92
-
93
-
// compareIndexes compares two indexes
94
-
func compareIndexes(local, target *bundleindex.Index) *IndexComparison {
95
-
localBundles := local.GetBundles()
96
-
targetBundles := target.GetBundles()
97
-
98
-
// Create maps for quick lookup
99
-
localMap := make(map[int]*bundleindex.BundleMetadata)
100
-
targetMap := make(map[int]*bundleindex.BundleMetadata)
101
-
102
-
for _, b := range localBundles {
103
-
localMap[b.BundleNumber] = b
104
-
}
105
-
for _, b := range targetBundles {
106
-
targetMap[b.BundleNumber] = b
107
-
}
108
-
109
-
comparison := &IndexComparison{
110
-
LocalCount: len(localBundles),
111
-
TargetCount: len(targetBundles),
112
-
MissingBundles: make([]int, 0),
113
-
ExtraBundles: make([]int, 0),
114
-
HashMismatches: make([]HashMismatch, 0),
115
-
ContentMismatches: make([]HashMismatch, 0),
116
-
}
117
-
118
-
// Get ranges
119
-
if len(localBundles) > 0 {
120
-
comparison.LocalRange = [2]int{localBundles[0].BundleNumber, localBundles[len(localBundles)-1].BundleNumber}
121
-
comparison.LocalUpdated = local.UpdatedAt
122
-
localStats := local.GetStats()
123
-
comparison.LocalTotalSize = localStats["total_size"].(int64)
124
-
}
125
-
126
-
if len(targetBundles) > 0 {
127
-
comparison.TargetRange = [2]int{targetBundles[0].BundleNumber, targetBundles[len(targetBundles)-1].BundleNumber}
128
-
comparison.TargetUpdated = target.UpdatedAt
129
-
targetStats := target.GetStats()
130
-
comparison.TargetTotalSize = targetStats["total_size"].(int64)
131
-
}
132
-
133
-
// Find missing bundles (in target but not in local)
134
-
for bundleNum := range targetMap {
135
-
if _, exists := localMap[bundleNum]; !exists {
136
-
comparison.MissingBundles = append(comparison.MissingBundles, bundleNum)
137
-
}
138
-
}
139
-
sort.Ints(comparison.MissingBundles)
140
-
141
-
// Find extra bundles (in local but not in target)
142
-
for bundleNum := range localMap {
143
-
if _, exists := targetMap[bundleNum]; !exists {
144
-
comparison.ExtraBundles = append(comparison.ExtraBundles, bundleNum)
145
-
}
146
-
}
147
-
sort.Ints(comparison.ExtraBundles)
148
-
149
-
// Compare hashes (Hash = chain hash, ContentHash = content hash)
150
-
for bundleNum, localMeta := range localMap {
151
-
if targetMeta, exists := targetMap[bundleNum]; exists {
152
-
comparison.CommonCount++
153
-
154
-
// Hash field is now the CHAIN HASH (most important!)
155
-
chainMismatch := localMeta.Hash != targetMeta.Hash
156
-
contentMismatch := localMeta.ContentHash != targetMeta.ContentHash
157
-
158
-
if chainMismatch || contentMismatch {
159
-
mismatch := HashMismatch{
160
-
BundleNumber: bundleNum,
161
-
LocalHash: localMeta.Hash, // Chain hash
162
-
TargetHash: targetMeta.Hash, // Chain hash
163
-
LocalContentHash: localMeta.ContentHash, // Content hash
164
-
TargetContentHash: targetMeta.ContentHash, // Content hash
165
-
}
166
-
167
-
// Separate chain hash mismatches (critical) from content mismatches
168
-
if chainMismatch {
169
-
comparison.HashMismatches = append(comparison.HashMismatches, mismatch)
170
-
}
171
-
if contentMismatch && !chainMismatch {
172
-
// Content mismatch but chain hash matches (unlikely but possible)
173
-
comparison.ContentMismatches = append(comparison.ContentMismatches, mismatch)
174
-
}
175
-
}
176
-
}
177
-
}
178
-
179
-
// Sort mismatches by bundle number
180
-
sort.Slice(comparison.HashMismatches, func(i, j int) bool {
181
-
return comparison.HashMismatches[i].BundleNumber < comparison.HashMismatches[j].BundleNumber
182
-
})
183
-
sort.Slice(comparison.ContentMismatches, func(i, j int) bool {
184
-
return comparison.ContentMismatches[i].BundleNumber < comparison.ContentMismatches[j].BundleNumber
185
-
})
186
-
187
-
return comparison
188
-
}
189
-
190
-
// displayComparison displays the comparison results
191
-
func displayComparison(c *IndexComparison, verbose bool) {
192
-
fmt.Printf("Comparison Results\n")
193
-
fmt.Printf("══════════════════\n\n")
194
-
195
-
// Summary
196
-
fmt.Printf("Summary\n")
197
-
fmt.Printf("───────\n")
198
-
fmt.Printf(" Local bundles: %d\n", c.LocalCount)
199
-
fmt.Printf(" Target bundles: %d\n", c.TargetCount)
200
-
fmt.Printf(" Common bundles: %d\n", c.CommonCount)
201
-
fmt.Printf(" Missing bundles: %s\n", formatCount(len(c.MissingBundles)))
202
-
fmt.Printf(" Extra bundles: %s\n", formatCount(len(c.ExtraBundles)))
203
-
fmt.Printf(" Hash mismatches: %s\n", formatCountCritical(len(c.HashMismatches)))
204
-
fmt.Printf(" Content mismatches: %s\n", formatCount(len(c.ContentMismatches)))
205
-
206
-
if c.LocalCount > 0 {
207
-
fmt.Printf("\n Local range: %06d - %06d\n", c.LocalRange[0], c.LocalRange[1])
208
-
fmt.Printf(" Local size: %.2f MB\n", float64(c.LocalTotalSize)/(1024*1024))
209
-
fmt.Printf(" Local updated: %s\n", c.LocalUpdated.Format("2006-01-02 15:04:05"))
210
-
}
211
-
212
-
if c.TargetCount > 0 {
213
-
fmt.Printf("\n Target range: %06d - %06d\n", c.TargetRange[0], c.TargetRange[1])
214
-
fmt.Printf(" Target size: %.2f MB\n", float64(c.TargetTotalSize)/(1024*1024))
215
-
fmt.Printf(" Target updated: %s\n", c.TargetUpdated.Format("2006-01-02 15:04:05"))
216
-
}
217
-
218
-
// Hash mismatches (CHAIN HASH - MOST CRITICAL)
219
-
if len(c.HashMismatches) > 0 {
220
-
fmt.Printf("\n")
221
-
fmt.Printf("⚠️ CHAIN HASH MISMATCHES (CRITICAL)\n")
222
-
fmt.Printf("════════════════════════════════════\n")
223
-
fmt.Printf("Chain hashes validate the entire bundle history.\n")
224
-
fmt.Printf("Mismatches indicate different bundle content or chain breaks.\n")
225
-
fmt.Printf("\n")
226
-
227
-
displayCount := len(c.HashMismatches)
228
-
if displayCount > 10 && !verbose {
229
-
displayCount = 10
230
-
}
231
-
232
-
for i := 0; i < displayCount; i++ {
233
-
m := c.HashMismatches[i]
234
-
fmt.Printf(" Bundle %06d:\n", m.BundleNumber)
235
-
236
-
// Show chain hashes (primary)
237
-
fmt.Printf(" Chain Hash:\n")
238
-
fmt.Printf(" Local: %s\n", m.LocalHash)
239
-
fmt.Printf(" Target: %s\n", m.TargetHash)
240
-
241
-
// Also show content hash if different
242
-
if m.LocalContentHash != m.TargetContentHash {
243
-
fmt.Printf(" Content Hash (also differs):\n")
244
-
fmt.Printf(" Local: %s\n", m.LocalContentHash)
245
-
fmt.Printf(" Target: %s\n", m.TargetContentHash)
246
-
}
247
-
fmt.Printf("\n")
248
-
}
249
-
250
-
if len(c.HashMismatches) > displayCount {
251
-
fmt.Printf(" ... and %d more (use -v to show all)\n\n", len(c.HashMismatches)-displayCount)
252
-
}
253
-
}
254
-
255
-
// Content hash mismatches (chain hash matches - unlikely but possible)
256
-
if len(c.ContentMismatches) > 0 {
257
-
fmt.Printf("\n")
258
-
fmt.Printf("Content Hash Mismatches (chain hash matches)\n")
259
-
fmt.Printf("─────────────────────────────────────────────\n")
260
-
fmt.Printf("This is unusual - content differs but chain hash matches.\n")
261
-
fmt.Printf("\n")
262
-
263
-
displayCount := len(c.ContentMismatches)
264
-
if displayCount > 10 && !verbose {
265
-
displayCount = 10
266
-
}
267
-
268
-
for i := 0; i < displayCount; i++ {
269
-
m := c.ContentMismatches[i]
270
-
fmt.Printf(" Bundle %06d:\n", m.BundleNumber)
271
-
fmt.Printf(" Content Hash:\n")
272
-
fmt.Printf(" Local: %s\n", m.LocalContentHash)
273
-
fmt.Printf(" Target: %s\n", m.TargetContentHash)
274
-
fmt.Printf(" Chain Hash (matches):\n")
275
-
fmt.Printf(" Both: %s\n", m.LocalHash)
276
-
}
277
-
278
-
if len(c.ContentMismatches) > displayCount {
279
-
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.ContentMismatches)-displayCount)
280
-
}
281
-
}
282
-
283
-
// Missing bundles
284
-
if len(c.MissingBundles) > 0 {
285
-
fmt.Printf("\n")
286
-
fmt.Printf("Missing Bundles (in target but not local)\n")
287
-
fmt.Printf("──────────────────────────────────────────\n")
288
-
289
-
if verbose || len(c.MissingBundles) <= 20 {
290
-
displayCount := len(c.MissingBundles)
291
-
if displayCount > 20 && !verbose {
292
-
displayCount = 20
293
-
}
294
-
295
-
for i := 0; i < displayCount; i++ {
296
-
fmt.Printf(" %06d\n", c.MissingBundles[i])
297
-
}
298
-
299
-
if len(c.MissingBundles) > displayCount {
300
-
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.MissingBundles)-displayCount)
301
-
}
302
-
} else {
303
-
displayBundleRanges(c.MissingBundles)
304
-
}
305
-
}
306
-
307
-
// Extra bundles
308
-
if len(c.ExtraBundles) > 0 {
309
-
fmt.Printf("\n")
310
-
fmt.Printf("Extra Bundles (in local but not target)\n")
311
-
fmt.Printf("────────────────────────────────────────\n")
312
-
313
-
if verbose || len(c.ExtraBundles) <= 20 {
314
-
displayCount := len(c.ExtraBundles)
315
-
if displayCount > 20 && !verbose {
316
-
displayCount = 20
317
-
}
318
-
319
-
for i := 0; i < displayCount; i++ {
320
-
fmt.Printf(" %06d\n", c.ExtraBundles[i])
321
-
}
322
-
323
-
if len(c.ExtraBundles) > displayCount {
324
-
fmt.Printf(" ... and %d more (use -v to show all)\n", len(c.ExtraBundles)-displayCount)
325
-
}
326
-
} else {
327
-
displayBundleRanges(c.ExtraBundles)
328
-
}
329
-
}
330
-
331
-
// Final status
332
-
fmt.Printf("\n")
333
-
if !c.HasDifferences() {
334
-
fmt.Printf("✓ Indexes are identical\n")
335
-
} else {
336
-
fmt.Printf("✗ Indexes have differences\n")
337
-
if len(c.HashMismatches) > 0 {
338
-
fmt.Printf("\n⚠️ WARNING: Chain hash mismatches detected!\n")
339
-
fmt.Printf("This indicates different bundle content or chain integrity issues.\n")
340
-
}
341
-
}
342
-
}
343
-
344
-
// formatCount formats a count with color/symbol
345
-
func formatCount(count int) string {
346
-
if count == 0 {
347
-
return "\033[32m0 ✓\033[0m" // Green with checkmark
348
-
}
349
-
return fmt.Sprintf("\033[33m%d ⚠️\033[0m", count) // Yellow with warning
350
-
}
351
-
352
-
// formatCountCritical formats a count for critical items (chain mismatches)
353
-
func formatCountCritical(count int) string {
354
-
if count == 0 {
355
-
return "\033[32m0 ✓\033[0m" // Green with checkmark
356
-
}
357
-
return fmt.Sprintf("\033[31m%d ✗\033[0m", count) // Red with X
358
-
}
359
-
360
-
// displayBundleRanges displays bundle numbers as ranges
361
-
func displayBundleRanges(bundles []int) {
362
-
if len(bundles) == 0 {
363
-
return
364
-
}
365
-
366
-
rangeStart := bundles[0]
367
-
rangeEnd := bundles[0]
368
-
369
-
for i := 1; i < len(bundles); i++ {
370
-
if bundles[i] == rangeEnd+1 {
371
-
rangeEnd = bundles[i]
372
-
} else {
373
-
// Print current range
374
-
if rangeStart == rangeEnd {
375
-
fmt.Printf(" %06d\n", rangeStart)
376
-
} else {
377
-
fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd)
378
-
}
379
-
rangeStart = bundles[i]
380
-
rangeEnd = bundles[i]
381
-
}
382
-
}
383
-
384
-
// Print last range
385
-
if rangeStart == rangeEnd {
386
-
fmt.Printf(" %06d\n", rangeStart)
387
-
} else {
388
-
fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd)
389
-
}
390
-
}
391
-
392
-
// fetchMissingBundles downloads missing bundles from target server
393
-
func fetchMissingBundles(mgr *bundle.Manager, baseURL string, missingBundles []int) {
394
-
client := &http.Client{
395
-
Timeout: 60 * time.Second,
396
-
}
397
-
398
-
successCount := 0
399
-
errorCount := 0
400
-
401
-
for _, bundleNum := range missingBundles {
402
-
fmt.Printf("Fetching bundle %06d... ", bundleNum)
403
-
404
-
// Download bundle data
405
-
url := fmt.Sprintf("%s/data/%d", baseURL, bundleNum)
406
-
resp, err := client.Get(url)
407
-
if err != nil {
408
-
fmt.Printf("ERROR: %v\n", err)
409
-
errorCount++
410
-
continue
411
-
}
412
-
413
-
if resp.StatusCode != http.StatusOK {
414
-
fmt.Printf("ERROR: status %d\n", resp.StatusCode)
415
-
resp.Body.Close()
416
-
errorCount++
417
-
continue
418
-
}
419
-
420
-
// Save to file
421
-
filename := fmt.Sprintf("%06d.jsonl.zst", bundleNum)
422
-
filepath := filepath.Join(mgr.GetInfo()["bundle_dir"].(string), filename)
423
-
424
-
outFile, err := os.Create(filepath)
425
-
if err != nil {
426
-
fmt.Printf("ERROR: %v\n", err)
427
-
resp.Body.Close()
428
-
errorCount++
429
-
continue
430
-
}
431
-
432
-
_, err = io.Copy(outFile, resp.Body)
433
-
outFile.Close()
434
-
resp.Body.Close()
435
-
436
-
if err != nil {
437
-
fmt.Printf("ERROR: %v\n", err)
438
-
os.Remove(filepath)
439
-
errorCount++
440
-
continue
441
-
}
442
-
443
-
// Scan and index the bundle
444
-
_, err = mgr.ScanAndIndexBundle(filepath, bundleNum)
445
-
if err != nil {
446
-
fmt.Printf("ERROR: %v\n", err)
447
-
errorCount++
448
-
continue
449
-
}
450
-
451
-
fmt.Printf("✓\n")
452
-
successCount++
453
-
454
-
// Small delay to be nice
455
-
time.Sleep(200 * time.Millisecond)
456
-
}
457
-
458
-
fmt.Printf("\n")
459
-
fmt.Printf("✓ Fetch complete: %d succeeded, %d failed\n", successCount, errorCount)
460
-
}
···
+150
-258
cmd/plcbundle/detector.go
cmd/plcbundle/commands/detector.go
+150
-258
cmd/plcbundle/detector.go
cmd/plcbundle/commands/detector.go
···
1
-
// cmd/plcbundle/detector.go
2
-
package main
3
4
import (
5
"bufio"
···
11
"os"
12
"sort"
13
"strings"
14
-
"time"
15
16
"github.com/goccy/go-json"
17
-
18
"tangled.org/atscan.net/plcbundle/detector"
19
"tangled.org/atscan.net/plcbundle/plcclient"
20
)
21
22
-
type defaultLogger struct{}
23
-
24
-
func (d *defaultLogger) Printf(format string, v ...interface{}) {
25
-
fmt.Fprintf(os.Stderr, format+"\n", v...)
26
-
}
27
-
28
-
func cmdDetector() {
29
-
if len(os.Args) < 3 {
30
printDetectorUsage()
31
-
os.Exit(1)
32
}
33
34
-
subcommand := os.Args[2]
35
36
switch subcommand {
37
case "list":
38
-
cmdDetectorList()
39
case "test":
40
-
cmdDetectorTest()
41
case "run":
42
-
cmdDetectorRun()
43
case "filter":
44
-
cmdDetectorFilter()
45
case "info":
46
-
cmdDetectorInfo()
47
default:
48
-
fmt.Fprintf(os.Stderr, "Unknown detector subcommand: %s\n", subcommand)
49
printDetectorUsage()
50
-
os.Exit(1)
51
}
52
}
53
···
62
info Show detailed detector information
63
64
Examples:
65
-
# List all built-in detectors
66
plcbundle detector list
67
-
68
-
# Run built-in detector
69
plcbundle detector run invalid_handle --bundles 1-100
70
-
71
-
# Run custom JavaScript detector
72
plcbundle detector run ./my_detector.js --bundles 1-100
73
-
74
-
# Run multiple detectors (built-in + custom)
75
-
plcbundle detector run invalid_handle ./my_detector.js --bundles 1-100
76
-
77
-
# Run all built-in detectors
78
plcbundle detector run all --bundles 1-100
79
-
80
-
# Filter with custom detector
81
plcbundle backfill | plcbundle detector filter ./my_detector.js > clean.jsonl
82
`)
83
}
84
85
-
// cmdDetectorFilter reads JSONL from stdin, filters OUT spam, outputs clean operations
86
-
func cmdDetectorFilter() {
87
-
if len(os.Args) < 4 {
88
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle detector filter <detector1|script.js> [detector2...] [--confidence 0.9]\n")
89
-
os.Exit(1)
90
-
}
91
-
92
-
// Parse detector names and flags
93
-
var detectorNames []string
94
-
var flagArgs []string
95
-
for i := 3; i < len(os.Args); i++ {
96
-
if strings.HasPrefix(os.Args[i], "-") {
97
-
flagArgs = os.Args[i:]
98
-
break
99
-
}
100
-
detectorNames = append(detectorNames, os.Args[i])
101
-
}
102
-
103
-
if len(detectorNames) == 0 {
104
-
fmt.Fprintf(os.Stderr, "Error: at least one detector name required\n")
105
-
os.Exit(1)
106
-
}
107
-
108
-
fs := flag.NewFlagSet("detector filter", flag.ExitOnError)
109
-
confidence := fs.Float64("confidence", 0.90, "minimum confidence")
110
-
fs.Parse(flagArgs)
111
-
112
-
// Load detectors (common logic)
113
-
setup, err := parseAndLoadDetectors(detectorNames, *confidence)
114
-
if err != nil {
115
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
116
-
os.Exit(1)
117
-
}
118
-
defer setup.cleanup()
119
-
120
-
fmt.Fprintf(os.Stderr, "Filtering with %d detector(s), min confidence: %.2f\n\n", len(setup.detectors), *confidence)
121
-
122
-
ctx := context.Background()
123
-
scanner := bufio.NewScanner(os.Stdin)
124
-
buf := make([]byte, 0, 64*1024)
125
-
scanner.Buffer(buf, 1024*1024)
126
-
127
-
cleanCount, filteredCount, totalCount := 0, 0, 0
128
-
totalBytes, filteredBytes := int64(0), int64(0)
129
-
130
-
for scanner.Scan() {
131
-
line := scanner.Bytes()
132
-
if len(line) == 0 {
133
-
continue
134
-
}
135
-
136
-
totalCount++
137
-
totalBytes += int64(len(line))
138
-
139
-
var op plcclient.PLCOperation
140
-
if err := json.Unmarshal(line, &op); err != nil {
141
-
continue
142
-
}
143
-
144
-
// Run detection (common logic)
145
-
labels, _ := detectOperation(ctx, setup.detectors, op, setup.confidence)
146
-
147
-
if len(labels) == 0 {
148
-
cleanCount++
149
-
fmt.Println(string(line))
150
-
} else {
151
-
filteredCount++
152
-
filteredBytes += int64(len(line))
153
-
}
154
-
155
-
if totalCount%1000 == 0 {
156
-
fmt.Fprintf(os.Stderr, "Processed: %d | Clean: %d | Filtered: %d\r", totalCount, cleanCount, filteredCount)
157
-
}
158
-
}
159
-
160
-
// Stats
161
-
fmt.Fprintf(os.Stderr, "\n\n✓ Filter complete\n")
162
-
fmt.Fprintf(os.Stderr, " Total: %d | Clean: %d (%.2f%%) | Filtered: %d (%.2f%%)\n",
163
-
totalCount, cleanCount, float64(cleanCount)/float64(totalCount)*100,
164
-
filteredCount, float64(filteredCount)/float64(totalCount)*100)
165
-
fmt.Fprintf(os.Stderr, " Size saved: %s (%.2f%%)\n", formatBytes(filteredBytes), float64(filteredBytes)/float64(totalBytes)*100)
166
-
}
167
-
168
-
func cmdDetectorList() {
169
registry := detector.DefaultRegistry()
170
detectors := registry.List()
171
172
-
// Sort by name
173
sort.Slice(detectors, func(i, j int) bool {
174
return detectors[i].Name() < detectors[j].Name()
175
})
···
179
fmt.Printf(" %-20s %s (v%s)\n", d.Name(), d.Description(), d.Version())
180
}
181
fmt.Printf("\nUse 'plcbundle detector info <name>' for details\n")
182
}
183
184
-
func cmdDetectorTest() {
185
-
// Extract detector name first
186
-
if len(os.Args) < 4 {
187
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle detector test <detector-name> --bundle N\n")
188
-
os.Exit(1)
189
}
190
191
-
detectorName := os.Args[3]
192
193
-
// Parse flags from os.Args[4:]
194
fs := flag.NewFlagSet("detector test", flag.ExitOnError)
195
bundleNum := fs.Int("bundle", 0, "bundle number to test")
196
confidence := fs.Float64("confidence", 0.90, "minimum confidence threshold")
197
verbose := fs.Bool("v", false, "verbose output")
198
-
fs.Parse(os.Args[4:])
199
200
if *bundleNum == 0 {
201
-
fmt.Fprintf(os.Stderr, "Error: --bundle required\n")
202
-
os.Exit(1)
203
}
204
205
-
// Load bundle
206
mgr, _, err := getManager("")
207
if err != nil {
208
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
209
-
os.Exit(1)
210
}
211
defer mgr.Close()
212
213
ctx := context.Background()
214
bundle, err := mgr.LoadBundle(ctx, *bundleNum)
215
if err != nil {
216
-
fmt.Fprintf(os.Stderr, "Error loading bundle: %v\n", err)
217
-
os.Exit(1)
218
}
219
220
fmt.Printf("Testing detector '%s' on bundle %06d...\n", detectorName, *bundleNum)
221
fmt.Printf("Min confidence: %.2f\n\n", *confidence)
222
223
-
// Run detector
224
registry := detector.DefaultRegistry()
225
config := detector.DefaultConfig()
226
config.MinConfidence = *confidence
···
228
runner := detector.NewRunner(registry, config, &defaultLogger{})
229
results, err := runner.RunOnBundle(ctx, detectorName, bundle)
230
if err != nil {
231
-
fmt.Fprintf(os.Stderr, "Detection failed: %v\n", err)
232
-
os.Exit(1)
233
}
234
235
-
// Calculate stats
236
stats := detector.CalculateStats(results, len(bundle.Operations))
237
238
-
// Display results
239
fmt.Printf("Results:\n")
240
fmt.Printf(" Total operations: %d\n", stats.TotalOperations)
241
-
fmt.Printf(" Matches found: %d (%.2f%%)\n", stats.MatchedCount, stats.MatchRate*100)
242
-
fmt.Printf("\n")
243
244
if len(stats.ByReason) > 0 {
245
fmt.Printf("Breakdown by reason:\n")
···
250
fmt.Printf("\n")
251
}
252
253
-
if len(stats.ByCategory) > 0 {
254
-
fmt.Printf("Breakdown by category:\n")
255
-
for category, count := range stats.ByCategory {
256
-
pct := float64(count) / float64(stats.MatchedCount) * 100
257
-
fmt.Printf(" %-25s %d (%.1f%%)\n", category, count, pct)
258
-
}
259
-
fmt.Printf("\n")
260
-
}
261
-
262
-
if len(stats.ByConfidence) > 0 {
263
-
fmt.Printf("Confidence distribution:\n")
264
-
for bucket, count := range stats.ByConfidence {
265
-
pct := float64(count) / float64(stats.MatchedCount) * 100
266
-
fmt.Printf(" %-25s %d (%.1f%%)\n", bucket, count, pct)
267
-
}
268
-
fmt.Printf("\n")
269
-
}
270
-
271
if *verbose && len(results) > 0 {
272
fmt.Printf("Sample matches (first 10):\n")
273
displayCount := 10
···
283
fmt.Printf(" Note: %s\n", res.Match.Note)
284
}
285
}
286
-
287
-
if len(results) > displayCount {
288
-
fmt.Printf(" ... and %d more\n", len(results)-displayCount)
289
-
}
290
}
291
}
292
293
-
func cmdDetectorRun() {
294
-
if len(os.Args) < 4 {
295
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle detector run <detector1|script.js> [detector2...] [--bundles 1-100]\n")
296
-
os.Exit(1)
297
}
298
299
-
// Parse detector names and flags
300
var detectorNames []string
301
var flagArgs []string
302
-
for i := 3; i < len(os.Args); i++ {
303
-
if strings.HasPrefix(os.Args[i], "-") {
304
-
flagArgs = os.Args[i:]
305
break
306
}
307
-
detectorNames = append(detectorNames, os.Args[i])
308
}
309
310
if len(detectorNames) == 0 {
311
-
fmt.Fprintf(os.Stderr, "Error: at least one detector name required\n")
312
-
os.Exit(1)
313
}
314
315
fs := flag.NewFlagSet("detector run", flag.ExitOnError)
316
-
bundleRange := fs.String("bundles", "", "bundle range, default: all bundles")
317
confidence := fs.Float64("confidence", 0.90, "minimum confidence")
318
pprofPort := fs.String("pprof", "", "enable pprof on port (e.g., :6060)")
319
-
fs.Parse(flagArgs)
320
321
-
// Start pprof server if requested
322
if *pprofPort != "" {
323
go func() {
324
fmt.Fprintf(os.Stderr, "pprof server starting on http://localhost%s/debug/pprof/\n", *pprofPort)
325
-
if err := http.ListenAndServe(*pprofPort, nil); err != nil {
326
-
fmt.Fprintf(os.Stderr, "pprof server failed: %v\n", err)
327
-
}
328
}()
329
-
time.Sleep(100 * time.Millisecond) // Let server start
330
}
331
332
-
// Load manager
333
mgr, _, err := getManager("")
334
if err != nil {
335
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
336
-
os.Exit(1)
337
}
338
defer mgr.Close()
339
340
-
// Determine bundle range
341
var start, end int
342
if *bundleRange == "" {
343
index := mgr.GetIndex()
344
bundles := index.GetBundles()
345
if len(bundles) == 0 {
346
-
fmt.Fprintf(os.Stderr, "Error: no bundles available\n")
347
-
os.Exit(1)
348
}
349
start = bundles[0].BundleNumber
350
end = bundles[len(bundles)-1].BundleNumber
···
352
} else {
353
start, end, err = parseBundleRange(*bundleRange)
354
if err != nil {
355
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
356
-
os.Exit(1)
357
}
358
}
359
360
-
// Load detectors (common logic)
361
setup, err := parseAndLoadDetectors(detectorNames, *confidence)
362
if err != nil {
363
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
364
-
os.Exit(1)
365
}
366
defer setup.cleanup()
367
···
371
ctx := context.Background()
372
fmt.Println("bundle,position,cid,size,confidence,labels")
373
374
-
// Stats
375
totalOps, matchCount := 0, 0
376
totalBytes, matchedBytes := int64(0), int64(0)
377
-
totalBundles := end - start + 1
378
-
progress := NewProgressBar(totalBundles)
379
-
progress.showBytes = true
380
381
-
// Process bundles
382
for bundleNum := start; bundleNum <= end; bundleNum++ {
383
bundle, err := mgr.LoadBundle(ctx, bundleNum)
384
if err != nil {
···
395
}
396
totalBytes += int64(opSize)
397
398
-
// Run detection (common logic)
399
-
labels, confidence := detectOperation(ctx, setup.detectors, op, setup.confidence)
400
401
if len(labels) > 0 {
402
matchCount++
···
408
}
409
410
fmt.Printf("%d,%d,%s,%d,%.2f,%s\n",
411
-
bundleNum, position, cidShort, opSize, confidence, strings.Join(labels, ";"))
412
}
413
}
414
-
415
-
progress.SetWithBytes(bundleNum-start+1, totalBytes)
416
}
417
418
-
progress.Finish()
419
-
420
-
// Stats
421
fmt.Fprintf(os.Stderr, "\n✓ Detection complete\n")
422
fmt.Fprintf(os.Stderr, " Total operations: %d\n", totalOps)
423
fmt.Fprintf(os.Stderr, " Matches found: %d (%.2f%%)\n", matchCount, float64(matchCount)/float64(totalOps)*100)
424
-
fmt.Fprintf(os.Stderr, " Total size: %s\n", formatBytes(totalBytes))
425
-
fmt.Fprintf(os.Stderr, " Matched size: %s (%.2f%%)\n", formatBytes(matchedBytes), float64(matchedBytes)/float64(totalBytes)*100)
426
}
427
428
-
func cmdDetectorInfo() {
429
-
if len(os.Args) < 4 {
430
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle detector info <name>\n")
431
-
os.Exit(1)
432
}
433
434
-
detectorName := os.Args[3]
435
436
-
registry := detector.DefaultRegistry()
437
-
d, err := registry.Get(detectorName)
438
if err != nil {
439
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
440
-
os.Exit(1)
441
}
442
443
-
fmt.Printf("Detector: %s\n", d.Name())
444
-
fmt.Printf("Version: %s\n", d.Version())
445
-
fmt.Printf("Description: %s\n", d.Description())
446
-
fmt.Printf("\n")
447
448
-
// Show example usage
449
-
fmt.Printf("Usage examples:\n")
450
-
fmt.Printf(" # Test on single bundle\n")
451
-
fmt.Printf(" plcbundle detector test %s --bundle 42\n\n", d.Name())
452
-
fmt.Printf(" # Run on range and save\n")
453
-
fmt.Printf(" plcbundle detector run %s --bundles 1-100 --output results.csv\n\n", d.Name())
454
-
fmt.Printf(" # Use with filter creation\n")
455
-
fmt.Printf(" plcbundle filter detect --detector %s --bundles 1-100\n", d.Name())
456
-
}
457
458
-
// Helper functions
459
460
-
func parseBundleRange(rangeStr string) (start, end int, err error) {
461
-
// Handle single bundle number
462
-
if !strings.Contains(rangeStr, "-") {
463
-
var num int
464
-
_, err = fmt.Sscanf(rangeStr, "%d", &num)
465
-
if err != nil {
466
-
return 0, 0, fmt.Errorf("invalid bundle number: %w", err)
467
}
468
-
return num, num, nil
469
-
}
470
471
-
// Handle range (e.g., "1-100")
472
-
parts := strings.Split(rangeStr, "-")
473
-
if len(parts) != 2 {
474
-
return 0, 0, fmt.Errorf("invalid range format (expected: N or start-end)")
475
}
476
477
-
_, err = fmt.Sscanf(parts[0], "%d", &start)
478
-
if err != nil {
479
-
return 0, 0, fmt.Errorf("invalid start: %w", err)
480
}
481
482
-
_, err = fmt.Sscanf(parts[1], "%d", &end)
483
if err != nil {
484
-
return 0, 0, fmt.Errorf("invalid end: %w", err)
485
}
486
487
-
if start > end {
488
-
return 0, 0, fmt.Errorf("start must be <= end")
489
-
}
490
491
-
return start, end, nil
492
}
493
494
-
// Common detector setup
495
type detectorSetup struct {
496
detectors []detector.Detector
497
scriptDetectors []interface{ Close() error }
···
504
}
505
}
506
507
-
// parseAndLoadDetectors handles common detector loading logic
508
func parseAndLoadDetectors(detectorNames []string, confidence float64) (*detectorSetup, error) {
509
registry := detector.DefaultRegistry()
510
···
521
522
for _, name := range detectorNames {
523
if strings.HasSuffix(name, ".js") {
524
-
sd, err := detector.NewScriptDetector(name) // Simple single process
525
if err != nil {
526
setup.cleanup()
527
return nil, fmt.Errorf("error loading script %s: %w", name, err)
···
543
return setup, nil
544
}
545
546
-
// detectOperation runs all detectors on an operation and returns labels + confidence
547
func detectOperation(ctx context.Context, detectors []detector.Detector, op plcclient.PLCOperation, minConfidence float64) ([]string, float64) {
548
-
// Parse Operation ONCE before running detectors
549
opData, err := op.GetOperationData()
550
if err != nil {
551
return nil, 0
552
}
553
-
op.ParsedOperation = opData // Set for detectors to use
554
555
var matchedLabels []string
556
var maxConfidence float64
···
561
continue
562
}
563
564
-
// Extract labels
565
var labels []string
566
if labelList, ok := match.Metadata["labels"].([]string); ok {
567
labels = labelList
···
585
586
return matchedLabels, maxConfidence
587
}
···
1
+
package commands
2
3
import (
4
"bufio"
···
10
"os"
11
"sort"
12
"strings"
13
14
"github.com/goccy/go-json"
15
"tangled.org/atscan.net/plcbundle/detector"
16
"tangled.org/atscan.net/plcbundle/plcclient"
17
)
18
19
+
// DetectorCommand handles the detector subcommand
20
+
func DetectorCommand(args []string) error {
21
+
if len(args) < 1 {
22
printDetectorUsage()
23
+
return fmt.Errorf("subcommand required")
24
}
25
26
+
subcommand := args[0]
27
28
switch subcommand {
29
case "list":
30
+
return detectorList(args[1:])
31
case "test":
32
+
return detectorTest(args[1:])
33
case "run":
34
+
return detectorRun(args[1:])
35
case "filter":
36
+
return detectorFilter(args[1:])
37
case "info":
38
+
return detectorInfo(args[1:])
39
default:
40
printDetectorUsage()
41
+
return fmt.Errorf("unknown detector subcommand: %s", subcommand)
42
}
43
}
44
···
53
info Show detailed detector information
54
55
Examples:
56
plcbundle detector list
57
plcbundle detector run invalid_handle --bundles 1-100
58
plcbundle detector run ./my_detector.js --bundles 1-100
59
plcbundle detector run all --bundles 1-100
60
plcbundle backfill | plcbundle detector filter ./my_detector.js > clean.jsonl
61
`)
62
}
63
64
+
func detectorList(args []string) error {
65
registry := detector.DefaultRegistry()
66
detectors := registry.List()
67
68
sort.Slice(detectors, func(i, j int) bool {
69
return detectors[i].Name() < detectors[j].Name()
70
})
···
74
fmt.Printf(" %-20s %s (v%s)\n", d.Name(), d.Description(), d.Version())
75
}
76
fmt.Printf("\nUse 'plcbundle detector info <name>' for details\n")
77
+
78
+
return nil
79
}
80
81
+
func detectorTest(args []string) error {
82
+
if len(args) < 1 {
83
+
return fmt.Errorf("usage: plcbundle detector test <detector-name> --bundle N")
84
}
85
86
+
detectorName := args[0]
87
88
fs := flag.NewFlagSet("detector test", flag.ExitOnError)
89
bundleNum := fs.Int("bundle", 0, "bundle number to test")
90
confidence := fs.Float64("confidence", 0.90, "minimum confidence threshold")
91
verbose := fs.Bool("v", false, "verbose output")
92
+
93
+
if err := fs.Parse(args[1:]); err != nil {
94
+
return err
95
+
}
96
97
if *bundleNum == 0 {
98
+
return fmt.Errorf("--bundle required")
99
}
100
101
mgr, _, err := getManager("")
102
if err != nil {
103
+
return err
104
}
105
defer mgr.Close()
106
107
ctx := context.Background()
108
bundle, err := mgr.LoadBundle(ctx, *bundleNum)
109
if err != nil {
110
+
return fmt.Errorf("error loading bundle: %w", err)
111
}
112
113
fmt.Printf("Testing detector '%s' on bundle %06d...\n", detectorName, *bundleNum)
114
fmt.Printf("Min confidence: %.2f\n\n", *confidence)
115
116
registry := detector.DefaultRegistry()
117
config := detector.DefaultConfig()
118
config.MinConfidence = *confidence
···
120
runner := detector.NewRunner(registry, config, &defaultLogger{})
121
results, err := runner.RunOnBundle(ctx, detectorName, bundle)
122
if err != nil {
123
+
return fmt.Errorf("detection failed: %w", err)
124
}
125
126
stats := detector.CalculateStats(results, len(bundle.Operations))
127
128
fmt.Printf("Results:\n")
129
fmt.Printf(" Total operations: %d\n", stats.TotalOperations)
130
+
fmt.Printf(" Matches found: %d (%.2f%%)\n\n", stats.MatchedCount, stats.MatchRate*100)
131
132
if len(stats.ByReason) > 0 {
133
fmt.Printf("Breakdown by reason:\n")
···
138
fmt.Printf("\n")
139
}
140
141
if *verbose && len(results) > 0 {
142
fmt.Printf("Sample matches (first 10):\n")
143
displayCount := 10
···
153
fmt.Printf(" Note: %s\n", res.Match.Note)
154
}
155
}
156
}
157
+
158
+
return nil
159
}
160
161
+
func detectorRun(args []string) error {
162
+
if len(args) < 1 {
163
+
return fmt.Errorf("usage: plcbundle detector run <detector1|script.js> [detector2...] [--bundles 1-100]")
164
}
165
166
var detectorNames []string
167
var flagArgs []string
168
+
for i := 0; i < len(args); i++ {
169
+
if strings.HasPrefix(args[i], "-") {
170
+
flagArgs = args[i:]
171
break
172
}
173
+
detectorNames = append(detectorNames, args[i])
174
}
175
176
if len(detectorNames) == 0 {
177
+
return fmt.Errorf("at least one detector name required")
178
}
179
180
fs := flag.NewFlagSet("detector run", flag.ExitOnError)
181
+
bundleRange := fs.String("bundles", "", "bundle range (default: all)")
182
confidence := fs.Float64("confidence", 0.90, "minimum confidence")
183
pprofPort := fs.String("pprof", "", "enable pprof on port (e.g., :6060)")
184
185
+
if err := fs.Parse(flagArgs); err != nil {
186
+
return err
187
+
}
188
+
189
+
// Start pprof if requested
190
if *pprofPort != "" {
191
go func() {
192
fmt.Fprintf(os.Stderr, "pprof server starting on http://localhost%s/debug/pprof/\n", *pprofPort)
193
+
http.ListenAndServe(*pprofPort, nil)
194
}()
195
}
196
197
mgr, _, err := getManager("")
198
if err != nil {
199
+
return err
200
}
201
defer mgr.Close()
202
203
+
// Determine range
204
var start, end int
205
if *bundleRange == "" {
206
index := mgr.GetIndex()
207
bundles := index.GetBundles()
208
if len(bundles) == 0 {
209
+
return fmt.Errorf("no bundles available")
210
}
211
start = bundles[0].BundleNumber
212
end = bundles[len(bundles)-1].BundleNumber
···
214
} else {
215
start, end, err = parseBundleRange(*bundleRange)
216
if err != nil {
217
+
return err
218
}
219
}
220
221
+
// Load detectors
222
setup, err := parseAndLoadDetectors(detectorNames, *confidence)
223
if err != nil {
224
+
return err
225
}
226
defer setup.cleanup()
227
···
231
ctx := context.Background()
232
fmt.Println("bundle,position,cid,size,confidence,labels")
233
234
totalOps, matchCount := 0, 0
235
totalBytes, matchedBytes := int64(0), int64(0)
236
237
for bundleNum := start; bundleNum <= end; bundleNum++ {
238
bundle, err := mgr.LoadBundle(ctx, bundleNum)
239
if err != nil {
···
250
}
251
totalBytes += int64(opSize)
252
253
+
labels, conf := detectOperation(ctx, setup.detectors, op, setup.confidence)
254
255
if len(labels) > 0 {
256
matchCount++
···
262
}
263
264
fmt.Printf("%d,%d,%s,%d,%.2f,%s\n",
265
+
bundleNum, position, cidShort, opSize, conf, strings.Join(labels, ";"))
266
}
267
}
268
}
269
270
fmt.Fprintf(os.Stderr, "\n✓ Detection complete\n")
271
fmt.Fprintf(os.Stderr, " Total operations: %d\n", totalOps)
272
fmt.Fprintf(os.Stderr, " Matches found: %d (%.2f%%)\n", matchCount, float64(matchCount)/float64(totalOps)*100)
273
+
274
+
return nil
275
}
276
277
+
func detectorFilter(args []string) error {
278
+
if len(args) < 1 {
279
+
return fmt.Errorf("usage: plcbundle detector filter <detector1|script.js> [detector2...] [--confidence 0.9]")
280
+
}
281
+
282
+
var detectorNames []string
283
+
var flagArgs []string
284
+
for i := 0; i < len(args); i++ {
285
+
if strings.HasPrefix(args[i], "-") {
286
+
flagArgs = args[i:]
287
+
break
288
+
}
289
+
detectorNames = append(detectorNames, args[i])
290
+
}
291
+
292
+
if len(detectorNames) == 0 {
293
+
return fmt.Errorf("at least one detector name required")
294
}
295
296
+
fs := flag.NewFlagSet("detector filter", flag.ExitOnError)
297
+
confidence := fs.Float64("confidence", 0.90, "minimum confidence")
298
+
299
+
if err := fs.Parse(flagArgs); err != nil {
300
+
return err
301
+
}
302
303
+
setup, err := parseAndLoadDetectors(detectorNames, *confidence)
304
if err != nil {
305
+
return err
306
}
307
+
defer setup.cleanup()
308
309
+
fmt.Fprintf(os.Stderr, "Filtering with %d detector(s), min confidence: %.2f\n\n", len(setup.detectors), *confidence)
310
311
+
ctx := context.Background()
312
+
scanner := bufio.NewScanner(os.Stdin)
313
+
buf := make([]byte, 0, 64*1024)
314
+
scanner.Buffer(buf, 1024*1024)
315
+
316
+
cleanCount, filteredCount, totalCount := 0, 0, 0
317
+
totalBytes, filteredBytes := int64(0), int64(0)
318
319
+
for scanner.Scan() {
320
+
line := scanner.Bytes()
321
+
if len(line) == 0 {
322
+
continue
323
+
}
324
+
325
+
totalCount++
326
+
totalBytes += int64(len(line))
327
+
328
+
var op plcclient.PLCOperation
329
+
if err := json.Unmarshal(line, &op); err != nil {
330
+
continue
331
+
}
332
+
333
+
labels, _ := detectOperation(ctx, setup.detectors, op, setup.confidence)
334
335
+
if len(labels) == 0 {
336
+
cleanCount++
337
+
fmt.Println(string(line))
338
+
} else {
339
+
filteredCount++
340
+
filteredBytes += int64(len(line))
341
}
342
343
+
if totalCount%1000 == 0 {
344
+
fmt.Fprintf(os.Stderr, "Processed: %d | Clean: %d | Filtered: %d\r", totalCount, cleanCount, filteredCount)
345
+
}
346
}
347
348
+
fmt.Fprintf(os.Stderr, "\n\n✓ Filter complete\n")
349
+
fmt.Fprintf(os.Stderr, " Total: %d | Clean: %d (%.2f%%) | Filtered: %d (%.2f%%)\n",
350
+
totalCount, cleanCount, float64(cleanCount)/float64(totalCount)*100,
351
+
filteredCount, float64(filteredCount)/float64(totalCount)*100)
352
+
fmt.Fprintf(os.Stderr, " Size saved: %s (%.2f%%)\n", formatBytes(filteredBytes), float64(filteredBytes)/float64(totalBytes)*100)
353
+
354
+
return nil
355
+
}
356
+
357
+
func detectorInfo(args []string) error {
358
+
if len(args) < 1 {
359
+
return fmt.Errorf("usage: plcbundle detector info <name>")
360
}
361
362
+
detectorName := args[0]
363
+
364
+
registry := detector.DefaultRegistry()
365
+
d, err := registry.Get(detectorName)
366
if err != nil {
367
+
return err
368
}
369
370
+
fmt.Printf("Detector: %s\n", d.Name())
371
+
fmt.Printf("Version: %s\n", d.Version())
372
+
fmt.Printf("Description: %s\n\n", d.Description())
373
+
374
+
fmt.Printf("Usage examples:\n")
375
+
fmt.Printf(" # Test on single bundle\n")
376
+
fmt.Printf(" plcbundle detector test %s --bundle 42\n\n", d.Name())
377
+
fmt.Printf(" # Run on range and save\n")
378
+
fmt.Printf(" plcbundle detector run %s --bundles 1-100 > results.csv\n\n", d.Name())
379
380
+
return nil
381
}
382
383
+
// Helper functions
384
+
385
type detectorSetup struct {
386
detectors []detector.Detector
387
scriptDetectors []interface{ Close() error }
···
394
}
395
}
396
397
func parseAndLoadDetectors(detectorNames []string, confidence float64) (*detectorSetup, error) {
398
registry := detector.DefaultRegistry()
399
···
410
411
for _, name := range detectorNames {
412
if strings.HasSuffix(name, ".js") {
413
+
sd, err := detector.NewScriptDetector(name)
414
if err != nil {
415
setup.cleanup()
416
return nil, fmt.Errorf("error loading script %s: %w", name, err)
···
432
return setup, nil
433
}
434
435
func detectOperation(ctx context.Context, detectors []detector.Detector, op plcclient.PLCOperation, minConfidence float64) ([]string, float64) {
436
opData, err := op.GetOperationData()
437
if err != nil {
438
return nil, 0
439
}
440
+
op.ParsedOperation = opData
441
442
var matchedLabels []string
443
var maxConfidence float64
···
448
continue
449
}
450
451
var labels []string
452
if labelList, ok := match.Metadata["labels"].([]string); ok {
453
labels = labelList
···
471
472
return matchedLabels, maxConfidence
473
}
474
+
475
+
type defaultLogger struct{}
476
+
477
+
func (d *defaultLogger) Printf(format string, v ...interface{}) {
478
+
fmt.Fprintf(os.Stderr, format+"\n", v...)
479
+
}
-609
cmd/plcbundle/did_index.go
-609
cmd/plcbundle/did_index.go
···
1
-
package main
2
-
3
-
import (
4
-
"context"
5
-
"flag"
6
-
"fmt"
7
-
"os"
8
-
"strings"
9
-
"time"
10
-
11
-
"github.com/goccy/go-json"
12
-
"tangled.org/atscan.net/plcbundle/internal/didindex"
13
-
"tangled.org/atscan.net/plcbundle/plcclient"
14
-
)
15
-
16
-
func cmdDIDIndex() {
17
-
if len(os.Args) < 3 {
18
-
printDIDIndexUsage()
19
-
os.Exit(1)
20
-
}
21
-
22
-
subcommand := os.Args[2]
23
-
24
-
switch subcommand {
25
-
case "build":
26
-
cmdDIDIndexBuild()
27
-
case "stats":
28
-
cmdDIDIndexStats()
29
-
case "lookup":
30
-
cmdDIDIndexLookup()
31
-
case "resolve":
32
-
cmdDIDIndexResolve()
33
-
default:
34
-
fmt.Fprintf(os.Stderr, "Unknown index subcommand: %s\n", subcommand)
35
-
printDIDIndexUsage()
36
-
os.Exit(1)
37
-
}
38
-
}
39
-
40
-
func printDIDIndexUsage() {
41
-
fmt.Printf(`Usage: plcbundle index <command> [options]
42
-
43
-
Commands:
44
-
build Build DID index from bundles
45
-
stats Show index statistics
46
-
lookup Lookup a specific DID
47
-
resolve Resolve DID to current document
48
-
49
-
Examples:
50
-
plcbundle index build
51
-
plcbundle index stats
52
-
plcbundle index lookup -v did:plc:524tuhdhh3m7li5gycdn6boe
53
-
plcbundle index resolve did:plc:524tuhdhh3m7li5gycdn6boe
54
-
`)
55
-
}
56
-
57
-
func cmdDIDIndexBuild() {
58
-
fs := flag.NewFlagSet("index build", flag.ExitOnError)
59
-
force := fs.Bool("force", false, "rebuild even if index exists")
60
-
fs.Parse(os.Args[3:])
61
-
62
-
mgr, dir, err := getManager("")
63
-
if err != nil {
64
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
65
-
os.Exit(1)
66
-
}
67
-
defer mgr.Close()
68
-
69
-
// Check if index exists
70
-
stats := mgr.GetDIDIndexStats()
71
-
if stats["exists"].(bool) && !*force {
72
-
fmt.Printf("DID index already exists (use --force to rebuild)\n")
73
-
fmt.Printf("Directory: %s\n", dir)
74
-
fmt.Printf("Total DIDs: %d\n", stats["total_dids"])
75
-
return
76
-
}
77
-
78
-
fmt.Printf("Building DID index in: %s\n", dir)
79
-
80
-
index := mgr.GetIndex()
81
-
bundleCount := index.Count()
82
-
83
-
if bundleCount == 0 {
84
-
fmt.Printf("No bundles to index\n")
85
-
return
86
-
}
87
-
88
-
fmt.Printf("Indexing %d bundles...\n\n", bundleCount)
89
-
90
-
progress := NewProgressBar(bundleCount)
91
-
92
-
start := time.Now()
93
-
ctx := context.Background()
94
-
95
-
err = mgr.BuildDIDIndex(ctx, func(current, total int) {
96
-
progress.Set(current)
97
-
})
98
-
99
-
progress.Finish()
100
-
101
-
if err != nil {
102
-
fmt.Fprintf(os.Stderr, "\nError building index: %v\n", err)
103
-
os.Exit(1)
104
-
}
105
-
106
-
elapsed := time.Since(start)
107
-
108
-
stats = mgr.GetDIDIndexStats()
109
-
110
-
fmt.Printf("\n✓ DID index built in %s\n", elapsed.Round(time.Millisecond))
111
-
fmt.Printf(" Total DIDs: %s\n", formatNumber(int(stats["total_dids"].(int64))))
112
-
fmt.Printf(" Shards: %d\n", stats["shard_count"])
113
-
fmt.Printf(" Location: %s/.plcbundle/\n", dir)
114
-
}
115
-
116
-
func cmdDIDIndexStats() {
117
-
mgr, dir, err := getManager("")
118
-
if err != nil {
119
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
120
-
os.Exit(1)
121
-
}
122
-
defer mgr.Close()
123
-
124
-
stats := mgr.GetDIDIndexStats()
125
-
126
-
if !stats["exists"].(bool) {
127
-
fmt.Printf("DID index does not exist\n")
128
-
fmt.Printf("Run: plcbundle index build\n")
129
-
return
130
-
}
131
-
132
-
indexedDIDs := stats["indexed_dids"].(int64)
133
-
mempoolDIDs := stats["mempool_dids"].(int64)
134
-
totalDIDs := stats["total_dids"].(int64)
135
-
136
-
fmt.Printf("\nDID Index Statistics\n")
137
-
fmt.Printf("════════════════════\n\n")
138
-
fmt.Printf(" Location: %s/.plcbundle/\n", dir)
139
-
140
-
if mempoolDIDs > 0 {
141
-
fmt.Printf(" Indexed DIDs: %s (in bundles)\n", formatNumber(int(indexedDIDs)))
142
-
fmt.Printf(" Mempool DIDs: %s (not yet bundled)\n", formatNumber(int(mempoolDIDs)))
143
-
fmt.Printf(" Total DIDs: %s\n", formatNumber(int(totalDIDs)))
144
-
} else {
145
-
fmt.Printf(" Total DIDs: %s\n", formatNumber(int(totalDIDs)))
146
-
}
147
-
148
-
fmt.Printf(" Shard count: %d\n", stats["shard_count"])
149
-
fmt.Printf(" Last bundle: %06d\n", stats["last_bundle"])
150
-
fmt.Printf(" Updated: %s\n", stats["updated_at"].(time.Time).Format("2006-01-02 15:04:05"))
151
-
fmt.Printf("\n")
152
-
fmt.Printf(" Cached shards: %d / %d\n", stats["cached_shards"], stats["cache_limit"])
153
-
154
-
if cachedList, ok := stats["cache_order"].([]int); ok && len(cachedList) > 0 {
155
-
fmt.Printf(" Hot shards: ")
156
-
for i, shard := range cachedList {
157
-
if i > 0 {
158
-
fmt.Printf(", ")
159
-
}
160
-
if i >= 10 {
161
-
fmt.Printf("... (+%d more)", len(cachedList)-10)
162
-
break
163
-
}
164
-
fmt.Printf("%02x", shard)
165
-
}
166
-
fmt.Printf("\n")
167
-
}
168
-
169
-
fmt.Printf("\n")
170
-
}
171
-
172
-
func cmdDIDIndexLookup() {
173
-
if len(os.Args) < 4 {
174
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle index lookup <did> [-v]\n")
175
-
os.Exit(1)
176
-
}
177
-
178
-
fs := flag.NewFlagSet("index lookup", flag.ExitOnError)
179
-
verbose := fs.Bool("v", false, "verbose debug output")
180
-
showJSON := fs.Bool("json", false, "output as JSON")
181
-
fs.Parse(os.Args[3:])
182
-
183
-
if fs.NArg() < 1 {
184
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle index lookup <did> [-v] [--json]\n")
185
-
os.Exit(1)
186
-
}
187
-
188
-
did := fs.Arg(0)
189
-
190
-
mgr, _, err := getManager("")
191
-
if err != nil {
192
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
193
-
os.Exit(1)
194
-
}
195
-
defer mgr.Close()
196
-
197
-
stats := mgr.GetDIDIndexStats()
198
-
if !stats["exists"].(bool) {
199
-
fmt.Fprintf(os.Stderr, "⚠️ DID index does not exist. Run: plcbundle index build\n")
200
-
fmt.Fprintf(os.Stderr, " Falling back to full scan (this will be slow)...\n\n")
201
-
}
202
-
203
-
if !*showJSON {
204
-
fmt.Printf("Looking up: %s\n", did)
205
-
if *verbose {
206
-
fmt.Printf("Verbose mode: enabled\n")
207
-
}
208
-
fmt.Printf("\n")
209
-
}
210
-
211
-
// === TIMING START ===
212
-
totalStart := time.Now()
213
-
ctx := context.Background()
214
-
215
-
// === STEP 1: Index/Scan Lookup ===
216
-
lookupStart := time.Now()
217
-
opsWithLoc, err := mgr.GetDIDOperationsWithLocations(ctx, did, *verbose)
218
-
if err != nil {
219
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
220
-
os.Exit(1)
221
-
}
222
-
lookupElapsed := time.Since(lookupStart)
223
-
224
-
// === STEP 2: Mempool Lookup ===
225
-
mempoolStart := time.Now()
226
-
mempoolOps, err := mgr.GetDIDOperationsFromMempool(did)
227
-
if err != nil {
228
-
fmt.Fprintf(os.Stderr, "Error checking mempool: %v\n", err)
229
-
os.Exit(1)
230
-
}
231
-
mempoolElapsed := time.Since(mempoolStart)
232
-
233
-
totalElapsed := time.Since(totalStart)
234
-
235
-
// === NOT FOUND ===
236
-
if len(opsWithLoc) == 0 && len(mempoolOps) == 0 {
237
-
if *showJSON {
238
-
fmt.Println("{\"found\": false, \"operations\": []}")
239
-
} else {
240
-
fmt.Printf("DID not found (searched in %s)\n", totalElapsed)
241
-
}
242
-
return
243
-
}
244
-
245
-
// === JSON OUTPUT MODE ===
246
-
if *showJSON {
247
-
output := map[string]interface{}{
248
-
"found": true,
249
-
"did": did,
250
-
"timing": map[string]interface{}{
251
-
"total_ms": totalElapsed.Milliseconds(),
252
-
"lookup_ms": lookupElapsed.Milliseconds(),
253
-
"mempool_ms": mempoolElapsed.Milliseconds(),
254
-
},
255
-
"bundled": make([]map[string]interface{}, 0),
256
-
"mempool": make([]map[string]interface{}, 0),
257
-
}
258
-
259
-
for _, owl := range opsWithLoc {
260
-
output["bundled"] = append(output["bundled"].([]map[string]interface{}), map[string]interface{}{
261
-
"bundle": owl.Bundle,
262
-
"position": owl.Position,
263
-
"cid": owl.Operation.CID,
264
-
"nullified": owl.Operation.IsNullified(),
265
-
"created_at": owl.Operation.CreatedAt.Format(time.RFC3339Nano),
266
-
})
267
-
}
268
-
269
-
for _, op := range mempoolOps {
270
-
output["mempool"] = append(output["mempool"].([]map[string]interface{}), map[string]interface{}{
271
-
"cid": op.CID,
272
-
"nullified": op.IsNullified(),
273
-
"created_at": op.CreatedAt.Format(time.RFC3339Nano),
274
-
})
275
-
}
276
-
277
-
data, _ := json.MarshalIndent(output, "", " ")
278
-
fmt.Println(string(data))
279
-
return
280
-
}
281
-
282
-
// === CALCULATE STATISTICS ===
283
-
nullifiedCount := 0
284
-
for _, owl := range opsWithLoc {
285
-
if owl.Operation.IsNullified() {
286
-
nullifiedCount++
287
-
}
288
-
}
289
-
290
-
totalOps := len(opsWithLoc) + len(mempoolOps)
291
-
activeOps := len(opsWithLoc) - nullifiedCount + len(mempoolOps)
292
-
293
-
// === DISPLAY SUMMARY ===
294
-
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
295
-
fmt.Printf(" DID Lookup Results\n")
296
-
fmt.Printf("═══════════════════════════════════════════════════════════════\n\n")
297
-
298
-
fmt.Printf("DID: %s\n\n", did)
299
-
300
-
fmt.Printf("Summary\n")
301
-
fmt.Printf("───────\n")
302
-
fmt.Printf(" Total operations: %d\n", totalOps)
303
-
fmt.Printf(" Active operations: %d\n", activeOps)
304
-
if nullifiedCount > 0 {
305
-
fmt.Printf(" Nullified: %d\n", nullifiedCount)
306
-
}
307
-
if len(opsWithLoc) > 0 {
308
-
fmt.Printf(" Bundled: %d\n", len(opsWithLoc))
309
-
}
310
-
if len(mempoolOps) > 0 {
311
-
fmt.Printf(" Mempool: %d\n", len(mempoolOps))
312
-
}
313
-
fmt.Printf("\n")
314
-
315
-
// === TIMING BREAKDOWN ===
316
-
fmt.Printf("Performance\n")
317
-
fmt.Printf("───────────\n")
318
-
fmt.Printf(" Index lookup: %s\n", lookupElapsed)
319
-
fmt.Printf(" Mempool check: %s\n", mempoolElapsed)
320
-
fmt.Printf(" Total time: %s\n", totalElapsed)
321
-
322
-
if len(opsWithLoc) > 0 {
323
-
avgPerOp := lookupElapsed / time.Duration(len(opsWithLoc))
324
-
fmt.Printf(" Avg per operation: %s\n", avgPerOp)
325
-
}
326
-
fmt.Printf("\n")
327
-
328
-
// === BUNDLED OPERATIONS ===
329
-
if len(opsWithLoc) > 0 {
330
-
fmt.Printf("Bundled Operations (%d total)\n", len(opsWithLoc))
331
-
fmt.Printf("══════════════════════════════════════════════════════════════\n\n")
332
-
333
-
for i, owl := range opsWithLoc {
334
-
op := owl.Operation
335
-
status := "✓ Active"
336
-
statusSymbol := "✓"
337
-
if op.IsNullified() {
338
-
status = "✗ Nullified"
339
-
statusSymbol = "✗"
340
-
}
341
-
342
-
fmt.Printf("%s Operation %d [Bundle %06d, Position %04d]\n",
343
-
statusSymbol, i+1, owl.Bundle, owl.Position)
344
-
fmt.Printf(" CID: %s\n", op.CID)
345
-
fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000 MST"))
346
-
fmt.Printf(" Status: %s\n", status)
347
-
348
-
if op.IsNullified() {
349
-
if nullCID := op.GetNullifyingCID(); nullCID != "" {
350
-
fmt.Printf(" Nullified: %s\n", nullCID)
351
-
}
352
-
}
353
-
354
-
// Show operation type if verbose
355
-
if *verbose {
356
-
if opData, err := op.GetOperationData(); err == nil && opData != nil {
357
-
if opType, ok := opData["type"].(string); ok {
358
-
fmt.Printf(" Type: %s\n", opType)
359
-
}
360
-
361
-
// Show handle if present
362
-
if handle, ok := opData["handle"].(string); ok {
363
-
fmt.Printf(" Handle: %s\n", handle)
364
-
} else if aka, ok := opData["alsoKnownAs"].([]interface{}); ok && len(aka) > 0 {
365
-
if akaStr, ok := aka[0].(string); ok {
366
-
handle := strings.TrimPrefix(akaStr, "at://")
367
-
fmt.Printf(" Handle: %s\n", handle)
368
-
}
369
-
}
370
-
371
-
// Show service if present
372
-
if services, ok := opData["services"].(map[string]interface{}); ok {
373
-
if pds, ok := services["atproto_pds"].(map[string]interface{}); ok {
374
-
if endpoint, ok := pds["endpoint"].(string); ok {
375
-
fmt.Printf(" PDS: %s\n", endpoint)
376
-
}
377
-
}
378
-
}
379
-
}
380
-
}
381
-
382
-
fmt.Printf("\n")
383
-
}
384
-
}
385
-
386
-
// === MEMPOOL OPERATIONS ===
387
-
if len(mempoolOps) > 0 {
388
-
fmt.Printf("Mempool Operations (%d total, not yet bundled)\n", len(mempoolOps))
389
-
fmt.Printf("══════════════════════════════════════════════════════════════\n\n")
390
-
391
-
for i, op := range mempoolOps {
392
-
status := "✓ Active"
393
-
statusSymbol := "✓"
394
-
if op.IsNullified() {
395
-
status = "✗ Nullified"
396
-
statusSymbol = "✗"
397
-
}
398
-
399
-
fmt.Printf("%s Operation %d [Mempool]\n", statusSymbol, i+1)
400
-
fmt.Printf(" CID: %s\n", op.CID)
401
-
fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000 MST"))
402
-
fmt.Printf(" Status: %s\n", status)
403
-
404
-
if op.IsNullified() {
405
-
if nullCID := op.GetNullifyingCID(); nullCID != "" {
406
-
fmt.Printf(" Nullified: %s\n", nullCID)
407
-
}
408
-
}
409
-
410
-
// Show operation type if verbose
411
-
if *verbose {
412
-
if opData, err := op.GetOperationData(); err == nil && opData != nil {
413
-
if opType, ok := opData["type"].(string); ok {
414
-
fmt.Printf(" Type: %s\n", opType)
415
-
}
416
-
417
-
// Show handle
418
-
if handle, ok := opData["handle"].(string); ok {
419
-
fmt.Printf(" Handle: %s\n", handle)
420
-
} else if aka, ok := opData["alsoKnownAs"].([]interface{}); ok && len(aka) > 0 {
421
-
if akaStr, ok := aka[0].(string); ok {
422
-
handle := strings.TrimPrefix(akaStr, "at://")
423
-
fmt.Printf(" Handle: %s\n", handle)
424
-
}
425
-
}
426
-
}
427
-
}
428
-
429
-
fmt.Printf("\n")
430
-
}
431
-
}
432
-
433
-
// === TIMELINE (if multiple operations) ===
434
-
if totalOps > 1 && !*verbose {
435
-
fmt.Printf("Timeline\n")
436
-
fmt.Printf("────────\n")
437
-
438
-
allTimes := make([]time.Time, 0, totalOps)
439
-
for _, owl := range opsWithLoc {
440
-
allTimes = append(allTimes, owl.Operation.CreatedAt)
441
-
}
442
-
for _, op := range mempoolOps {
443
-
allTimes = append(allTimes, op.CreatedAt)
444
-
}
445
-
446
-
if len(allTimes) > 0 {
447
-
firstTime := allTimes[0]
448
-
lastTime := allTimes[len(allTimes)-1]
449
-
timespan := lastTime.Sub(firstTime)
450
-
451
-
fmt.Printf(" First operation: %s\n", firstTime.Format("2006-01-02 15:04:05"))
452
-
fmt.Printf(" Latest operation: %s\n", lastTime.Format("2006-01-02 15:04:05"))
453
-
fmt.Printf(" Timespan: %s\n", formatDuration(timespan))
454
-
fmt.Printf(" Activity age: %s ago\n", formatDuration(time.Since(lastTime)))
455
-
}
456
-
fmt.Printf("\n")
457
-
}
458
-
459
-
// === FINAL SUMMARY ===
460
-
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
461
-
fmt.Printf("✓ Lookup complete in %s\n", totalElapsed)
462
-
if stats["exists"].(bool) {
463
-
fmt.Printf(" Method: DID index (fast)\n")
464
-
} else {
465
-
fmt.Printf(" Method: Full scan (slow)\n")
466
-
}
467
-
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
468
-
}
469
-
470
-
func cmdDIDIndexResolve() {
471
-
if len(os.Args) < 4 {
472
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle index resolve <did> [-v]\n")
473
-
os.Exit(1)
474
-
}
475
-
476
-
fs := flag.NewFlagSet("index resolve", flag.ExitOnError)
477
-
//verbose := fs.Bool("v", false, "verbose debug output")
478
-
fs.Parse(os.Args[3:])
479
-
480
-
if fs.NArg() < 1 {
481
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle index resolve <did> [-v]\n")
482
-
os.Exit(1)
483
-
}
484
-
485
-
did := fs.Arg(0)
486
-
487
-
mgr, _, err := getManager("")
488
-
if err != nil {
489
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
490
-
os.Exit(1)
491
-
}
492
-
defer mgr.Close()
493
-
494
-
ctx := context.Background()
495
-
fmt.Fprintf(os.Stderr, "Resolving: %s\n", did)
496
-
497
-
start := time.Now()
498
-
499
-
// ✨ STEP 0: Check mempool first (most recent data)
500
-
mempoolStart := time.Now()
501
-
var latestOp *plcclient.PLCOperation
502
-
foundInMempool := false
503
-
504
-
if mgr.GetMempool() != nil {
505
-
mempoolOps, err := mgr.GetMempoolOperations()
506
-
if err == nil && len(mempoolOps) > 0 {
507
-
// Search backward for this DID
508
-
for i := len(mempoolOps) - 1; i >= 0; i-- {
509
-
if mempoolOps[i].DID == did && !mempoolOps[i].IsNullified() {
510
-
latestOp = &mempoolOps[i]
511
-
foundInMempool = true
512
-
break
513
-
}
514
-
}
515
-
}
516
-
}
517
-
mempoolTime := time.Since(mempoolStart)
518
-
519
-
if foundInMempool {
520
-
fmt.Fprintf(os.Stderr, "Mempool check: %s (✓ found in mempool)\n", mempoolTime)
521
-
522
-
// Build document from mempool operation
523
-
ops := []plcclient.PLCOperation{*latestOp}
524
-
doc, err := plcclient.ResolveDIDDocument(did, ops)
525
-
if err != nil {
526
-
fmt.Fprintf(os.Stderr, "Build document failed: %v\n", err)
527
-
os.Exit(1)
528
-
}
529
-
530
-
totalTime := time.Since(start)
531
-
fmt.Fprintf(os.Stderr, "Total: %s (resolved from mempool)\n\n", totalTime)
532
-
533
-
// Output to stdout
534
-
data, _ := json.MarshalIndent(doc, "", " ")
535
-
fmt.Println(string(data))
536
-
return
537
-
}
538
-
539
-
fmt.Fprintf(os.Stderr, "Mempool check: %s (not found)\n", mempoolTime)
540
-
541
-
// Not in mempool - check index
542
-
stats := mgr.GetDIDIndexStats()
543
-
if !stats["exists"].(bool) {
544
-
fmt.Fprintf(os.Stderr, "⚠️ DID index does not exist. Run: plcbundle index build\n\n")
545
-
os.Exit(1)
546
-
}
547
-
548
-
// STEP 1: Index lookup timing
549
-
indexStart := time.Now()
550
-
locations, err := mgr.GetDIDIndex().GetDIDLocations(did)
551
-
if err != nil {
552
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
553
-
os.Exit(1)
554
-
}
555
-
indexTime := time.Since(indexStart)
556
-
557
-
if len(locations) == 0 {
558
-
fmt.Fprintf(os.Stderr, "DID not found in index or mempool\n")
559
-
os.Exit(1)
560
-
}
561
-
562
-
// Find latest non-nullified location
563
-
var latestLoc *didindex.OpLocation
564
-
for i := range locations {
565
-
if locations[i].Nullified {
566
-
continue
567
-
}
568
-
if latestLoc == nil ||
569
-
locations[i].Bundle > latestLoc.Bundle ||
570
-
(locations[i].Bundle == latestLoc.Bundle && locations[i].Position > latestLoc.Position) {
571
-
latestLoc = &locations[i]
572
-
}
573
-
}
574
-
575
-
if latestLoc == nil {
576
-
fmt.Fprintf(os.Stderr, "No valid operations (all nullified)\n")
577
-
os.Exit(1)
578
-
}
579
-
580
-
fmt.Fprintf(os.Stderr, "Index lookup: %s (shard access)\n", indexTime)
581
-
582
-
// STEP 2: Operation loading timing (single op, not full bundle!)
583
-
opStart := time.Now()
584
-
op, err := mgr.LoadOperation(ctx, int(latestLoc.Bundle), int(latestLoc.Position))
585
-
if err != nil {
586
-
fmt.Fprintf(os.Stderr, "Error loading operation: %v\n", err)
587
-
os.Exit(1)
588
-
}
589
-
opTime := time.Since(opStart)
590
-
591
-
fmt.Fprintf(os.Stderr, "Operation load: %s (bundle %d, pos %d)\n",
592
-
opTime, latestLoc.Bundle, latestLoc.Position)
593
-
594
-
// STEP 3: Build DID document
595
-
ops := []plcclient.PLCOperation{*op}
596
-
doc, err := plcclient.ResolveDIDDocument(did, ops)
597
-
if err != nil {
598
-
fmt.Fprintf(os.Stderr, "Build document failed: %v\n", err)
599
-
os.Exit(1)
600
-
}
601
-
602
-
totalTime := time.Since(start)
603
-
fmt.Fprintf(os.Stderr, "Total: %s\n\n", totalTime)
604
-
605
-
// Output to stdout
606
-
data, _ := json.MarshalIndent(doc, "", " ")
607
-
fmt.Println(string(data))
608
-
609
-
}
···
-52
cmd/plcbundle/get_op.go
-52
cmd/plcbundle/get_op.go
···
1
-
package main
2
-
3
-
import (
4
-
"context"
5
-
"fmt"
6
-
"os"
7
-
"strconv"
8
-
9
-
"github.com/goccy/go-json"
10
-
)
11
-
12
-
func cmdGetOp() {
13
-
if len(os.Args) < 4 {
14
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle get-op <bundle> <position>\n")
15
-
fmt.Fprintf(os.Stderr, "Example: plcbundle get-op 42 1337\n")
16
-
os.Exit(1)
17
-
}
18
-
19
-
bundleNum, err := strconv.Atoi(os.Args[2])
20
-
if err != nil {
21
-
fmt.Fprintf(os.Stderr, "Error: invalid bundle number\n")
22
-
os.Exit(1)
23
-
}
24
-
25
-
position, err := strconv.Atoi(os.Args[3])
26
-
if err != nil {
27
-
fmt.Fprintf(os.Stderr, "Error: invalid position\n")
28
-
os.Exit(1)
29
-
}
30
-
31
-
mgr, _, err := getManager("")
32
-
if err != nil {
33
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
34
-
os.Exit(1)
35
-
}
36
-
defer mgr.Close()
37
-
38
-
ctx := context.Background()
39
-
op, err := mgr.LoadOperation(ctx, bundleNum, position)
40
-
if err != nil {
41
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
42
-
os.Exit(1)
43
-
}
44
-
45
-
// Output JSON
46
-
if len(op.RawJSON) > 0 {
47
-
fmt.Println(string(op.RawJSON))
48
-
} else {
49
-
data, _ := json.Marshal(op)
50
-
fmt.Println(string(data))
51
-
}
52
-
}
···
+146
-104
cmd/plcbundle/info.go
cmd/plcbundle/commands/info.go
+146
-104
cmd/plcbundle/info.go
cmd/plcbundle/commands/info.go
···
1
-
package main
2
3
import (
4
"context"
5
"fmt"
6
-
"os"
7
"path/filepath"
8
"sort"
9
"strings"
···
14
"tangled.org/atscan.net/plcbundle/internal/types"
15
)
16
17
-
func showGeneralInfo(mgr *bundle.Manager, dir string, verbose bool, showBundles bool, verify bool, showTimeline bool) {
18
index := mgr.GetIndex()
19
info := mgr.GetInfo()
20
stats := index.GetStats()
21
bundleCount := stats["bundle_count"].(int)
22
23
-
fmt.Printf("\n")
24
-
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
25
fmt.Printf(" PLC Bundle Repository Overview\n")
26
-
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
27
-
fmt.Printf("\n")
28
29
-
// Location
30
fmt.Printf("📁 Location\n")
31
fmt.Printf(" Directory: %s\n", dir)
32
-
fmt.Printf(" Index: %s\n", filepath.Base(info["index_path"].(string)))
33
-
fmt.Printf("\n")
34
fmt.Printf("🌐 Origin\n")
35
-
fmt.Printf(" Source: %s\n", index.Origin)
36
-
fmt.Printf("\n")
37
38
if bundleCount == 0 {
39
-
fmt.Printf("⚠️ No bundles found\n")
40
-
fmt.Printf("\n")
41
fmt.Printf("Get started:\n")
42
fmt.Printf(" plcbundle fetch # Fetch bundles from PLC\n")
43
-
fmt.Printf(" plcbundle rebuild # Rebuild index from existing files\n")
44
-
fmt.Printf("\n")
45
-
return
46
}
47
48
firstBundle := stats["first_bundle"].(int)
49
lastBundle := stats["last_bundle"].(int)
50
totalCompressedSize := stats["total_size"].(int64)
51
startTime := stats["start_time"].(time.Time)
52
endTime := stats["end_time"].(time.Time)
53
updatedAt := stats["updated_at"].(time.Time)
54
55
-
// Calculate total uncompressed size
56
-
bundles := index.GetBundles()
57
-
var totalUncompressedSize int64
58
-
for _, meta := range bundles {
59
-
totalUncompressedSize += meta.UncompressedSize
60
-
}
61
-
62
// Summary
63
fmt.Printf("📊 Summary\n")
64
fmt.Printf(" Bundles: %s\n", formatNumber(bundleCount))
···
69
ratio := float64(totalUncompressedSize) / float64(totalCompressedSize)
70
fmt.Printf(" Ratio: %.2fx compression\n", ratio)
71
}
72
-
fmt.Printf(" Avg/Bundle: %s\n", formatBytes(totalCompressedSize/int64(bundleCount)))
73
-
fmt.Printf("\n")
74
75
// Timeline
76
duration := endTime.Sub(startTime)
···
79
fmt.Printf(" Last Op: %s\n", endTime.Format("2006-01-02 15:04:05 MST"))
80
fmt.Printf(" Timespan: %s\n", formatDuration(duration))
81
fmt.Printf(" Last Updated: %s\n", updatedAt.Format("2006-01-02 15:04:05 MST"))
82
-
fmt.Printf(" Age: %s ago\n", formatDuration(time.Since(updatedAt)))
83
-
fmt.Printf("\n")
84
85
-
// Operations count (exact calculation)
86
mempoolStats := mgr.GetMempoolStats()
87
mempoolCount := mempoolStats["count"].(int)
88
bundleOpsCount := bundleCount * types.BUNDLE_SIZE
···
100
}
101
fmt.Printf("\n")
102
103
-
// Hashes (full, not trimmed)
104
firstMeta, err := index.GetBundle(firstBundle)
105
if err == nil {
106
fmt.Printf("🔐 Chain Hashes\n")
···
154
fmt.Printf(" Operations: %s / %s\n", formatNumber(mempoolCount), formatNumber(types.BUNDLE_SIZE))
155
fmt.Printf(" Progress: %.1f%%\n", progress)
156
157
-
// Progress bar
158
barWidth := 40
159
filled := int(float64(barWidth) * float64(mempoolCount) / float64(types.BUNDLE_SIZE))
160
if filled > barWidth {
···
185
fmt.Printf(" ✓ Chain is valid\n")
186
fmt.Printf(" ✓ All %d bundles verified\n", len(result.VerifiedBundles))
187
188
-
// Show head hash (full)
189
lastMeta, _ := index.GetBundle(lastBundle)
190
if lastMeta != nil {
191
fmt.Printf(" Head: %s\n", lastMeta.Hash)
···
199
fmt.Printf("\n")
200
}
201
202
-
// Timeline visualization
203
if showTimeline {
204
fmt.Printf("📈 Timeline Visualization\n")
205
visualizeTimeline(index, verbose)
···
209
// Bundle list
210
if showBundles {
211
bundles := index.GetBundles()
212
-
fmt.Printf("📚 Bundle List (%d total)\n", len(bundles))
213
-
fmt.Printf("\n")
214
fmt.Printf(" Number | Start Time | End Time | Ops | DIDs | Size\n")
215
fmt.Printf(" ---------|---------------------|---------------------|--------|--------|--------\n")
216
···
227
} else if bundleCount > 0 {
228
fmt.Printf("💡 Tip: Use --bundles to see detailed bundle list\n")
229
fmt.Printf(" Use --timeline to see timeline visualization\n")
230
-
fmt.Printf(" Use --verify to verify chain integrity\n")
231
-
fmt.Printf("\n")
232
}
233
234
-
// File system stats (verbose)
235
-
if verbose {
236
-
fmt.Printf("💾 File System\n")
237
238
-
// Calculate average compression ratio
239
-
if totalCompressedSize > 0 && totalUncompressedSize > 0 {
240
-
avgRatio := float64(totalUncompressedSize) / float64(totalCompressedSize)
241
-
savings := (1 - float64(totalCompressedSize)/float64(totalUncompressedSize)) * 100
242
-
fmt.Printf(" Compression: %.2fx average ratio\n", avgRatio)
243
-
fmt.Printf(" Space Saved: %.1f%% (%s)\n", savings, formatBytes(totalUncompressedSize-totalCompressedSize))
244
-
}
245
246
-
// Index size
247
-
indexPath := info["index_path"].(string)
248
-
if indexInfo, err := os.Stat(indexPath); err == nil {
249
-
fmt.Printf(" Index Size: %s\n", formatBytes(indexInfo.Size()))
250
}
251
252
-
fmt.Printf("\n")
253
}
254
}
255
256
func visualizeTimeline(index *bundleindex.Index, verbose bool) {
···
259
return
260
}
261
262
-
// Group bundles by date
263
type dateGroup struct {
264
date string
265
count int
···
283
}
284
}
285
286
-
// Sort dates
287
var dates []string
288
for date := range dateMap {
289
dates = append(dates, date)
290
}
291
sort.Strings(dates)
292
293
-
// Find max count for scaling
294
maxCount := 0
295
for _, group := range dateMap {
296
if group.count > maxCount {
···
298
}
299
}
300
301
-
// Display
302
fmt.Printf("\n")
303
barWidth := 40
304
for _, date := range dates {
···
316
fmt.Printf("\n")
317
}
318
}
319
-
320
-
// Helper formatting functions
321
-
322
-
func formatNumber(n int) string {
323
-
s := fmt.Sprintf("%d", n)
324
-
// Add thousand separators
325
-
var result []byte
326
-
for i, c := range s {
327
-
if i > 0 && (len(s)-i)%3 == 0 {
328
-
result = append(result, ',')
329
-
}
330
-
result = append(result, byte(c))
331
-
}
332
-
return string(result)
333
-
}
334
-
335
-
func formatBytes(bytes int64) string {
336
-
const unit = 1000
337
-
if bytes < unit {
338
-
return fmt.Sprintf("%d B", bytes)
339
-
}
340
-
div, exp := int64(unit), 0
341
-
for n := bytes / unit; n >= unit; n /= unit {
342
-
div *= unit
343
-
exp++
344
-
}
345
-
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
346
-
}
347
-
348
-
func formatDuration(d time.Duration) string {
349
-
if d < time.Minute {
350
-
return fmt.Sprintf("%.0f seconds", d.Seconds())
351
-
}
352
-
if d < time.Hour {
353
-
return fmt.Sprintf("%.1f minutes", d.Minutes())
354
-
}
355
-
if d < 24*time.Hour {
356
-
return fmt.Sprintf("%.1f hours", d.Hours())
357
-
}
358
-
days := d.Hours() / 24
359
-
if days < 30 {
360
-
return fmt.Sprintf("%.1f days", days)
361
-
}
362
-
if days < 365 {
363
-
return fmt.Sprintf("%.1f months", days/30)
364
-
}
365
-
return fmt.Sprintf("%.1f years", days/365)
366
-
}
···
1
+
package commands
2
3
import (
4
"context"
5
+
"flag"
6
"fmt"
7
"path/filepath"
8
"sort"
9
"strings"
···
14
"tangled.org/atscan.net/plcbundle/internal/types"
15
)
16
17
+
// InfoCommand handles the info subcommand
18
+
func InfoCommand(args []string) error {
19
+
fs := flag.NewFlagSet("info", flag.ExitOnError)
20
+
bundleNum := fs.Int("bundle", 0, "specific bundle info (0 = general info)")
21
+
verbose := fs.Bool("v", false, "verbose output")
22
+
showBundles := fs.Bool("bundles", false, "show bundle list")
23
+
verify := fs.Bool("verify", false, "verify chain integrity")
24
+
showTimeline := fs.Bool("timeline", false, "show timeline visualization")
25
+
26
+
if err := fs.Parse(args); err != nil {
27
+
return err
28
+
}
29
+
30
+
mgr, dir, err := getManager("")
31
+
if err != nil {
32
+
return err
33
+
}
34
+
defer mgr.Close()
35
+
36
+
if *bundleNum > 0 {
37
+
return showBundleInfo(mgr, dir, *bundleNum, *verbose)
38
+
}
39
+
40
+
return showGeneralInfo(mgr, dir, *verbose, *showBundles, *verify, *showTimeline)
41
+
}
42
+
43
+
func showGeneralInfo(mgr *bundle.Manager, dir string, verbose, showBundles, verify, showTimeline bool) error {
44
index := mgr.GetIndex()
45
info := mgr.GetInfo()
46
stats := index.GetStats()
47
bundleCount := stats["bundle_count"].(int)
48
49
+
fmt.Printf("\n═══════════════════════════════════════════════════════════════\n")
50
fmt.Printf(" PLC Bundle Repository Overview\n")
51
+
fmt.Printf("═══════════════════════════════════════════════════════════════\n\n")
52
53
fmt.Printf("📁 Location\n")
54
fmt.Printf(" Directory: %s\n", dir)
55
+
fmt.Printf(" Index: %s\n\n", filepath.Base(info["index_path"].(string)))
56
+
57
fmt.Printf("🌐 Origin\n")
58
+
fmt.Printf(" Source: %s\n\n", index.Origin)
59
60
if bundleCount == 0 {
61
+
fmt.Printf("⚠️ No bundles found\n\n")
62
fmt.Printf("Get started:\n")
63
fmt.Printf(" plcbundle fetch # Fetch bundles from PLC\n")
64
+
fmt.Printf(" plcbundle rebuild # Rebuild index from existing files\n\n")
65
+
return nil
66
}
67
68
firstBundle := stats["first_bundle"].(int)
69
lastBundle := stats["last_bundle"].(int)
70
totalCompressedSize := stats["total_size"].(int64)
71
+
totalUncompressedSize := stats["total_uncompressed_size"].(int64)
72
startTime := stats["start_time"].(time.Time)
73
endTime := stats["end_time"].(time.Time)
74
updatedAt := stats["updated_at"].(time.Time)
75
76
// Summary
77
fmt.Printf("📊 Summary\n")
78
fmt.Printf(" Bundles: %s\n", formatNumber(bundleCount))
···
83
ratio := float64(totalUncompressedSize) / float64(totalCompressedSize)
84
fmt.Printf(" Ratio: %.2fx compression\n", ratio)
85
}
86
+
fmt.Printf(" Avg/Bundle: %s\n\n", formatBytes(totalCompressedSize/int64(bundleCount)))
87
88
// Timeline
89
duration := endTime.Sub(startTime)
···
92
fmt.Printf(" Last Op: %s\n", endTime.Format("2006-01-02 15:04:05 MST"))
93
fmt.Printf(" Timespan: %s\n", formatDuration(duration))
94
fmt.Printf(" Last Updated: %s\n", updatedAt.Format("2006-01-02 15:04:05 MST"))
95
+
fmt.Printf(" Age: %s ago\n\n", formatDuration(time.Since(updatedAt)))
96
97
+
// Operations
98
mempoolStats := mgr.GetMempoolStats()
99
mempoolCount := mempoolStats["count"].(int)
100
bundleOpsCount := bundleCount * types.BUNDLE_SIZE
···
112
}
113
fmt.Printf("\n")
114
115
+
// Hashes
116
firstMeta, err := index.GetBundle(firstBundle)
117
if err == nil {
118
fmt.Printf("🔐 Chain Hashes\n")
···
166
fmt.Printf(" Operations: %s / %s\n", formatNumber(mempoolCount), formatNumber(types.BUNDLE_SIZE))
167
fmt.Printf(" Progress: %.1f%%\n", progress)
168
169
barWidth := 40
170
filled := int(float64(barWidth) * float64(mempoolCount) / float64(types.BUNDLE_SIZE))
171
if filled > barWidth {
···
196
fmt.Printf(" ✓ Chain is valid\n")
197
fmt.Printf(" ✓ All %d bundles verified\n", len(result.VerifiedBundles))
198
199
lastMeta, _ := index.GetBundle(lastBundle)
200
if lastMeta != nil {
201
fmt.Printf(" Head: %s\n", lastMeta.Hash)
···
209
fmt.Printf("\n")
210
}
211
212
+
// Timeline
213
if showTimeline {
214
fmt.Printf("📈 Timeline Visualization\n")
215
visualizeTimeline(index, verbose)
···
219
// Bundle list
220
if showBundles {
221
bundles := index.GetBundles()
222
+
fmt.Printf("📚 Bundle List (%d total)\n\n", len(bundles))
223
fmt.Printf(" Number | Start Time | End Time | Ops | DIDs | Size\n")
224
fmt.Printf(" ---------|---------------------|---------------------|--------|--------|--------\n")
225
···
236
} else if bundleCount > 0 {
237
fmt.Printf("💡 Tip: Use --bundles to see detailed bundle list\n")
238
fmt.Printf(" Use --timeline to see timeline visualization\n")
239
+
fmt.Printf(" Use --verify to verify chain integrity\n\n")
240
}
241
242
+
return nil
243
+
}
244
245
+
func showBundleInfo(mgr *bundle.Manager, dir string, bundleNum int, verbose bool) error {
246
+
ctx := context.Background()
247
+
b, err := mgr.LoadBundle(ctx, bundleNum)
248
+
if err != nil {
249
+
return err
250
+
}
251
252
+
fmt.Printf("\n═══════════════════════════════════════════════════════════════\n")
253
+
fmt.Printf(" Bundle %06d\n", b.BundleNumber)
254
+
fmt.Printf("═══════════════════════════════════════════════════════════════\n\n")
255
+
256
+
fmt.Printf("📁 Location\n")
257
+
fmt.Printf(" Directory: %s\n", dir)
258
+
fmt.Printf(" File: %06d.jsonl.zst\n\n", b.BundleNumber)
259
+
260
+
duration := b.EndTime.Sub(b.StartTime)
261
+
fmt.Printf("📅 Time Range\n")
262
+
fmt.Printf(" Start: %s\n", b.StartTime.Format("2006-01-02 15:04:05.000 MST"))
263
+
fmt.Printf(" End: %s\n", b.EndTime.Format("2006-01-02 15:04:05.000 MST"))
264
+
fmt.Printf(" Duration: %s\n", formatDuration(duration))
265
+
fmt.Printf(" Created: %s\n\n", b.CreatedAt.Format("2006-01-02 15:04:05 MST"))
266
+
267
+
fmt.Printf("📊 Content\n")
268
+
fmt.Printf(" Operations: %s\n", formatNumber(len(b.Operations)))
269
+
fmt.Printf(" Unique DIDs: %s\n", formatNumber(b.DIDCount))
270
+
if len(b.Operations) > 0 && b.DIDCount > 0 {
271
+
avgOpsPerDID := float64(len(b.Operations)) / float64(b.DIDCount)
272
+
fmt.Printf(" Avg ops/DID: %.2f\n", avgOpsPerDID)
273
+
}
274
+
fmt.Printf("\n")
275
+
276
+
fmt.Printf("💾 Size\n")
277
+
fmt.Printf(" Compressed: %s\n", formatBytes(b.CompressedSize))
278
+
fmt.Printf(" Uncompressed: %s\n", formatBytes(b.UncompressedSize))
279
+
fmt.Printf(" Ratio: %.2fx\n", b.CompressionRatio())
280
+
fmt.Printf(" Efficiency: %.1f%% savings\n\n", (1-float64(b.CompressedSize)/float64(b.UncompressedSize))*100)
281
+
282
+
fmt.Printf("🔐 Cryptographic Hashes\n")
283
+
fmt.Printf(" Chain Hash:\n %s\n", b.Hash)
284
+
fmt.Printf(" Content Hash:\n %s\n", b.ContentHash)
285
+
fmt.Printf(" Compressed:\n %s\n", b.CompressedHash)
286
+
if b.Parent != "" {
287
+
fmt.Printf(" Parent Chain Hash:\n %s\n", b.Parent)
288
+
}
289
+
fmt.Printf("\n")
290
+
291
+
if verbose && len(b.Operations) > 0 {
292
+
showBundleSamples(b)
293
+
showBundleDIDStats(b)
294
+
}
295
+
296
+
return nil
297
+
}
298
+
299
+
func showBundleSamples(b *bundle.Bundle) {
300
+
fmt.Printf("📝 Sample Operations (first 5)\n")
301
+
showCount := 5
302
+
if len(b.Operations) < showCount {
303
+
showCount = len(b.Operations)
304
+
}
305
+
306
+
for i := 0; i < showCount; i++ {
307
+
op := b.Operations[i]
308
+
fmt.Printf(" %d. %s\n", i+1, op.DID)
309
+
fmt.Printf(" CID: %s\n", op.CID)
310
+
fmt.Printf(" Time: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000"))
311
+
if op.IsNullified() {
312
+
fmt.Printf(" ⚠️ Nullified: %s\n", op.GetNullifyingCID())
313
}
314
+
}
315
+
fmt.Printf("\n")
316
+
}
317
+
318
+
func showBundleDIDStats(b *bundle.Bundle) {
319
+
didOps := make(map[string]int)
320
+
for _, op := range b.Operations {
321
+
didOps[op.DID]++
322
+
}
323
+
324
+
type didCount struct {
325
+
did string
326
+
count int
327
+
}
328
+
329
+
var counts []didCount
330
+
for did, count := range didOps {
331
+
counts = append(counts, didCount{did, count})
332
+
}
333
+
334
+
sort.Slice(counts, func(i, j int) bool {
335
+
return counts[i].count > counts[j].count
336
+
})
337
338
+
fmt.Printf("🏆 Most Active DIDs\n")
339
+
showCount := 5
340
+
if len(counts) < showCount {
341
+
showCount = len(counts)
342
}
343
+
344
+
for i := 0; i < showCount; i++ {
345
+
fmt.Printf(" %d. %s (%d ops)\n", i+1, counts[i].did, counts[i].count)
346
+
}
347
+
fmt.Printf("\n")
348
}
349
350
func visualizeTimeline(index *bundleindex.Index, verbose bool) {
···
353
return
354
}
355
356
type dateGroup struct {
357
date string
358
count int
···
376
}
377
}
378
379
var dates []string
380
for date := range dateMap {
381
dates = append(dates, date)
382
}
383
sort.Strings(dates)
384
385
maxCount := 0
386
for _, group := range dateMap {
387
if group.count > maxCount {
···
389
}
390
}
391
392
fmt.Printf("\n")
393
barWidth := 40
394
for _, date := range dates {
···
406
fmt.Printf("\n")
407
}
408
}
+30
-1472
cmd/plcbundle/main.go
+30
-1472
cmd/plcbundle/main.go
···
1
package main
2
3
import (
4
-
"context"
5
-
"flag"
6
"fmt"
7
-
"net/http"
8
"os"
9
-
"os/signal"
10
-
"path/filepath"
11
-
"runtime"
12
"runtime/debug"
13
-
"sort"
14
-
"strings"
15
-
"sync"
16
-
"syscall"
17
-
"time"
18
19
-
"github.com/goccy/go-json"
20
-
21
-
"tangled.org/atscan.net/plcbundle/internal/bundle"
22
-
"tangled.org/atscan.net/plcbundle/internal/bundleindex"
23
-
"tangled.org/atscan.net/plcbundle/internal/didindex"
24
-
internalsync "tangled.org/atscan.net/plcbundle/internal/sync"
25
-
"tangled.org/atscan.net/plcbundle/internal/types"
26
-
"tangled.org/atscan.net/plcbundle/plcclient"
27
)
28
29
-
// Version information (injected at build time via ldflags or read from build info)
30
-
var (
31
-
version = "dev"
32
-
gitCommit = "unknown"
33
-
buildDate = "unknown"
34
-
)
35
-
36
-
func init() {
37
-
// Try to get version from build info (works with go install)
38
-
if info, ok := debug.ReadBuildInfo(); ok {
39
-
if info.Main.Version != "" && info.Main.Version != "(devel)" {
40
-
version = info.Main.Version
41
-
}
42
-
43
-
// Extract git commit and build time from build settings
44
-
for _, setting := range info.Settings {
45
-
switch setting.Key {
46
-
case "vcs.revision":
47
-
if setting.Value != "" {
48
-
gitCommit = setting.Value
49
-
if len(gitCommit) > 7 {
50
-
gitCommit = gitCommit[:7] // Short hash
51
-
}
52
-
}
53
-
case "vcs.time":
54
-
if setting.Value != "" {
55
-
buildDate = setting.Value
56
-
}
57
-
}
58
-
}
59
-
}
60
-
}
61
-
62
func main() {
63
-
64
debug.SetGCPercent(400)
65
66
if len(os.Args) < 2 {
···
70
71
command := os.Args[1]
72
73
switch command {
74
case "fetch":
75
-
cmdFetch()
76
case "clone":
77
-
cmdClone()
78
case "rebuild":
79
-
cmdRebuild()
80
case "verify":
81
-
cmdVerify()
82
case "info":
83
-
cmdInfo()
84
case "export":
85
-
cmdExport()
86
case "backfill":
87
-
cmdBackfill()
88
case "mempool":
89
-
cmdMempool()
90
case "serve":
91
-
cmdServe()
92
case "compare":
93
-
cmdCompare()
94
case "detector":
95
-
cmdDetector()
96
case "index":
97
-
cmdDIDIndex()
98
case "get-op":
99
-
cmdGetOp()
100
case "version":
101
-
fmt.Printf("plcbundle version %s\n", version)
102
-
fmt.Printf(" commit: %s\n", gitCommit)
103
-
fmt.Printf(" built: %s\n", buildDate)
104
default:
105
fmt.Fprintf(os.Stderr, "Unknown command: %s\n", command)
106
printUsage()
107
os.Exit(1)
108
}
109
}
110
111
func printUsage() {
···
116
117
Commands:
118
fetch Fetch next bundle from PLC directory
119
-
clone Clone bundles from remote HTTP endpoint
120
rebuild Rebuild index from existing bundle files
121
verify Verify bundle integrity
122
info Show bundle information
···
127
compare Compare local index with target index
128
detector Run spam detectors
129
index Manage DID position index
130
version Show version
131
132
-
Security Model:
133
-
Bundles are cryptographically chained but require external verification:
134
-
- Verify against original PLC directory
135
-
- Compare with multiple independent mirrors
136
-
- Check published root and head hashes
137
-
- Anyone can reproduce bundles from PLC directory
138
-
139
-
`, version)
140
-
}
141
-
142
-
// getManager creates or opens a bundle manager in the detected directory
143
-
func getManager(plcURL string) (*bundle.Manager, string, error) {
144
-
dir, err := os.Getwd()
145
-
if err != nil {
146
-
return nil, "", err
147
-
}
148
-
149
-
// Ensure directory exists
150
-
if err := os.MkdirAll(dir, 0755); err != nil {
151
-
return nil, "", fmt.Errorf("failed to create directory: %w", err)
152
-
}
153
-
154
-
config := bundle.DefaultConfig(dir)
155
-
156
-
var client *plcclient.Client
157
-
if plcURL != "" {
158
-
client = plcclient.NewClient(plcURL)
159
-
}
160
-
161
-
mgr, err := bundle.NewManager(config, client)
162
-
if err != nil {
163
-
return nil, "", err
164
-
}
165
-
166
-
return mgr, dir, nil
167
-
}
168
-
169
-
func cmdFetch() {
170
-
fs := flag.NewFlagSet("fetch", flag.ExitOnError)
171
-
plcURL := fs.String("plc", "https://plc.directory", "PLC directory URL")
172
-
count := fs.Int("count", 0, "number of bundles to fetch (0 = fetch all available)")
173
-
verbose := fs.Bool("verbose", false, "verbose sync logging")
174
-
fs.Parse(os.Args[2:])
175
-
176
-
mgr, dir, err := getManager(*plcURL)
177
-
if err != nil {
178
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
179
-
os.Exit(1)
180
-
}
181
-
defer mgr.Close()
182
-
183
-
fmt.Printf("Working in: %s\n", dir)
184
-
185
-
ctx := context.Background()
186
-
187
-
// Get starting bundle info
188
-
index := mgr.GetIndex()
189
-
lastBundle := index.GetLastBundle()
190
-
startBundle := 1
191
-
if lastBundle != nil {
192
-
startBundle = lastBundle.BundleNumber + 1
193
-
}
194
-
195
-
fmt.Printf("Starting from bundle %06d\n", startBundle)
196
-
197
-
if *count > 0 {
198
-
fmt.Printf("Fetching %d bundles...\n", *count)
199
-
} else {
200
-
fmt.Printf("Fetching all available bundles...\n")
201
-
}
202
-
203
-
fetchedCount := 0
204
-
consecutiveErrors := 0
205
-
maxConsecutiveErrors := 3
206
-
207
-
for {
208
-
// Check if we've reached the requested count
209
-
if *count > 0 && fetchedCount >= *count {
210
-
break
211
-
}
212
-
213
-
currentBundle := startBundle + fetchedCount
214
-
215
-
if *count > 0 {
216
-
fmt.Printf("Fetching bundle %d/%d (bundle %06d)...\n", fetchedCount+1, *count, currentBundle)
217
-
} else {
218
-
fmt.Printf("Fetching bundle %06d...\n", currentBundle)
219
-
}
220
-
221
-
b, err := mgr.FetchNextBundle(ctx, !*verbose)
222
-
if err != nil {
223
-
// Check if we've reached the end (insufficient operations)
224
-
if isEndOfDataError(err) {
225
-
fmt.Printf("\n✓ Caught up! No more complete bundles available.\n")
226
-
fmt.Printf(" Last bundle: %06d\n", currentBundle-1)
227
-
break
228
-
}
229
-
230
-
// Handle other errors
231
-
consecutiveErrors++
232
-
fmt.Fprintf(os.Stderr, "Error fetching bundle %06d: %v\n", currentBundle, err)
233
-
234
-
if consecutiveErrors >= maxConsecutiveErrors {
235
-
fmt.Fprintf(os.Stderr, "Too many consecutive errors, stopping.\n")
236
-
os.Exit(1)
237
-
}
238
-
239
-
// Wait a bit before retrying
240
-
fmt.Printf("Waiting 5 seconds before retry...\n")
241
-
time.Sleep(5 * time.Second)
242
-
continue
243
-
}
244
-
245
-
// Reset error counter on success
246
-
consecutiveErrors = 0
247
-
248
-
if err := mgr.SaveBundle(ctx, b, !*verbose); err != nil {
249
-
fmt.Fprintf(os.Stderr, "Error saving bundle %06d: %v\n", b.BundleNumber, err)
250
-
os.Exit(1)
251
-
}
252
-
253
-
fetchedCount++
254
-
fmt.Printf("✓ Saved bundle %06d (%d operations, %d DIDs)\n",
255
-
b.BundleNumber, len(b.Operations), b.DIDCount)
256
-
}
257
-
258
-
if fetchedCount > 0 {
259
-
fmt.Printf("\n✓ Fetch complete: %d bundles retrieved\n", fetchedCount)
260
-
fmt.Printf(" Current range: %06d - %06d\n", startBundle, startBundle+fetchedCount-1)
261
-
} else {
262
-
fmt.Printf("\n✓ Already up to date!\n")
263
-
}
264
-
}
265
-
266
-
func cmdClone() {
267
-
fs := flag.NewFlagSet("clone", flag.ExitOnError)
268
-
workers := fs.Int("workers", 4, "number of concurrent download workers")
269
-
verbose := fs.Bool("v", false, "verbose output")
270
-
skipExisting := fs.Bool("skip-existing", true, "skip bundles that already exist locally")
271
-
saveInterval := fs.Duration("save-interval", 5*time.Second, "interval to save index during download")
272
-
fs.Parse(os.Args[2:])
273
-
274
-
if fs.NArg() < 1 {
275
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle clone <remote-url> [options]\n")
276
-
fmt.Fprintf(os.Stderr, "\nClone bundles from a remote plcbundle HTTP endpoint\n\n")
277
-
fmt.Fprintf(os.Stderr, "Options:\n")
278
-
fs.PrintDefaults()
279
-
fmt.Fprintf(os.Stderr, "\nExample:\n")
280
-
fmt.Fprintf(os.Stderr, " plcbundle clone https://plc.example.com\n")
281
-
os.Exit(1)
282
-
}
283
-
284
-
remoteURL := strings.TrimSuffix(fs.Arg(0), "/")
285
-
286
-
// Create manager
287
-
mgr, dir, err := getManager("")
288
-
if err != nil {
289
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
290
-
os.Exit(1)
291
-
}
292
-
defer mgr.Close()
293
-
294
-
fmt.Printf("Cloning from: %s\n", remoteURL)
295
-
fmt.Printf("Target directory: %s\n", dir)
296
-
fmt.Printf("Workers: %d\n", *workers)
297
-
fmt.Printf("(Press Ctrl+C to safely interrupt - progress will be saved)\n\n")
298
-
299
-
// Set up signal handling
300
-
ctx, cancel := context.WithCancel(context.Background())
301
-
defer cancel()
302
-
303
-
sigChan := make(chan os.Signal, 1)
304
-
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
305
-
306
-
// Set up progress bar
307
-
var progress *ProgressBar
308
-
var progressMu sync.Mutex
309
-
progressActive := true
310
-
311
-
go func() {
312
-
<-sigChan
313
-
progressMu.Lock()
314
-
progressActive = false
315
-
if progress != nil {
316
-
fmt.Println()
317
-
}
318
-
progressMu.Unlock()
319
-
320
-
fmt.Printf("\n⚠️ Interrupt received! Finishing current downloads and saving progress...\n")
321
-
cancel()
322
-
}()
323
-
324
-
// Clone with library
325
-
result, err := mgr.CloneFromRemote(ctx, internalsync.CloneOptions{
326
-
RemoteURL: remoteURL,
327
-
Workers: *workers,
328
-
SkipExisting: *skipExisting,
329
-
SaveInterval: *saveInterval,
330
-
Verbose: *verbose,
331
-
ProgressFunc: func(downloaded, total int, bytesDownloaded, bytesTotal int64) {
332
-
progressMu.Lock()
333
-
defer progressMu.Unlock()
334
-
335
-
// Stop updating progress if interrupted
336
-
if !progressActive {
337
-
return
338
-
}
339
-
340
-
if progress == nil {
341
-
progress = NewProgressBarWithBytes(total, bytesTotal)
342
-
progress.showBytes = true
343
-
}
344
-
progress.SetWithBytes(downloaded, bytesDownloaded)
345
-
},
346
-
})
347
-
348
-
// Ensure progress is stopped
349
-
progressMu.Lock()
350
-
progressActive = false
351
-
if progress != nil {
352
-
progress.Finish()
353
-
}
354
-
progressMu.Unlock()
355
-
356
-
if err != nil {
357
-
fmt.Fprintf(os.Stderr, "Clone failed: %v\n", err)
358
-
os.Exit(1)
359
-
}
360
-
361
-
// Display results
362
-
if result.Interrupted {
363
-
fmt.Printf("⚠️ Download interrupted by user\n")
364
-
} else {
365
-
fmt.Printf("\n✓ Clone complete in %s\n", result.Duration.Round(time.Millisecond))
366
-
}
367
-
368
-
fmt.Printf("\nResults:\n")
369
-
fmt.Printf(" Remote bundles: %d\n", result.RemoteBundles)
370
-
if result.Skipped > 0 {
371
-
fmt.Printf(" Skipped (existing): %d\n", result.Skipped)
372
-
}
373
-
fmt.Printf(" Downloaded: %d\n", result.Downloaded)
374
-
if result.Failed > 0 {
375
-
fmt.Printf(" Failed: %d\n", result.Failed)
376
-
}
377
-
fmt.Printf(" Total size: %s\n", formatBytes(result.TotalBytes))
378
-
379
-
if result.Duration.Seconds() > 0 && result.Downloaded > 0 {
380
-
mbPerSec := float64(result.TotalBytes) / result.Duration.Seconds() / (1024 * 1024)
381
-
bundlesPerSec := float64(result.Downloaded) / result.Duration.Seconds()
382
-
fmt.Printf(" Average speed: %.1f MB/s (%.1f bundles/s)\n", mbPerSec, bundlesPerSec)
383
-
}
384
-
385
-
if result.Failed > 0 {
386
-
fmt.Printf("\n⚠️ Failed bundles: ")
387
-
for i, num := range result.FailedBundles {
388
-
if i > 0 {
389
-
fmt.Printf(", ")
390
-
}
391
-
if i > 10 {
392
-
fmt.Printf("... and %d more", len(result.FailedBundles)-10)
393
-
break
394
-
}
395
-
fmt.Printf("%06d", num)
396
-
}
397
-
fmt.Printf("\nRe-run the clone command to retry failed bundles.\n")
398
-
os.Exit(1)
399
-
}
400
-
401
-
if result.Interrupted {
402
-
fmt.Printf("\n✓ Progress saved. Re-run the clone command to resume.\n")
403
-
os.Exit(1)
404
-
}
405
-
406
-
fmt.Printf("\n✓ Clone complete!\n")
407
-
}
408
-
409
-
func cmdRebuild() {
410
-
fs := flag.NewFlagSet("rebuild", flag.ExitOnError)
411
-
verbose := fs.Bool("v", false, "verbose output")
412
-
workers := fs.Int("workers", 4, "number of parallel workers (0 = CPU count)")
413
-
noProgress := fs.Bool("no-progress", false, "disable progress bar")
414
-
fs.Parse(os.Args[2:])
415
-
416
-
// Auto-detect CPU count
417
-
if *workers == 0 {
418
-
*workers = runtime.NumCPU()
419
-
}
420
-
421
-
// Create manager WITHOUT auto-rebuild (we'll do it manually)
422
-
dir, err := os.Getwd()
423
-
if err != nil {
424
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
425
-
os.Exit(1)
426
-
}
427
-
428
-
// Ensure directory exists
429
-
if err := os.MkdirAll(dir, 0755); err != nil {
430
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
431
-
os.Exit(1)
432
-
}
433
-
434
-
config := bundle.DefaultConfig(dir)
435
-
config.AutoRebuild = false
436
-
config.RebuildWorkers = *workers
437
-
438
-
mgr, err := bundle.NewManager(config, nil)
439
-
if err != nil {
440
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
441
-
os.Exit(1)
442
-
}
443
-
defer mgr.Close()
444
-
445
-
fmt.Printf("Rebuilding index from: %s\n", dir)
446
-
fmt.Printf("Using %d workers\n", *workers)
447
-
448
-
// Find all bundle files
449
-
files, err := filepath.Glob(filepath.Join(dir, "*.jsonl.zst"))
450
-
if err != nil {
451
-
fmt.Fprintf(os.Stderr, "Error scanning directory: %v\n", err)
452
-
os.Exit(1)
453
-
}
454
-
455
-
if len(files) == 0 {
456
-
fmt.Println("No bundle files found")
457
-
return
458
-
}
459
-
460
-
fmt.Printf("Found %d bundle files\n", len(files))
461
-
fmt.Printf("\n")
462
-
463
-
start := time.Now()
464
-
465
-
// Create progress bar
466
-
var progress *ProgressBar
467
-
var progressCallback func(int, int, int64)
468
-
469
-
if !*noProgress {
470
-
fmt.Println("Processing bundles:")
471
-
progress = NewProgressBar(len(files))
472
-
progress.showBytes = true // Enable byte tracking
473
-
474
-
progressCallback = func(current, total int, bytesProcessed int64) {
475
-
progress.SetWithBytes(current, bytesProcessed)
476
-
}
477
-
}
478
-
479
-
// Use parallel scan
480
-
result, err := mgr.ScanDirectoryParallel(*workers, progressCallback)
481
-
482
-
if err != nil {
483
-
if progress != nil {
484
-
progress.Finish()
485
-
}
486
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
487
-
os.Exit(1)
488
-
}
489
-
490
-
// Finish progress bar
491
-
if progress != nil {
492
-
progress.Finish()
493
-
}
494
-
495
-
elapsed := time.Since(start)
496
-
497
-
fmt.Printf("\n")
498
-
fmt.Printf("✓ Index rebuilt in %s\n", elapsed.Round(time.Millisecond))
499
-
fmt.Printf(" Total bundles: %d\n", result.BundleCount)
500
-
fmt.Printf(" Compressed size: %s\n", formatBytes(result.TotalSize))
501
-
fmt.Printf(" Uncompressed size: %s\n", formatBytes(result.TotalUncompressed))
502
-
503
-
// Calculate compression ratio
504
-
if result.TotalUncompressed > 0 {
505
-
ratio := float64(result.TotalUncompressed) / float64(result.TotalSize)
506
-
fmt.Printf(" Compression ratio: %.2fx\n", ratio)
507
-
}
508
-
509
-
fmt.Printf(" Average speed: %.1f bundles/sec\n", float64(result.BundleCount)/elapsed.Seconds())
510
-
511
-
if elapsed.Seconds() > 0 {
512
-
compressedThroughput := float64(result.TotalSize) / elapsed.Seconds() / (1000 * 1000)
513
-
uncompressedThroughput := float64(result.TotalUncompressed) / elapsed.Seconds() / (1000 * 1000)
514
-
fmt.Printf(" Throughput (compressed): %.1f MB/s\n", compressedThroughput)
515
-
fmt.Printf(" Throughput (uncompressed): %.1f MB/s\n", uncompressedThroughput)
516
-
}
517
-
518
-
fmt.Printf(" Index file: %s\n", filepath.Join(dir, bundleindex.INDEX_FILE))
519
-
520
-
if len(result.MissingGaps) > 0 {
521
-
fmt.Printf(" ⚠️ Missing gaps: %d bundles\n", len(result.MissingGaps))
522
-
}
523
-
524
-
// Verify chain if requested
525
-
if *verbose {
526
-
fmt.Printf("\n")
527
-
fmt.Printf("Verifying chain integrity...\n")
528
-
529
-
ctx := context.Background()
530
-
verifyResult, err := mgr.VerifyChain(ctx)
531
-
if err != nil {
532
-
fmt.Printf(" ⚠️ Verification error: %v\n", err)
533
-
} else if verifyResult.Valid {
534
-
fmt.Printf(" ✓ Chain is valid (%d bundles verified)\n", len(verifyResult.VerifiedBundles))
535
-
536
-
// Show head hash
537
-
index := mgr.GetIndex()
538
-
if lastMeta := index.GetLastBundle(); lastMeta != nil {
539
-
fmt.Printf(" Chain head: %s...\n", lastMeta.Hash[:16])
540
-
}
541
-
} else {
542
-
fmt.Printf(" ✗ Chain verification failed\n")
543
-
fmt.Printf(" Broken at: bundle %06d\n", verifyResult.BrokenAt)
544
-
fmt.Printf(" Error: %s\n", verifyResult.Error)
545
-
}
546
-
}
547
-
}
548
-
549
-
func cmdVerify() {
550
-
fs := flag.NewFlagSet("verify", flag.ExitOnError)
551
-
bundleNum := fs.Int("bundle", 0, "specific bundle to verify (0 = verify chain)")
552
-
verbose := fs.Bool("v", false, "verbose output")
553
-
fs.Parse(os.Args[2:])
554
-
555
-
mgr, dir, err := getManager("")
556
-
if err != nil {
557
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
558
-
os.Exit(1)
559
-
}
560
-
defer mgr.Close()
561
-
562
-
fmt.Printf("Working in: %s\n", dir)
563
-
564
-
ctx := context.Background()
565
-
566
-
if *bundleNum > 0 {
567
-
// Verify specific bundle
568
-
fmt.Printf("Verifying bundle %06d...\n", *bundleNum)
569
-
570
-
result, err := mgr.VerifyBundle(ctx, *bundleNum)
571
-
if err != nil {
572
-
fmt.Fprintf(os.Stderr, "Verification failed: %v\n", err)
573
-
os.Exit(1)
574
-
}
575
-
576
-
if result.Valid {
577
-
fmt.Printf("✓ Bundle %06d is valid\n", *bundleNum)
578
-
if *verbose {
579
-
fmt.Printf(" File exists: %v\n", result.FileExists)
580
-
fmt.Printf(" Hash match: %v\n", result.HashMatch)
581
-
fmt.Printf(" Hash: %s\n", result.LocalHash[:16]+"...")
582
-
}
583
-
} else {
584
-
fmt.Printf("✗ Bundle %06d is invalid\n", *bundleNum)
585
-
if result.Error != nil {
586
-
fmt.Printf(" Error: %v\n", result.Error)
587
-
}
588
-
if !result.FileExists {
589
-
fmt.Printf(" File not found\n")
590
-
}
591
-
if !result.HashMatch && result.FileExists {
592
-
fmt.Printf(" Expected hash: %s...\n", result.ExpectedHash[:16])
593
-
fmt.Printf(" Actual hash: %s...\n", result.LocalHash[:16])
594
-
}
595
-
os.Exit(1)
596
-
}
597
-
} else {
598
-
// Verify entire chain
599
-
index := mgr.GetIndex()
600
-
bundles := index.GetBundles()
601
-
602
-
if len(bundles) == 0 {
603
-
fmt.Println("No bundles to verify")
604
-
return
605
-
}
606
-
607
-
fmt.Printf("Verifying chain of %d bundles...\n", len(bundles))
608
-
fmt.Println()
609
-
610
-
verifiedCount := 0
611
-
errorCount := 0
612
-
lastPercent := -1
613
-
614
-
for i, meta := range bundles {
615
-
bundleNum := meta.BundleNumber
616
-
617
-
// Show progress
618
-
percent := (i * 100) / len(bundles)
619
-
if percent != lastPercent || *verbose {
620
-
if *verbose {
621
-
fmt.Printf(" [%3d%%] Verifying bundle %06d...", percent, bundleNum)
622
-
} else if percent%10 == 0 && percent != lastPercent {
623
-
fmt.Printf(" [%3d%%] Verified %d/%d bundles...\n", percent, i, len(bundles))
624
-
}
625
-
lastPercent = percent
626
-
}
627
-
628
-
// Verify file hash
629
-
result, err := mgr.VerifyBundle(ctx, bundleNum)
630
-
if err != nil {
631
-
if *verbose {
632
-
fmt.Printf(" ERROR\n")
633
-
}
634
-
fmt.Printf("\n✗ Failed to verify bundle %06d: %v\n", bundleNum, err)
635
-
errorCount++
636
-
continue
637
-
}
638
-
639
-
if !result.Valid {
640
-
if *verbose {
641
-
fmt.Printf(" INVALID\n")
642
-
}
643
-
fmt.Printf("\n✗ Bundle %06d hash verification failed\n", bundleNum)
644
-
if result.Error != nil {
645
-
fmt.Printf(" Error: %v\n", result.Error)
646
-
}
647
-
errorCount++
648
-
continue
649
-
}
650
-
651
-
// Verify chain link (prev_bundle_hash)
652
-
if i > 0 {
653
-
prevMeta := bundles[i-1]
654
-
if meta.Parent != prevMeta.Hash {
655
-
if *verbose {
656
-
fmt.Printf(" CHAIN BROKEN\n")
657
-
}
658
-
fmt.Printf("\n✗ Chain broken at bundle %06d\n", bundleNum)
659
-
fmt.Printf(" Expected parent: %s...\n", prevMeta.Hash[:16])
660
-
fmt.Printf(" Actual parent: %s...\n", meta.Parent[:16])
661
-
errorCount++
662
-
continue
663
-
}
664
-
}
665
-
666
-
if *verbose {
667
-
fmt.Printf(" ✓\n")
668
-
}
669
-
verifiedCount++
670
-
}
671
-
672
-
// Final summary
673
-
fmt.Println()
674
-
if errorCount == 0 {
675
-
fmt.Printf("✓ Chain is valid (%d bundles verified)\n", verifiedCount)
676
-
fmt.Printf(" First bundle: %06d\n", bundles[0].BundleNumber)
677
-
fmt.Printf(" Last bundle: %06d\n", bundles[len(bundles)-1].BundleNumber)
678
-
fmt.Printf(" Chain head: %s...\n", bundles[len(bundles)-1].Hash[:16])
679
-
} else {
680
-
fmt.Printf("✗ Chain verification failed\n")
681
-
fmt.Printf(" Verified: %d/%d bundles\n", verifiedCount, len(bundles))
682
-
fmt.Printf(" Errors: %d\n", errorCount)
683
-
os.Exit(1)
684
-
}
685
-
}
686
-
}
687
-
688
-
func cmdInfo() {
689
-
fs := flag.NewFlagSet("info", flag.ExitOnError)
690
-
bundleNum := fs.Int("bundle", 0, "specific bundle info (0 = general info)")
691
-
verbose := fs.Bool("v", false, "verbose output")
692
-
showBundles := fs.Bool("bundles", false, "show bundle list")
693
-
verify := fs.Bool("verify", false, "verify chain integrity")
694
-
showTimeline := fs.Bool("timeline", false, "show timeline visualization")
695
-
fs.Parse(os.Args[2:])
696
-
697
-
mgr, dir, err := getManager("")
698
-
if err != nil {
699
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
700
-
os.Exit(1)
701
-
}
702
-
defer mgr.Close()
703
-
704
-
if *bundleNum > 0 {
705
-
showBundleInfo(mgr, dir, *bundleNum, *verbose)
706
-
} else {
707
-
showGeneralInfo(mgr, dir, *verbose, *showBundles, *verify, *showTimeline)
708
-
}
709
-
}
710
-
711
-
func showBundleInfo(mgr *bundle.Manager, dir string, bundleNum int, verbose bool) {
712
-
ctx := context.Background()
713
-
b, err := mgr.LoadBundle(ctx, bundleNum)
714
-
if err != nil {
715
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
716
-
os.Exit(1)
717
-
}
718
-
719
-
fmt.Printf("\n")
720
-
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
721
-
fmt.Printf(" Bundle %06d\n", b.BundleNumber)
722
-
fmt.Printf("═══════════════════════════════════════════════════════════════\n")
723
-
fmt.Printf("\n")
724
-
725
-
// Location
726
-
fmt.Printf("📁 Location\n")
727
-
fmt.Printf(" Directory: %s\n", dir)
728
-
fmt.Printf(" File: %06d.jsonl.zst\n", b.BundleNumber)
729
-
fmt.Printf("\n")
730
-
731
-
// Time Range
732
-
duration := b.EndTime.Sub(b.StartTime)
733
-
fmt.Printf("📅 Time Range\n")
734
-
fmt.Printf(" Start: %s\n", b.StartTime.Format("2006-01-02 15:04:05.000 MST"))
735
-
fmt.Printf(" End: %s\n", b.EndTime.Format("2006-01-02 15:04:05.000 MST"))
736
-
fmt.Printf(" Duration: %s\n", formatDuration(duration))
737
-
fmt.Printf(" Created: %s\n", b.CreatedAt.Format("2006-01-02 15:04:05 MST"))
738
-
fmt.Printf("\n")
739
-
740
-
// Content
741
-
fmt.Printf("📊 Content\n")
742
-
fmt.Printf(" Operations: %s\n", formatNumber(len(b.Operations)))
743
-
fmt.Printf(" Unique DIDs: %s\n", formatNumber(b.DIDCount))
744
-
if len(b.Operations) > 0 {
745
-
avgOpsPerDID := float64(len(b.Operations)) / float64(b.DIDCount)
746
-
fmt.Printf(" Avg ops/DID: %.2f\n", avgOpsPerDID)
747
-
}
748
-
fmt.Printf("\n")
749
-
750
-
// Size
751
-
fmt.Printf("💾 Size\n")
752
-
fmt.Printf(" Compressed: %s\n", formatBytes(b.CompressedSize))
753
-
fmt.Printf(" Uncompressed: %s\n", formatBytes(b.UncompressedSize))
754
-
fmt.Printf(" Ratio: %.2fx\n", b.CompressionRatio())
755
-
fmt.Printf(" Efficiency: %.1f%% savings\n", (1-float64(b.CompressedSize)/float64(b.UncompressedSize))*100)
756
-
fmt.Printf("\n")
757
-
758
-
// Hashes
759
-
fmt.Printf("🔐 Cryptographic Hashes\n")
760
-
fmt.Printf(" Chain Hash:\n")
761
-
fmt.Printf(" %s\n", b.Hash)
762
-
fmt.Printf(" Content Hash:\n")
763
-
fmt.Printf(" %s\n", b.ContentHash)
764
-
fmt.Printf(" Compressed:\n")
765
-
fmt.Printf(" %s\n", b.CompressedHash)
766
-
if b.Parent != "" {
767
-
fmt.Printf(" Parent Chain Hash:\n")
768
-
fmt.Printf(" %s\n", b.Parent)
769
-
}
770
-
fmt.Printf("\n")
771
-
772
-
// Chain
773
-
if b.Parent != "" || b.Cursor != "" {
774
-
fmt.Printf("🔗 Chain Information\n")
775
-
if b.Cursor != "" {
776
-
fmt.Printf(" Cursor: %s\n", b.Cursor)
777
-
}
778
-
if b.Parent != "" {
779
-
fmt.Printf(" Links to: Bundle %06d\n", bundleNum-1)
780
-
}
781
-
if len(b.BoundaryCIDs) > 0 {
782
-
fmt.Printf(" Boundary: %d CIDs at same timestamp\n", len(b.BoundaryCIDs))
783
-
}
784
-
fmt.Printf("\n")
785
-
}
786
-
787
-
// Verbose: Show sample operations
788
-
if verbose && len(b.Operations) > 0 {
789
-
fmt.Printf("📝 Sample Operations (first 5)\n")
790
-
showCount := 5
791
-
if len(b.Operations) < showCount {
792
-
showCount = len(b.Operations)
793
-
}
794
-
for i := 0; i < showCount; i++ {
795
-
op := b.Operations[i]
796
-
fmt.Printf(" %d. %s\n", i+1, op.DID)
797
-
fmt.Printf(" CID: %s\n", op.CID)
798
-
fmt.Printf(" Time: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000"))
799
-
if op.IsNullified() {
800
-
fmt.Printf(" ⚠️ Nullified: %s\n", op.GetNullifyingCID())
801
-
}
802
-
}
803
-
fmt.Printf("\n")
804
-
}
805
-
806
-
// Verbose: Show DID statistics
807
-
if verbose && len(b.Operations) > 0 {
808
-
didOps := make(map[string]int)
809
-
for _, op := range b.Operations {
810
-
didOps[op.DID]++
811
-
}
812
-
813
-
// Find most active DIDs
814
-
type didCount struct {
815
-
did string
816
-
count int
817
-
}
818
-
var counts []didCount
819
-
for did, count := range didOps {
820
-
counts = append(counts, didCount{did, count})
821
-
}
822
-
sort.Slice(counts, func(i, j int) bool {
823
-
return counts[i].count > counts[j].count
824
-
})
825
-
826
-
fmt.Printf("🏆 Most Active DIDs\n")
827
-
showCount := 5
828
-
if len(counts) < showCount {
829
-
showCount = len(counts)
830
-
}
831
-
for i := 0; i < showCount; i++ {
832
-
fmt.Printf(" %d. %s (%d ops)\n", i+1, counts[i].did, counts[i].count)
833
-
}
834
-
fmt.Printf("\n")
835
-
}
836
-
}
837
-
838
-
func cmdExport() {
839
-
fs := flag.NewFlagSet("export", flag.ExitOnError)
840
-
bundles := fs.String("bundles", "", "bundle number or range (e.g., '42' or '1-100')")
841
-
all := fs.Bool("all", false, "export all bundles")
842
-
count := fs.Int("count", 0, "limit number of operations (0 = all)")
843
-
after := fs.String("after", "", "timestamp to start after (RFC3339)")
844
-
fs.Parse(os.Args[2:])
845
-
846
-
// Validate flags
847
-
if !*all && *bundles == "" {
848
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle export --bundles <number|range> [options]\n")
849
-
fmt.Fprintf(os.Stderr, " or: plcbundle export --all [options]\n")
850
-
fmt.Fprintf(os.Stderr, "\nExamples:\n")
851
-
fmt.Fprintf(os.Stderr, " plcbundle export --bundles 42\n")
852
-
fmt.Fprintf(os.Stderr, " plcbundle export --bundles 1-100\n")
853
-
fmt.Fprintf(os.Stderr, " plcbundle export --all\n")
854
-
fmt.Fprintf(os.Stderr, " plcbundle export --all --count 50000\n")
855
-
fmt.Fprintf(os.Stderr, " plcbundle export --bundles 42 | jq .\n")
856
-
os.Exit(1)
857
-
}
858
-
859
-
// Load manager
860
-
mgr, _, err := getManager("")
861
-
if err != nil {
862
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
863
-
os.Exit(1)
864
-
}
865
-
defer mgr.Close()
866
-
867
-
// Determine bundle range
868
-
var start, end int
869
-
if *all {
870
-
// Export all bundles
871
-
index := mgr.GetIndex()
872
-
bundles := index.GetBundles()
873
-
if len(bundles) == 0 {
874
-
fmt.Fprintf(os.Stderr, "No bundles available\n")
875
-
os.Exit(1)
876
-
}
877
-
start = bundles[0].BundleNumber
878
-
end = bundles[len(bundles)-1].BundleNumber
879
-
880
-
fmt.Fprintf(os.Stderr, "Exporting all bundles (%d-%d)\n", start, end)
881
-
} else {
882
-
// Parse bundle range
883
-
start, end, err = parseBundleRange(*bundles)
884
-
if err != nil {
885
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
886
-
os.Exit(1)
887
-
}
888
-
fmt.Fprintf(os.Stderr, "Exporting bundles %d-%d\n", start, end)
889
-
}
890
-
891
-
// Log to stderr
892
-
if *count > 0 {
893
-
fmt.Fprintf(os.Stderr, "Limit: %d operations\n", *count)
894
-
}
895
-
if *after != "" {
896
-
fmt.Fprintf(os.Stderr, "After: %s\n", *after)
897
-
}
898
-
fmt.Fprintf(os.Stderr, "\n")
899
-
900
-
// Parse after time if provided
901
-
var afterTime time.Time
902
-
if *after != "" {
903
-
afterTime, err = time.Parse(time.RFC3339, *after)
904
-
if err != nil {
905
-
fmt.Fprintf(os.Stderr, "Invalid after time: %v\n", err)
906
-
os.Exit(1)
907
-
}
908
-
}
909
-
910
-
ctx := context.Background()
911
-
exported := 0
912
-
913
-
// Export operations from bundles
914
-
for bundleNum := start; bundleNum <= end; bundleNum++ {
915
-
// Check if we've reached the limit
916
-
if *count > 0 && exported >= *count {
917
-
break
918
-
}
919
-
920
-
fmt.Fprintf(os.Stderr, "Processing bundle %d...\r", bundleNum)
921
-
922
-
bundle, err := mgr.LoadBundle(ctx, bundleNum)
923
-
if err != nil {
924
-
fmt.Fprintf(os.Stderr, "\nWarning: failed to load bundle %d: %v\n", bundleNum, err)
925
-
continue
926
-
}
927
-
928
-
// Output operations
929
-
for _, op := range bundle.Operations {
930
-
// Check after time filter
931
-
if !afterTime.IsZero() && op.CreatedAt.Before(afterTime) {
932
-
continue
933
-
}
934
-
935
-
// Check count limit
936
-
if *count > 0 && exported >= *count {
937
-
break
938
-
}
939
-
940
-
// Output operation as JSONL
941
-
if len(op.RawJSON) > 0 {
942
-
fmt.Println(string(op.RawJSON))
943
-
} else {
944
-
// Fallback to marshaling
945
-
data, _ := json.Marshal(op)
946
-
fmt.Println(string(data))
947
-
}
948
-
949
-
exported++
950
-
}
951
-
}
952
-
953
-
// Final stats to stderr
954
-
fmt.Fprintf(os.Stderr, "\n\n")
955
-
fmt.Fprintf(os.Stderr, "✓ Export complete\n")
956
-
fmt.Fprintf(os.Stderr, " Exported: %d operations\n", exported)
957
-
}
958
-
959
-
func cmdBackfill() {
960
-
fs := flag.NewFlagSet("backfill", flag.ExitOnError)
961
-
plcURL := fs.String("plc", "https://plc.directory", "PLC directory URL")
962
-
startFrom := fs.Int("start", 1, "bundle number to start from")
963
-
endAt := fs.Int("end", 0, "bundle number to end at (0 = until caught up)")
964
-
verbose := fs.Bool("verbose", false, "verbose sync logging")
965
-
fs.Parse(os.Args[2:])
966
-
967
-
mgr, dir, err := getManager(*plcURL)
968
-
if err != nil {
969
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
970
-
os.Exit(1)
971
-
}
972
-
defer mgr.Close()
973
-
974
-
fmt.Fprintf(os.Stderr, "Starting backfill from: %s\n", dir)
975
-
fmt.Fprintf(os.Stderr, "Starting from bundle: %06d\n", *startFrom)
976
-
if *endAt > 0 {
977
-
fmt.Fprintf(os.Stderr, "Ending at bundle: %06d\n", *endAt)
978
-
} else {
979
-
fmt.Fprintf(os.Stderr, "Ending: when caught up\n")
980
-
}
981
-
fmt.Fprintf(os.Stderr, "\n")
982
-
983
-
ctx := context.Background()
984
-
985
-
currentBundle := *startFrom
986
-
processedCount := 0
987
-
fetchedCount := 0
988
-
loadedCount := 0
989
-
operationCount := 0
990
-
991
-
for {
992
-
// Check if we've reached the end bundle
993
-
if *endAt > 0 && currentBundle > *endAt {
994
-
break
995
-
}
996
-
997
-
fmt.Fprintf(os.Stderr, "Processing bundle %06d... ", currentBundle)
998
-
999
-
// Try to load from disk first
1000
-
bundle, err := mgr.LoadBundle(ctx, currentBundle)
1001
-
1002
-
if err != nil {
1003
-
// Bundle doesn't exist, try to fetch it
1004
-
fmt.Fprintf(os.Stderr, "fetching... ")
1005
-
1006
-
bundle, err = mgr.FetchNextBundle(ctx, !*verbose)
1007
-
if err != nil {
1008
-
if isEndOfDataError(err) {
1009
-
fmt.Fprintf(os.Stderr, "\n✓ Caught up! No more complete bundles available.\n")
1010
-
break
1011
-
}
1012
-
fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
1013
-
1014
-
// If we can't fetch, we're done
1015
-
break
1016
-
}
1017
-
1018
-
// Save the fetched bundle
1019
-
if err := mgr.SaveBundle(ctx, bundle, !*verbose); err != nil {
1020
-
fmt.Fprintf(os.Stderr, "ERROR saving: %v\n", err)
1021
-
os.Exit(1)
1022
-
}
1023
-
1024
-
fetchedCount++
1025
-
fmt.Fprintf(os.Stderr, "saved... ")
1026
-
} else {
1027
-
loadedCount++
1028
-
}
1029
-
1030
-
// Output operations to stdout (JSONL)
1031
-
for _, op := range bundle.Operations {
1032
-
if len(op.RawJSON) > 0 {
1033
-
fmt.Println(string(op.RawJSON))
1034
-
}
1035
-
}
1036
-
1037
-
operationCount += len(bundle.Operations)
1038
-
processedCount++
1039
-
1040
-
fmt.Fprintf(os.Stderr, "✓ (%d ops, %d DIDs)\n", len(bundle.Operations), bundle.DIDCount)
1041
-
1042
-
currentBundle++
1043
-
1044
-
// Show progress summary every 100 bundles
1045
-
if processedCount%100 == 0 {
1046
-
fmt.Fprintf(os.Stderr, "\n--- Progress: %d bundles processed (%d fetched, %d loaded) ---\n",
1047
-
processedCount, fetchedCount, loadedCount)
1048
-
fmt.Fprintf(os.Stderr, " Total operations: %d\n\n", operationCount)
1049
-
}
1050
-
}
1051
-
1052
-
// Final summary
1053
-
fmt.Fprintf(os.Stderr, "\n")
1054
-
fmt.Fprintf(os.Stderr, "✓ Backfill complete\n")
1055
-
fmt.Fprintf(os.Stderr, " Bundles processed: %d\n", processedCount)
1056
-
fmt.Fprintf(os.Stderr, " Newly fetched: %d\n", fetchedCount)
1057
-
fmt.Fprintf(os.Stderr, " Loaded from disk: %d\n", loadedCount)
1058
-
fmt.Fprintf(os.Stderr, " Total operations: %d\n", operationCount)
1059
-
fmt.Fprintf(os.Stderr, " Range: %06d - %06d\n", *startFrom, currentBundle-1)
1060
-
}
1061
-
1062
-
func cmdMempool() {
1063
-
fs := flag.NewFlagSet("mempool", flag.ExitOnError)
1064
-
clear := fs.Bool("clear", false, "clear the mempool")
1065
-
export := fs.Bool("export", false, "export mempool operations as JSONL to stdout")
1066
-
refresh := fs.Bool("refresh", false, "reload mempool from disk")
1067
-
validate := fs.Bool("validate", false, "validate chronological order")
1068
-
verbose := fs.Bool("v", false, "verbose output")
1069
-
fs.Parse(os.Args[2:])
1070
-
1071
-
mgr, dir, err := getManager("")
1072
-
if err != nil {
1073
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
1074
-
os.Exit(1)
1075
-
}
1076
-
defer mgr.Close()
1077
-
1078
-
fmt.Printf("Working in: %s\n", dir)
1079
-
fmt.Println()
1080
-
1081
-
// Handle validate
1082
-
if *validate {
1083
-
fmt.Printf("Validating mempool chronological order...\n")
1084
-
if err := mgr.ValidateMempool(); err != nil {
1085
-
fmt.Fprintf(os.Stderr, "✗ Validation failed: %v\n", err)
1086
-
os.Exit(1)
1087
-
}
1088
-
fmt.Printf("✓ Mempool validation passed\n")
1089
-
return
1090
-
}
1091
-
1092
-
// Handle refresh
1093
-
if *refresh {
1094
-
fmt.Printf("Refreshing mempool from disk...\n")
1095
-
if err := mgr.RefreshMempool(); err != nil {
1096
-
fmt.Fprintf(os.Stderr, "Error refreshing mempool: %v\n", err)
1097
-
os.Exit(1)
1098
-
}
1099
-
1100
-
// Validate after refresh
1101
-
if err := mgr.ValidateMempool(); err != nil {
1102
-
fmt.Fprintf(os.Stderr, "⚠ Warning: mempool validation failed after refresh: %v\n", err)
1103
-
} else {
1104
-
fmt.Printf("✓ Mempool refreshed and validated\n\n")
1105
-
}
1106
-
}
1107
-
1108
-
// Handle clear
1109
-
if *clear {
1110
-
stats := mgr.GetMempoolStats()
1111
-
count := stats["count"].(int)
1112
-
1113
-
if count == 0 {
1114
-
fmt.Println("Mempool is already empty")
1115
-
return
1116
-
}
1117
-
1118
-
fmt.Printf("⚠ This will clear %d operations from the mempool.\n", count)
1119
-
fmt.Printf("Are you sure? [y/N]: ")
1120
-
var response string
1121
-
fmt.Scanln(&response)
1122
-
if strings.ToLower(strings.TrimSpace(response)) != "y" {
1123
-
fmt.Println("Cancelled")
1124
-
return
1125
-
}
1126
-
1127
-
if err := mgr.ClearMempool(); err != nil {
1128
-
fmt.Fprintf(os.Stderr, "Error clearing mempool: %v\n", err)
1129
-
os.Exit(1)
1130
-
}
1131
-
1132
-
fmt.Printf("✓ Mempool cleared (%d operations removed)\n", count)
1133
-
return
1134
-
}
1135
1136
-
// Handle export
1137
-
if *export {
1138
-
ops, err := mgr.GetMempoolOperations()
1139
-
if err != nil {
1140
-
fmt.Fprintf(os.Stderr, "Error getting mempool operations: %v\n", err)
1141
-
os.Exit(1)
1142
-
}
1143
-
1144
-
if len(ops) == 0 {
1145
-
fmt.Fprintf(os.Stderr, "Mempool is empty\n")
1146
-
return
1147
-
}
1148
-
1149
-
// Output as JSONL to stdout
1150
-
for _, op := range ops {
1151
-
if len(op.RawJSON) > 0 {
1152
-
fmt.Println(string(op.RawJSON))
1153
-
}
1154
-
}
1155
-
1156
-
fmt.Fprintf(os.Stderr, "Exported %d operations from mempool\n", len(ops))
1157
-
return
1158
-
}
1159
-
1160
-
// Default: Show mempool stats
1161
-
stats := mgr.GetMempoolStats()
1162
-
count := stats["count"].(int)
1163
-
canCreate := stats["can_create_bundle"].(bool)
1164
-
targetBundle := stats["target_bundle"].(int)
1165
-
minTimestamp := stats["min_timestamp"].(time.Time)
1166
-
validated := stats["validated"].(bool)
1167
-
1168
-
fmt.Printf("Mempool Status:\n")
1169
-
fmt.Printf(" Target bundle: %06d\n", targetBundle)
1170
-
fmt.Printf(" Operations: %d\n", count)
1171
-
fmt.Printf(" Can create bundle: %v (need %d)\n", canCreate, types.BUNDLE_SIZE)
1172
-
fmt.Printf(" Min timestamp: %s\n", minTimestamp.Format("2006-01-02 15:04:05"))
1173
-
1174
-
validationIcon := "✓"
1175
-
if !validated {
1176
-
validationIcon = "⚠"
1177
-
}
1178
-
fmt.Printf(" Validated: %s %v\n", validationIcon, validated)
1179
-
1180
-
if count > 0 {
1181
-
if sizeBytes, ok := stats["size_bytes"].(int); ok {
1182
-
fmt.Printf(" Size: %.2f KB\n", float64(sizeBytes)/1024)
1183
-
}
1184
-
1185
-
if firstTime, ok := stats["first_time"].(time.Time); ok {
1186
-
fmt.Printf(" First operation: %s\n", firstTime.Format("2006-01-02 15:04:05"))
1187
-
}
1188
-
1189
-
if lastTime, ok := stats["last_time"].(time.Time); ok {
1190
-
fmt.Printf(" Last operation: %s\n", lastTime.Format("2006-01-02 15:04:05"))
1191
-
}
1192
-
1193
-
progress := float64(count) / float64(types.BUNDLE_SIZE) * 100
1194
-
fmt.Printf(" Progress: %.1f%% (%d/%d)\n", progress, count, types.BUNDLE_SIZE)
1195
-
1196
-
// Show progress bar
1197
-
barWidth := 40
1198
-
filled := int(float64(barWidth) * float64(count) / float64(types.BUNDLE_SIZE))
1199
-
if filled > barWidth {
1200
-
filled = barWidth
1201
-
}
1202
-
bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled)
1203
-
fmt.Printf(" [%s]\n", bar)
1204
-
} else {
1205
-
fmt.Printf(" (empty)\n")
1206
-
}
1207
-
1208
-
// Verbose: Show sample operations
1209
-
if *verbose && count > 0 {
1210
-
fmt.Println()
1211
-
fmt.Printf("Sample operations (showing up to 10):\n")
1212
-
1213
-
ops, err := mgr.GetMempoolOperations()
1214
-
if err != nil {
1215
-
fmt.Fprintf(os.Stderr, "Error getting operations: %v\n", err)
1216
-
os.Exit(1)
1217
-
}
1218
-
1219
-
showCount := 10
1220
-
if len(ops) < showCount {
1221
-
showCount = len(ops)
1222
-
}
1223
-
1224
-
for i := 0; i < showCount; i++ {
1225
-
op := ops[i]
1226
-
fmt.Printf(" %d. DID: %s\n", i+1, op.DID)
1227
-
fmt.Printf(" CID: %s\n", op.CID)
1228
-
fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000"))
1229
-
}
1230
-
1231
-
if len(ops) > showCount {
1232
-
fmt.Printf(" ... and %d more\n", len(ops)-showCount)
1233
-
}
1234
-
}
1235
-
1236
-
fmt.Println()
1237
-
1238
-
// Show mempool file
1239
-
mempoolFilename := fmt.Sprintf("plc_mempool_%06d.jsonl", targetBundle)
1240
-
fmt.Printf("File: %s\n", filepath.Join(dir, mempoolFilename))
1241
-
}
1242
-
1243
-
func cmdServe() {
1244
-
fs := flag.NewFlagSet("serve", flag.ExitOnError)
1245
-
port := fs.String("port", "8080", "HTTP server port")
1246
-
host := fs.String("host", "127.0.0.1", "HTTP server host")
1247
-
sync := fs.Bool("sync", false, "enable sync mode (auto-sync from PLC)")
1248
-
plcURL := fs.String("plc", "https://plc.directory", "PLC directory URL (for sync mode)")
1249
-
syncIntervalFlag := fs.Duration("sync-interval", 1*time.Minute, "sync interval for sync mode")
1250
-
enableWebSocket := fs.Bool("websocket", false, "enable WebSocket endpoint for streaming records")
1251
-
workers := fs.Int("workers", 4, "number of workers for auto-rebuild (0 = CPU count)")
1252
-
verbose := fs.Bool("verbose", false, "verbose sync logging")
1253
-
enableResolver := fs.Bool("resolver", false, "enable DID resolution endpoints (/<did>)")
1254
-
fs.Parse(os.Args[2:])
1255
-
1256
-
serverStartTime = time.Now()
1257
-
syncInterval = *syncIntervalFlag
1258
-
verboseMode = *verbose
1259
-
resolverEnabled = *enableResolver
1260
-
1261
-
// Auto-detect CPU count
1262
-
if *workers == 0 {
1263
-
*workers = runtime.NumCPU()
1264
-
}
1265
-
1266
-
// Create manager with PLC client if sync mode is enabled
1267
-
var plcURLForManager string
1268
-
if *sync {
1269
-
plcURLForManager = *plcURL
1270
-
}
1271
-
1272
-
dir, err := os.Getwd()
1273
-
if err != nil {
1274
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
1275
-
os.Exit(1)
1276
-
}
1277
-
1278
-
if err := os.MkdirAll(dir, 0755); err != nil {
1279
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
1280
-
os.Exit(1)
1281
-
}
1282
-
1283
-
// Create manager config with progress tracking
1284
-
config := bundle.DefaultConfig(dir)
1285
-
config.RebuildWorkers = *workers
1286
-
config.RebuildProgress = func(current, total int) {
1287
-
if current%100 == 0 || current == total {
1288
-
fmt.Printf(" Rebuild progress: %d/%d bundles (%.1f%%) \r",
1289
-
current, total, float64(current)/float64(total)*100)
1290
-
if current == total {
1291
-
fmt.Println()
1292
-
}
1293
-
}
1294
-
}
1295
-
1296
-
var client *plcclient.Client
1297
-
if plcURLForManager != "" {
1298
-
client = plcclient.NewClient(plcURLForManager)
1299
-
}
1300
-
1301
-
fmt.Printf("Starting plcbundle HTTP server...\n")
1302
-
fmt.Printf(" Directory: %s\n", dir)
1303
-
1304
-
// NewManager handles auto-rebuild of bundle index
1305
-
mgr, err := bundle.NewManager(config, client)
1306
-
if err != nil {
1307
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
1308
-
os.Exit(1)
1309
-
}
1310
-
1311
-
if *enableResolver {
1312
-
index := mgr.GetIndex()
1313
-
bundleCount := index.Count()
1314
-
didStats := mgr.GetDIDIndexStats()
1315
-
1316
-
if bundleCount > 0 {
1317
-
needsBuild := false
1318
-
reason := ""
1319
-
1320
-
if !didStats["exists"].(bool) {
1321
-
needsBuild = true
1322
-
reason = "index does not exist"
1323
-
} else {
1324
-
// Check version
1325
-
didIndex := mgr.GetDIDIndex()
1326
-
if didIndex != nil {
1327
-
config := didIndex.GetConfig()
1328
-
if config.Version != didindex.DIDINDEX_VERSION {
1329
-
needsBuild = true
1330
-
reason = fmt.Sprintf("index version outdated (v%d, need v%d)",
1331
-
config.Version, didindex.DIDINDEX_VERSION)
1332
-
} else {
1333
-
// Check if index is behind bundles
1334
-
lastBundle := index.GetLastBundle()
1335
-
if lastBundle != nil && config.LastBundle < lastBundle.BundleNumber {
1336
-
needsBuild = true
1337
-
reason = fmt.Sprintf("index is behind (bundle %d, need %d)",
1338
-
config.LastBundle, lastBundle.BundleNumber)
1339
-
}
1340
-
}
1341
-
}
1342
-
}
1343
-
1344
-
if needsBuild {
1345
-
fmt.Printf(" DID Index: BUILDING (%s)\n", reason)
1346
-
fmt.Printf(" This may take several minutes...\n\n")
1347
-
1348
-
buildStart := time.Now()
1349
-
ctx := context.Background()
1350
-
1351
-
progress := NewProgressBar(bundleCount)
1352
-
err := mgr.BuildDIDIndex(ctx, func(current, total int) {
1353
-
progress.Set(current)
1354
-
})
1355
-
progress.Finish()
1356
-
1357
-
if err != nil {
1358
-
fmt.Fprintf(os.Stderr, "\n⚠️ Warning: Failed to build DID index: %v\n", err)
1359
-
fmt.Fprintf(os.Stderr, " Resolver will use slower fallback mode\n\n")
1360
-
} else {
1361
-
buildTime := time.Since(buildStart)
1362
-
updatedStats := mgr.GetDIDIndexStats()
1363
-
fmt.Printf("\n✓ DID index built in %s\n", buildTime.Round(time.Millisecond))
1364
-
fmt.Printf(" Total DIDs: %s\n\n", formatNumber(int(updatedStats["total_dids"].(int64))))
1365
-
}
1366
-
} else {
1367
-
fmt.Printf(" DID Index: ready (%s DIDs)\n",
1368
-
formatNumber(int(didStats["total_dids"].(int64))))
1369
-
}
1370
-
}
1371
-
1372
-
// ✨ NEW: Verify index consistency on startup
1373
-
if didStats["exists"].(bool) {
1374
-
fmt.Printf(" Verifying index consistency...\n")
1375
-
1376
-
ctx := context.Background()
1377
-
if err := mgr.GetDIDIndex().VerifyAndRepairIndex(ctx, mgr); err != nil {
1378
-
fmt.Fprintf(os.Stderr, "⚠️ Warning: Index verification/repair failed: %v\n", err)
1379
-
fmt.Fprintf(os.Stderr, " Recommend running: plcbundle index build --force\n\n")
1380
-
} else {
1381
-
fmt.Printf(" ✓ Index verified\n")
1382
-
}
1383
-
}
1384
-
}
1385
-
1386
-
addr := fmt.Sprintf("%s:%s", *host, *port)
1387
-
1388
-
fmt.Printf(" Listening: http://%s\n", addr)
1389
-
1390
-
if *sync {
1391
-
fmt.Printf(" Sync mode: ENABLED\n")
1392
-
fmt.Printf(" PLC URL: %s\n", *plcURL)
1393
-
fmt.Printf(" Sync interval: %s\n", syncInterval)
1394
-
} else {
1395
-
fmt.Printf(" Sync mode: disabled\n")
1396
-
}
1397
-
1398
-
if *enableWebSocket {
1399
-
wsScheme := "ws"
1400
-
fmt.Printf(" WebSocket: ENABLED (%s://%s/ws)\n", wsScheme, addr)
1401
-
} else {
1402
-
fmt.Printf(" WebSocket: disabled (use --websocket to enable)\n")
1403
-
}
1404
-
1405
-
if *enableResolver {
1406
-
fmt.Printf(" Resolver: ENABLED (/<did> endpoints)\n")
1407
-
} else {
1408
-
fmt.Printf(" Resolver: disabled (use --resolver to enable)\n")
1409
-
}
1410
-
1411
-
bundleCount := mgr.GetIndex().Count()
1412
-
if bundleCount > 0 {
1413
-
fmt.Printf(" Bundles available: %d\n", bundleCount)
1414
-
} else {
1415
-
fmt.Printf(" Bundles available: 0\n")
1416
-
}
1417
-
1418
-
fmt.Printf("\nPress Ctrl+C to stop\n\n")
1419
-
1420
-
ctx, cancel := context.WithCancel(context.Background())
1421
-
1422
-
// ✨ NEW: Graceful shutdown handler
1423
-
sigChan := make(chan os.Signal, 1)
1424
-
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
1425
-
1426
-
go func() {
1427
-
<-sigChan
1428
-
fmt.Fprintf(os.Stderr, "\n\n⚠️ Shutdown signal received...\n")
1429
-
fmt.Fprintf(os.Stderr, " Saving mempool...\n")
1430
-
1431
-
if err := mgr.SaveMempool(); err != nil {
1432
-
fmt.Fprintf(os.Stderr, " ✗ Failed to save mempool: %v\n", err)
1433
-
} else {
1434
-
fmt.Fprintf(os.Stderr, " ✓ Mempool saved\n")
1435
-
}
1436
-
1437
-
fmt.Fprintf(os.Stderr, " Closing DID index...\n")
1438
-
if err := mgr.GetDIDIndex().Close(); err != nil {
1439
-
fmt.Fprintf(os.Stderr, " ✗ Failed to close index: %v\n", err)
1440
-
} else {
1441
-
fmt.Fprintf(os.Stderr, " ✓ Index closed\n")
1442
-
}
1443
-
1444
-
fmt.Fprintf(os.Stderr, " ✓ Shutdown complete\n")
1445
-
1446
-
cancel()
1447
-
os.Exit(0)
1448
-
}()
1449
-
1450
-
if *sync {
1451
-
go runSync(ctx, mgr, syncInterval, *verbose, *enableResolver)
1452
-
}
1453
-
1454
-
handler := newServerHandler(mgr, *sync, *enableWebSocket, *enableResolver)
1455
-
server := &http.Server{
1456
-
Addr: addr,
1457
-
Handler: handler,
1458
-
}
1459
-
1460
-
if err := server.ListenAndServe(); err != nil {
1461
-
fmt.Fprintf(os.Stderr, "Server error: %v\n", err)
1462
-
mgr.SaveMempool()
1463
-
mgr.Close()
1464
-
os.Exit(1)
1465
-
}
1466
-
}
1467
-
1468
-
func cmdCompare() {
1469
-
fs := flag.NewFlagSet("compare", flag.ExitOnError)
1470
-
verbose := fs.Bool("v", false, "verbose output (show all differences)")
1471
-
fetchMissing := fs.Bool("fetch-missing", false, "fetch missing bundles from target")
1472
-
fs.Parse(os.Args[2:])
1473
-
1474
-
if fs.NArg() < 1 {
1475
-
fmt.Fprintf(os.Stderr, "Usage: plcbundle compare <target> [options]\n")
1476
-
fmt.Fprintf(os.Stderr, " target: URL or path to remote plcbundle server/index\n")
1477
-
fmt.Fprintf(os.Stderr, "\nExamples:\n")
1478
-
fmt.Fprintf(os.Stderr, " plcbundle compare https://plc.example.com\n")
1479
-
fmt.Fprintf(os.Stderr, " plcbundle compare https://plc.example.com/index.json\n")
1480
-
fmt.Fprintf(os.Stderr, " plcbundle compare /path/to/plc_bundles.json\n")
1481
-
fmt.Fprintf(os.Stderr, " plcbundle compare https://plc.example.com --fetch-missing\n")
1482
-
fmt.Fprintf(os.Stderr, "\nOptions:\n")
1483
-
fs.PrintDefaults()
1484
-
os.Exit(1)
1485
-
}
1486
-
1487
-
target := fs.Arg(0)
1488
-
1489
-
mgr, dir, err := getManager("")
1490
-
if err != nil {
1491
-
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
1492
-
os.Exit(1)
1493
-
}
1494
-
defer mgr.Close()
1495
-
1496
-
fmt.Printf("Comparing: %s\n", dir)
1497
-
fmt.Printf(" Against: %s\n\n", target)
1498
-
1499
-
// Load local index
1500
-
localIndex := mgr.GetIndex()
1501
-
1502
-
// Load target index
1503
-
fmt.Printf("Loading target index...\n")
1504
-
targetIndex, err := loadTargetIndex(target)
1505
-
if err != nil {
1506
-
fmt.Fprintf(os.Stderr, "Error loading target index: %v\n", err)
1507
-
os.Exit(1)
1508
-
}
1509
-
1510
-
// Perform comparison
1511
-
comparison := compareIndexes(localIndex, targetIndex)
1512
-
1513
-
// Display results
1514
-
displayComparison(comparison, *verbose)
1515
-
1516
-
// Fetch missing bundles if requested
1517
-
if *fetchMissing && len(comparison.MissingBundles) > 0 {
1518
-
fmt.Printf("\n")
1519
-
if !strings.HasPrefix(target, "http://") && !strings.HasPrefix(target, "https://") {
1520
-
fmt.Fprintf(os.Stderr, "Error: --fetch-missing only works with remote URLs\n")
1521
-
os.Exit(1)
1522
-
}
1523
-
1524
-
baseURL := strings.TrimSuffix(target, "/index.json")
1525
-
baseURL = strings.TrimSuffix(baseURL, "/plc_bundles.json")
1526
-
1527
-
fmt.Printf("Fetching %d missing bundles...\n\n", len(comparison.MissingBundles))
1528
-
fetchMissingBundles(mgr, baseURL, comparison.MissingBundles)
1529
-
}
1530
-
1531
-
// Exit with error if there are differences
1532
-
if comparison.HasDifferences() {
1533
-
os.Exit(1)
1534
-
}
1535
}
···
1
package main
2
3
import (
4
"fmt"
5
"os"
6
"runtime/debug"
7
8
+
"tangled.org/atscan.net/plcbundle/cmd/plcbundle/commands"
9
)
10
11
func main() {
12
debug.SetGCPercent(400)
13
14
if len(os.Args) < 2 {
···
18
19
command := os.Args[1]
20
21
+
var err error
22
switch command {
23
case "fetch":
24
+
err = commands.FetchCommand(os.Args[2:])
25
case "clone":
26
+
err = commands.CloneCommand(os.Args[2:])
27
case "rebuild":
28
+
err = commands.RebuildCommand(os.Args[2:])
29
case "verify":
30
+
err = commands.VerifyCommand(os.Args[2:])
31
case "info":
32
+
err = commands.InfoCommand(os.Args[2:])
33
case "export":
34
+
err = commands.ExportCommand(os.Args[2:])
35
case "backfill":
36
+
err = commands.BackfillCommand(os.Args[2:])
37
case "mempool":
38
+
err = commands.MempoolCommand(os.Args[2:])
39
case "serve":
40
+
err = commands.ServerCommand(os.Args[2:])
41
case "compare":
42
+
err = commands.CompareCommand(os.Args[2:])
43
case "detector":
44
+
err = commands.DetectorCommand(os.Args[2:])
45
case "index":
46
+
err = commands.IndexCommand(os.Args[2:])
47
case "get-op":
48
+
err = commands.GetOpCommand(os.Args[2:])
49
case "version":
50
+
err = commands.VersionCommand(os.Args[2:])
51
default:
52
fmt.Fprintf(os.Stderr, "Unknown command: %s\n", command)
53
printUsage()
54
os.Exit(1)
55
}
56
+
57
+
if err != nil {
58
+
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
59
+
os.Exit(1)
60
+
}
61
}
62
63
func printUsage() {
···
68
69
Commands:
70
fetch Fetch next bundle from PLC directory
71
+
clone Clone bundles from remote HTTP endpoint
72
rebuild Rebuild index from existing bundle files
73
verify Verify bundle integrity
74
info Show bundle information
···
79
compare Compare local index with target index
80
detector Run spam detectors
81
index Manage DID position index
82
+
get-op Get specific operation by bundle and position
83
version Show version
84
85
+
Examples:
86
+
plcbundle fetch
87
+
plcbundle clone https://plc.example.com
88
+
plcbundle info --bundles
89
+
plcbundle serve --sync --websocket
90
+
plcbundle detector run invalid_handle --bundles 1-100
91
92
+
`, commands.GetVersion())
93
}
-154
cmd/plcbundle/progress.go
-154
cmd/plcbundle/progress.go
···
1
-
package main
2
-
3
-
import (
4
-
"fmt"
5
-
"os"
6
-
"strings"
7
-
"sync"
8
-
"time"
9
-
)
10
-
11
-
// ProgressBar shows progress of an operation
12
-
type ProgressBar struct {
13
-
total int
14
-
current int
15
-
totalBytes int64
16
-
currentBytes int64
17
-
startTime time.Time
18
-
mu sync.Mutex
19
-
width int
20
-
lastPrint time.Time
21
-
showBytes bool
22
-
}
23
-
24
-
// NewProgressBar creates a new progress bar
25
-
func NewProgressBar(total int) *ProgressBar {
26
-
return &ProgressBar{
27
-
total: total,
28
-
current: 0,
29
-
totalBytes: 0,
30
-
currentBytes: 0,
31
-
startTime: time.Now(),
32
-
width: 40,
33
-
lastPrint: time.Now(),
34
-
showBytes: false,
35
-
}
36
-
}
37
-
38
-
// NewProgressBarWithBytes creates a new progress bar that tracks bytes
39
-
func NewProgressBarWithBytes(total int, totalBytes int64) *ProgressBar {
40
-
return &ProgressBar{
41
-
total: total,
42
-
current: 0,
43
-
totalBytes: totalBytes,
44
-
currentBytes: 0,
45
-
startTime: time.Now(),
46
-
width: 40,
47
-
lastPrint: time.Now(),
48
-
showBytes: true,
49
-
}
50
-
}
51
-
52
-
// Increment increases the progress by 1
53
-
func (pb *ProgressBar) Increment() {
54
-
pb.mu.Lock()
55
-
defer pb.mu.Unlock()
56
-
pb.current++
57
-
pb.print()
58
-
}
59
-
60
-
// Set sets the current progress
61
-
func (pb *ProgressBar) Set(current int) {
62
-
pb.mu.Lock()
63
-
defer pb.mu.Unlock()
64
-
pb.current = current
65
-
pb.print()
66
-
}
67
-
68
-
// SetWithBytes sets the current progress and bytes processed
69
-
func (pb *ProgressBar) SetWithBytes(current int, bytesProcessed int64) {
70
-
pb.mu.Lock()
71
-
defer pb.mu.Unlock()
72
-
pb.current = current
73
-
pb.currentBytes = bytesProcessed
74
-
pb.print()
75
-
}
76
-
77
-
// Finish completes the progress bar
78
-
func (pb *ProgressBar) Finish() {
79
-
pb.mu.Lock()
80
-
defer pb.mu.Unlock()
81
-
pb.current = pb.total
82
-
pb.currentBytes = pb.totalBytes
83
-
pb.print()
84
-
fmt.Fprintf(os.Stderr, "\n")
85
-
}
86
-
87
-
// print renders the progress bar (must be called with lock held)
88
-
func (pb *ProgressBar) print() {
89
-
// Rate limit updates (max 10 per second)
90
-
if time.Since(pb.lastPrint) < 100*time.Millisecond && pb.current < pb.total {
91
-
return
92
-
}
93
-
pb.lastPrint = time.Now()
94
-
95
-
// Calculate percentage
96
-
percent := float64(pb.current) / float64(pb.total) * 100
97
-
if pb.total == 0 {
98
-
percent = 0
99
-
}
100
-
101
-
// Calculate bar
102
-
filled := int(float64(pb.width) * float64(pb.current) / float64(pb.total))
103
-
if filled > pb.width {
104
-
filled = pb.width
105
-
}
106
-
bar := strings.Repeat("█", filled) + strings.Repeat("░", pb.width-filled)
107
-
108
-
// Calculate speed and ETA
109
-
elapsed := time.Since(pb.startTime)
110
-
speed := float64(pb.current) / elapsed.Seconds()
111
-
remaining := pb.total - pb.current
112
-
var eta time.Duration
113
-
if speed > 0 {
114
-
eta = time.Duration(float64(remaining)/speed) * time.Second
115
-
}
116
-
117
-
// Show MB/s if bytes are being tracked (changed condition)
118
-
if pb.showBytes && pb.currentBytes > 0 {
119
-
// Calculate MB/s (using decimal units: 1 MB = 1,000,000 bytes)
120
-
mbProcessed := float64(pb.currentBytes) / (1000 * 1000)
121
-
mbPerSec := mbProcessed / elapsed.Seconds()
122
-
123
-
fmt.Fprintf(os.Stderr, "\r [%s] %6.2f%% | %d/%d bundles | %.1f/s | %.1f MB/s | ETA: %s ",
124
-
bar,
125
-
percent,
126
-
pb.current,
127
-
pb.total,
128
-
speed,
129
-
mbPerSec,
130
-
formatETA(eta))
131
-
} else {
132
-
fmt.Fprintf(os.Stderr, "\r [%s] %6.2f%% | %d/%d bundles | %.1f/s | ETA: %s ",
133
-
bar,
134
-
percent,
135
-
pb.current,
136
-
pb.total,
137
-
speed,
138
-
formatETA(eta))
139
-
}
140
-
}
141
-
142
-
// formatETA formats the ETA duration
143
-
func formatETA(d time.Duration) string {
144
-
if d == 0 {
145
-
return "calculating..."
146
-
}
147
-
if d < time.Minute {
148
-
return fmt.Sprintf("%ds", int(d.Seconds()))
149
-
}
150
-
if d < time.Hour {
151
-
return fmt.Sprintf("%dm %ds", int(d.Minutes()), int(d.Seconds())%60)
152
-
}
153
-
return fmt.Sprintf("%dh %dm", int(d.Hours()), int(d.Minutes())%60)
154
-
}
···
-1165
cmd/plcbundle/server.go
-1165
cmd/plcbundle/server.go
···
1
-
package main
2
-
3
-
import (
4
-
"bufio"
5
-
"context"
6
-
"fmt"
7
-
"io"
8
-
"net/http"
9
-
"os"
10
-
"runtime"
11
-
"strconv"
12
-
"strings"
13
-
"time"
14
-
15
-
"github.com/goccy/go-json"
16
-
"github.com/gorilla/websocket"
17
-
18
-
"tangled.org/atscan.net/plcbundle/internal/bundle"
19
-
"tangled.org/atscan.net/plcbundle/internal/types"
20
-
"tangled.org/atscan.net/plcbundle/plcclient"
21
-
)
22
-
23
-
var upgrader = websocket.Upgrader{
24
-
ReadBufferSize: 1024,
25
-
WriteBufferSize: 1024,
26
-
CheckOrigin: func(r *http.Request) bool {
27
-
return true
28
-
},
29
-
}
30
-
31
-
var serverStartTime time.Time
32
-
var syncInterval time.Duration
33
-
var verboseMode bool
34
-
var resolverEnabled bool
35
-
36
-
// newServerHandler creates HTTP handler with all routes
37
-
func newServerHandler(mgr *bundle.Manager, syncMode bool, wsEnabled bool, resolverEnabled bool) http.Handler {
38
-
mux := http.NewServeMux()
39
-
40
-
// Specific routes first (highest priority)
41
-
mux.HandleFunc("GET /index.json", handleIndexJSONNative(mgr))
42
-
mux.HandleFunc("GET /bundle/{number}", handleBundleNative(mgr))
43
-
mux.HandleFunc("GET /data/{number}", handleBundleDataNative(mgr))
44
-
mux.HandleFunc("GET /jsonl/{number}", handleBundleJSONLNative(mgr))
45
-
mux.HandleFunc("GET /status", handleStatusNative(mgr, syncMode, wsEnabled))
46
-
mux.HandleFunc("GET /debug/memory", handleDebugMemoryNative(mgr))
47
-
48
-
// WebSocket endpoint
49
-
if wsEnabled {
50
-
mux.HandleFunc("GET /ws", handleWebSocketNative(mgr))
51
-
}
52
-
53
-
// Sync mode endpoints
54
-
if syncMode {
55
-
mux.HandleFunc("GET /mempool", handleMempoolNative(mgr))
56
-
}
57
-
58
-
// Combined root and DID resolver handler
59
-
mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
60
-
path := r.URL.Path
61
-
62
-
// Handle exact root
63
-
if path == "/" {
64
-
handleRootNative(mgr, syncMode, wsEnabled, resolverEnabled)(w, r)
65
-
return
66
-
}
67
-
68
-
// Handle DID routes if enabled
69
-
if resolverEnabled {
70
-
handleDIDRouting(w, r, mgr)
71
-
return
72
-
}
73
-
74
-
// 404 for everything else
75
-
sendJSON(w, 404, map[string]string{"error": "not found"})
76
-
})
77
-
78
-
// Wrap with CORS middleware
79
-
return corsMiddleware(mux)
80
-
}
81
-
82
-
// handleDIDRouting routes DID-related requests
83
-
func handleDIDRouting(w http.ResponseWriter, r *http.Request, mgr *bundle.Manager) {
84
-
path := strings.TrimPrefix(r.URL.Path, "/")
85
-
86
-
// Parse DID and sub-path
87
-
parts := strings.SplitN(path, "/", 2)
88
-
did := parts[0]
89
-
90
-
// Validate it's a DID
91
-
if !strings.HasPrefix(did, "did:plc:") {
92
-
sendJSON(w, 404, map[string]string{"error": "not found"})
93
-
return
94
-
}
95
-
96
-
// Route based on sub-path
97
-
if len(parts) == 1 {
98
-
// /did:plc:xxx -> DID document
99
-
handleDIDDocumentLatestNative(mgr, did)(w, r)
100
-
} else if parts[1] == "data" {
101
-
// /did:plc:xxx/data -> PLC state
102
-
handleDIDDataNative(mgr, did)(w, r)
103
-
} else if parts[1] == "log/audit" {
104
-
// /did:plc:xxx/log/audit -> Audit log
105
-
handleDIDAuditLogNative(mgr, did)(w, r)
106
-
} else {
107
-
sendJSON(w, 404, map[string]string{"error": "not found"})
108
-
}
109
-
}
110
-
111
-
// corsMiddleware adds CORS headers (skips WebSocket upgrade requests)
112
-
func corsMiddleware(next http.Handler) http.Handler {
113
-
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
114
-
// Check if this is a WebSocket upgrade request
115
-
if r.Header.Get("Upgrade") == "websocket" {
116
-
// Skip CORS for WebSocket - pass through directly
117
-
next.ServeHTTP(w, r)
118
-
return
119
-
}
120
-
121
-
// Normal CORS handling for non-WebSocket requests
122
-
w.Header().Set("Access-Control-Allow-Origin", "*")
123
-
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
124
-
125
-
if requestedHeaders := r.Header.Get("Access-Control-Request-Headers"); requestedHeaders != "" {
126
-
w.Header().Set("Access-Control-Allow-Headers", requestedHeaders)
127
-
} else {
128
-
w.Header().Set("Access-Control-Allow-Headers", "*")
129
-
}
130
-
131
-
w.Header().Set("Access-Control-Max-Age", "86400")
132
-
133
-
if r.Method == "OPTIONS" {
134
-
w.WriteHeader(204)
135
-
return
136
-
}
137
-
138
-
next.ServeHTTP(w, r)
139
-
})
140
-
}
141
-
142
-
// sendJSON sends JSON response
143
-
func sendJSON(w http.ResponseWriter, statusCode int, data interface{}) {
144
-
w.Header().Set("Content-Type", "application/json")
145
-
146
-
jsonData, err := json.Marshal(data)
147
-
if err != nil {
148
-
w.WriteHeader(500)
149
-
w.Write([]byte(`{"error":"failed to marshal JSON"}`))
150
-
return
151
-
}
152
-
153
-
w.WriteHeader(statusCode)
154
-
w.Write(jsonData)
155
-
}
156
-
157
-
// Handler implementations
158
-
159
-
func handleRootNative(mgr *bundle.Manager, syncMode bool, wsEnabled bool, resolverEnabled bool) http.HandlerFunc {
160
-
return func(w http.ResponseWriter, r *http.Request) {
161
-
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
162
-
163
-
index := mgr.GetIndex()
164
-
stats := index.GetStats()
165
-
bundleCount := stats["bundle_count"].(int)
166
-
167
-
baseURL := getBaseURL(r)
168
-
wsURL := getWSURL(r)
169
-
170
-
var sb strings.Builder
171
-
172
-
sb.WriteString(`
173
-
174
-
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠄⠀⡀⠀⠀⠀⠀⠀⠀⢀⠀⠀⡀⠀⢀⠀⢀⡀⣤⡢⣤⡤⡀⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
175
-
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⡄⡄⠐⡀⠈⣀⠀⡠⡠⠀⣢⣆⢌⡾⢙⠺⢽⠾⡋⣻⡷⡫⢵⣭⢦⣴⠦⠀⢠⠀⠀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
176
-
⠀⠀⠀⠀⠀⠀⠀⠀⢠⣤⣽⣥⡈⠧⣂⢧⢾⠕⠞⠡⠊⠁⣐⠉⠀⠉⢍⠀⠉⠌⡉⠀⠂⠁⠱⠉⠁⢝⠻⠎⣬⢌⡌⣬⣡⣀⣢⣄⡄⠀⡀⠀⠀⠀⠀⠀⠀⠀⠀
177
-
⠀⠀⠀⠀⠀⠀⠀⢀⢸⣿⣿⢿⣾⣯⣑⢄⡂⠀⠄⠂⠀⠀⢀⠀⠀⠐⠀⠀⠀⠀⠀⠀⠀⠀⠄⠐⠀⠀⠀⠀⣄⠭⠂⠈⠜⣩⣿⢝⠃⠀⠁⠀⠀⠀⠀⠀⠀⠀⠀
178
-
⠀⠀⠀⠀⠀⠀⠀⢀⣻⡟⠏⠀⠚⠈⠚⡉⡝⢶⣱⢤⣅⠈⠀⠄⠀⠀⠀⠀⠀⠠⠀⠀⡂⠐⣤⢕⡪⢼⣈⡹⡇⠏⠏⠋⠅⢃⣪⡏⡇⡍⠀⠀⠀⠀⠀⠀⠀⠀⠀
179
-
⠀⠀⠀⠀⠀⠀⠀⠀⠺⣻⡄⠀⠀⠀⢠⠌⠃⠐⠉⢡⠱⠧⠝⡯⣮⢶⣴⣤⡆⢐⣣⢅⣮⡟⠦⠍⠉⠀⠁⠐⠀⠀⠀⠄⠐⠡⣽⡸⣎⢁⠀⠀⠀⠀⠀⠀⠀⠀⠀
180
-
⠀⠀⠀⠀⠀⠀⠀⢈⡻⣧⠀⠁⠐⠀⠀⠀⠀⠀⠀⠊⠀⠕⢀⡉⠈⡫⠽⡿⡟⠿⠟⠁⠀⠀⠄⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠬⠥⣋⡯⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
181
-
⠀⠀⠀⠀⠀⠀⠀⡀⣾⡍⠕⡀⠀⠀⠀⠄⠠⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠥⣤⢌⠀⠀⠀⠀⠀⠀⠀⠀⠀⠁⠀⠀⠄⢀⠀⢝⢞⣫⡆⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀
182
-
⠀⠀⠀⠀⠀⠀⠀⠀⣽⡶⡄⠐⡀⠀⠀⠀⠀⠀⠀⢀⠀⠄⠀⠀⠀⠄⠁⠇⣷⡆⠀⠀⠀⢀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⡸⢝⣮⠍⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
183
-
⠀⠀⠀⠀⠀⠀⢀⠀⢾⣷⠀⠠⡀⠀⠀⠀⠀⢀⠀⠀⠀⠀⠀⠁⡁⠀⠀⣾⡥⠖⠀⠀⠀⠂⠀⠀⠀⠀⠀⠁⠀⡀⠁⠀⠀⠻⢳⣻⢄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
184
-
⠀⠀⠀⠀⠀⠀⠀⠀⣞⡙⠨⣀⠠⠄⠀⠂⠀⠀⠀⠈⢀⠀⠀⠀⠀⠀⠤⢚⢢⣟⠀⠀⠀⠀⡐⠀⠀⡀⠀⠀⠀⠀⠁⠈⠌⠊⣯⣮⡏⠡⠂⠀⠀⠀⠀⠀⠀⠀⠀
185
-
⠀⠀⠀⠀⠀⠀⠀⠀⣻⡟⡄⡡⣄⠀⠠⠀⠀⡅⠀⠐⠀⡀⠀⡀⠀⠄⠈⠃⠳⠪⠤⠀⠀⠀⠀⡀⠀⠂⠀⠀⠀⠁⠈⢠⣠⠒⠻⣻⡧⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
186
-
⠀⠀⠀⠀⠀⠀⠀⠀⠪⡎⠠⢌⠑⡀⠂⠀⠄⠠⠀⠠⠀⠁⡀⠠⠠⡀⣀⠜⢏⡅⠀⠀⡀⠁⠀⠀⠁⠁⠐⠄⡀⢀⠂⠀⠄⢑⣿⣿⣿⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀
187
-
⠀⠀⠀⠀⠀⠀⠀⠀⠼⣻⠧⣣⣀⠐⠨⠁⠕⢈⢀⢀⡁⠀⠈⠠⢀⠀⠐⠜⣽⡗⡤⠀⠂⠀⠠⠀⢂⠠⠀⠁⠄⠀⠔⠀⠑⣨⣿⢯⠋⡅⠀⠀⠀⠀⠀⠀⠀⠀⠀
188
-
⠀⠀⠀⠀⠀⠀⠀⠀⡚⣷⣭⠎⢃⡗⠄⡄⢀⠁⠀⠅⢀⢅⡀⠠⠀⢠⡀⡩⠷⢇⠀⡀⠄⡠⠤⠆⣀⡀⠄⠉⣠⠃⠴⠀⠈⢁⣿⡛⡯⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
189
-
⠀⠀⠀⠀⠀⠀⠀⠘⡬⡿⣿⡏⡻⡯⠌⢁⢛⠠⠓⠐⠐⠐⠌⠃⠋⠂⡢⢰⣈⢏⣰⠂⠈⠀⠠⠒⠡⠌⠫⠭⠩⠢⡬⠆⠿⢷⢿⡽⡧⠉⠊⠀⠀⠀⠀⠀⠀⠀⠀
190
-
⠀⠀⠀⠀⠀⠀⠀⠀⠺⣷⣺⣗⣿⡶⡎⡅⣣⢎⠠⡅⣢⡖⠴⠬⡈⠂⡨⢡⠾⣣⣢⠀⠀⡹⠄⡄⠄⡇⣰⡖⡊⠔⢹⣄⣿⣭⣵⣿⢷⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀
191
-
⠀⠀⠀⠀⠀⠀⠀⠀⠩⣿⣿⣲⣿⣷⣟⣼⠟⣬⢉⡠⣪⢜⣂⣁⠥⠓⠚⡁⢶⣷⣠⠂⡄⡢⣀⡐⠧⢆⣒⡲⡳⡫⢟⡃⢪⡧⣟⡟⣯⠐⠀⠀⠀⠀⠀⠀⠀⠀⠀
192
-
⠀⠀⠀⠀⠀⠀⠀⠀⢺⠟⢿⢟⢻⡗⡮⡿⣲⢷⣆⣏⣇⡧⣄⢖⠾⡷⣿⣤⢳⢷⣣⣦⡜⠗⣭⢂⠩⣹⢿⡲⢎⡧⣕⣖⣓⣽⡿⡖⡿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
193
-
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠉⠂⠂⠏⠿⢻⣥⡪⢽⣳⣳⣥⡶⣫⣍⢐⣥⣻⣾⡻⣅⢭⡴⢭⣿⠕⣧⡭⣞⣻⣣⣻⢿⠟⠛⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
194
-
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠄⠋⠫⠯⣍⢻⣿⣿⣷⣕⣵⣹⣽⣿⣷⣇⡏⣿⡿⣍⡝⠵⠯⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
195
-
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠐⠠⠁⠋⢣⠓⡍⣫⠹⣿⣿⣷⡿⠯⠺⠁⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
196
-
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠋⢀⠋⢈⡿⠿⠁⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
197
-
198
-
plcbundle server
199
-
200
-
*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*
201
-
| ⚠️ Preview Version – Do Not Use In Production! |
202
-
*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*
203
-
| This project and plcbundle specification is currently |
204
-
| unstable and under heavy development. Things can break at |
205
-
| any time. Do not use this for production systems. |
206
-
| Please wait for the 1.0 release. |
207
-
|________________________________________________________________|
208
-
209
-
`)
210
-
211
-
sb.WriteString("\nplcbundle server\n\n")
212
-
sb.WriteString("What is PLC Bundle?\n")
213
-
sb.WriteString("━━━━━━━━━━━━━━━━━━━━\n")
214
-
sb.WriteString("plcbundle archives AT Protocol's DID PLC Directory operations into\n")
215
-
sb.WriteString("immutable, cryptographically-chained bundles of 10,000 operations.\n\n")
216
-
sb.WriteString("More info: https://tangled.org/@atscan.net/plcbundle\n\n")
217
-
218
-
if bundleCount > 0 {
219
-
sb.WriteString("Bundles\n")
220
-
sb.WriteString("━━━━━━━\n")
221
-
sb.WriteString(fmt.Sprintf(" Bundle count: %d\n", bundleCount))
222
-
223
-
firstBundle := stats["first_bundle"].(int)
224
-
lastBundle := stats["last_bundle"].(int)
225
-
totalSize := stats["total_size"].(int64)
226
-
totalUncompressed := stats["total_uncompressed_size"].(int64)
227
-
228
-
sb.WriteString(fmt.Sprintf(" Last bundle: %d (%s)\n", lastBundle,
229
-
stats["updated_at"].(time.Time).Format("2006-01-02 15:04:05")))
230
-
sb.WriteString(fmt.Sprintf(" Range: %06d - %06d\n", firstBundle, lastBundle))
231
-
sb.WriteString(fmt.Sprintf(" Total size: %.2f MB\n", float64(totalSize)/(1000*1000)))
232
-
sb.WriteString(fmt.Sprintf(" Uncompressed: %.2f MB (%.2fx)\n",
233
-
float64(totalUncompressed)/(1000*1000),
234
-
float64(totalUncompressed)/float64(totalSize)))
235
-
236
-
if gaps, ok := stats["gaps"].(int); ok && gaps > 0 {
237
-
sb.WriteString(fmt.Sprintf(" ⚠ Gaps: %d missing bundles\n", gaps))
238
-
}
239
-
240
-
firstMeta, err := index.GetBundle(firstBundle)
241
-
if err == nil {
242
-
sb.WriteString(fmt.Sprintf("\n Root: %s\n", firstMeta.Hash))
243
-
}
244
-
245
-
lastMeta, err := index.GetBundle(lastBundle)
246
-
if err == nil {
247
-
sb.WriteString(fmt.Sprintf(" Head: %s\n", lastMeta.Hash))
248
-
}
249
-
}
250
-
251
-
if syncMode {
252
-
mempoolStats := mgr.GetMempoolStats()
253
-
count := mempoolStats["count"].(int)
254
-
targetBundle := mempoolStats["target_bundle"].(int)
255
-
canCreate := mempoolStats["can_create_bundle"].(bool)
256
-
257
-
sb.WriteString("\nMempool Stats\n")
258
-
sb.WriteString("━━━━━━━━━━━━━\n")
259
-
sb.WriteString(fmt.Sprintf(" Target bundle: %d\n", targetBundle))
260
-
sb.WriteString(fmt.Sprintf(" Operations: %d / %d\n", count, types.BUNDLE_SIZE))
261
-
sb.WriteString(fmt.Sprintf(" Can create bundle: %v\n", canCreate))
262
-
263
-
if count > 0 {
264
-
progress := float64(count) / float64(types.BUNDLE_SIZE) * 100
265
-
sb.WriteString(fmt.Sprintf(" Progress: %.1f%%\n", progress))
266
-
267
-
barWidth := 50
268
-
filled := int(float64(barWidth) * float64(count) / float64(types.BUNDLE_SIZE))
269
-
if filled > barWidth {
270
-
filled = barWidth
271
-
}
272
-
bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled)
273
-
sb.WriteString(fmt.Sprintf(" [%s]\n", bar))
274
-
275
-
if firstTime, ok := mempoolStats["first_time"].(time.Time); ok {
276
-
sb.WriteString(fmt.Sprintf(" First op: %s\n", firstTime.Format("2006-01-02 15:04:05")))
277
-
}
278
-
if lastTime, ok := mempoolStats["last_time"].(time.Time); ok {
279
-
sb.WriteString(fmt.Sprintf(" Last op: %s\n", lastTime.Format("2006-01-02 15:04:05")))
280
-
}
281
-
} else {
282
-
sb.WriteString(" (empty)\n")
283
-
}
284
-
}
285
-
286
-
if didStats := mgr.GetDIDIndexStats(); didStats["exists"].(bool) {
287
-
sb.WriteString("\nDID Index\n")
288
-
sb.WriteString("━━━━━━━━━\n")
289
-
sb.WriteString(" Status: enabled\n")
290
-
291
-
indexedDIDs := didStats["indexed_dids"].(int64)
292
-
mempoolDIDs := didStats["mempool_dids"].(int64)
293
-
totalDIDs := didStats["total_dids"].(int64)
294
-
295
-
if mempoolDIDs > 0 {
296
-
sb.WriteString(fmt.Sprintf(" Total DIDs: %s (%s indexed + %s mempool)\n",
297
-
formatNumber(int(totalDIDs)),
298
-
formatNumber(int(indexedDIDs)),
299
-
formatNumber(int(mempoolDIDs))))
300
-
} else {
301
-
sb.WriteString(fmt.Sprintf(" Total DIDs: %s\n", formatNumber(int(totalDIDs))))
302
-
}
303
-
304
-
sb.WriteString(fmt.Sprintf(" Cached shards: %d / %d\n",
305
-
didStats["cached_shards"], didStats["cache_limit"]))
306
-
sb.WriteString("\n")
307
-
}
308
-
309
-
sb.WriteString("Server Stats\n")
310
-
sb.WriteString("━━━━━━━━━━━━\n")
311
-
sb.WriteString(fmt.Sprintf(" Version: %s\n", version))
312
-
if origin := mgr.GetPLCOrigin(); origin != "" {
313
-
sb.WriteString(fmt.Sprintf(" Origin: %s\n", origin))
314
-
}
315
-
sb.WriteString(fmt.Sprintf(" Sync mode: %v\n", syncMode))
316
-
sb.WriteString(fmt.Sprintf(" WebSocket: %v\n", wsEnabled))
317
-
sb.WriteString(fmt.Sprintf(" Resolver: %v\n", resolverEnabled))
318
-
sb.WriteString(fmt.Sprintf(" Uptime: %s\n", time.Since(serverStartTime).Round(time.Second)))
319
-
320
-
sb.WriteString("\n\nAPI Endpoints\n")
321
-
sb.WriteString("━━━━━━━━━━━━━\n")
322
-
sb.WriteString(" GET / This info page\n")
323
-
sb.WriteString(" GET /index.json Full bundle index\n")
324
-
sb.WriteString(" GET /bundle/:number Bundle metadata (JSON)\n")
325
-
sb.WriteString(" GET /data/:number Raw bundle (zstd compressed)\n")
326
-
sb.WriteString(" GET /jsonl/:number Decompressed JSONL stream\n")
327
-
sb.WriteString(" GET /status Server status\n")
328
-
sb.WriteString(" GET /mempool Mempool operations (JSONL)\n")
329
-
330
-
if resolverEnabled {
331
-
sb.WriteString("\nDID Resolution\n")
332
-
sb.WriteString("━━━━━━━━━━━━━━\n")
333
-
sb.WriteString(" GET /:did DID Document (W3C format)\n")
334
-
sb.WriteString(" GET /:did/data PLC State (raw format)\n")
335
-
sb.WriteString(" GET /:did/log/audit Operation history\n")
336
-
337
-
didStats := mgr.GetDIDIndexStats()
338
-
if didStats["exists"].(bool) {
339
-
sb.WriteString(fmt.Sprintf("\n Index: %s DIDs indexed\n",
340
-
formatNumber(int(didStats["total_dids"].(int64)))))
341
-
} else {
342
-
sb.WriteString("\n ⚠️ Index: not built (will use slow scan)\n")
343
-
}
344
-
sb.WriteString("\n")
345
-
}
346
-
347
-
if wsEnabled {
348
-
sb.WriteString("\nWebSocket Endpoints\n")
349
-
sb.WriteString("━━━━━━━━━━━━━━━━━━━\n")
350
-
sb.WriteString(" WS /ws Live stream (new operations only)\n")
351
-
sb.WriteString(" WS /ws?cursor=0 Stream all from beginning\n")
352
-
sb.WriteString(" WS /ws?cursor=N Stream from cursor N\n\n")
353
-
sb.WriteString("Cursor Format:\n")
354
-
sb.WriteString(" Global record number: (bundleNumber × 10,000) + position\n")
355
-
sb.WriteString(" Example: 88410345 = bundle 8841, position 345\n")
356
-
sb.WriteString(" Default: starts from latest (skips all historical data)\n")
357
-
358
-
latestCursor := mgr.GetCurrentCursor()
359
-
bundledOps := len(index.GetBundles()) * types.BUNDLE_SIZE
360
-
mempoolOps := latestCursor - bundledOps
361
-
362
-
if syncMode && mempoolOps > 0 {
363
-
sb.WriteString(fmt.Sprintf(" Current latest: %d (%d bundled + %d mempool)\n",
364
-
latestCursor, bundledOps, mempoolOps))
365
-
} else {
366
-
sb.WriteString(fmt.Sprintf(" Current latest: %d (%d bundles)\n",
367
-
latestCursor, len(index.GetBundles())))
368
-
}
369
-
}
370
-
371
-
sb.WriteString("\nExamples\n")
372
-
sb.WriteString("━━━━━━━━\n")
373
-
sb.WriteString(fmt.Sprintf(" curl %s/bundle/1\n", baseURL))
374
-
sb.WriteString(fmt.Sprintf(" curl %s/data/42 -o 000042.jsonl.zst\n", baseURL))
375
-
sb.WriteString(fmt.Sprintf(" curl %s/jsonl/1\n", baseURL))
376
-
377
-
if wsEnabled {
378
-
sb.WriteString(fmt.Sprintf(" websocat %s/ws\n", wsURL))
379
-
sb.WriteString(fmt.Sprintf(" websocat '%s/ws?cursor=0'\n", wsURL))
380
-
}
381
-
382
-
if syncMode {
383
-
sb.WriteString(fmt.Sprintf(" curl %s/status\n", baseURL))
384
-
sb.WriteString(fmt.Sprintf(" curl %s/mempool\n", baseURL))
385
-
}
386
-
387
-
sb.WriteString("\n────────────────────────────────────────────────────────────────\n")
388
-
sb.WriteString("https://tangled.org/@atscan.net/plcbundle\n")
389
-
390
-
w.Write([]byte(sb.String()))
391
-
}
392
-
}
393
-
394
-
func handleIndexJSONNative(mgr *bundle.Manager) http.HandlerFunc {
395
-
return func(w http.ResponseWriter, r *http.Request) {
396
-
index := mgr.GetIndex()
397
-
sendJSON(w, 200, index)
398
-
}
399
-
}
400
-
401
-
func handleBundleNative(mgr *bundle.Manager) http.HandlerFunc {
402
-
return func(w http.ResponseWriter, r *http.Request) {
403
-
bundleNum, err := strconv.Atoi(r.PathValue("number"))
404
-
if err != nil {
405
-
sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"})
406
-
return
407
-
}
408
-
409
-
meta, err := mgr.GetIndex().GetBundle(bundleNum)
410
-
if err != nil {
411
-
sendJSON(w, 404, map[string]string{"error": "Bundle not found"})
412
-
return
413
-
}
414
-
415
-
sendJSON(w, 200, meta)
416
-
}
417
-
}
418
-
419
-
func handleBundleDataNative(mgr *bundle.Manager) http.HandlerFunc {
420
-
return func(w http.ResponseWriter, r *http.Request) {
421
-
bundleNum, err := strconv.Atoi(r.PathValue("number"))
422
-
if err != nil {
423
-
sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"})
424
-
return
425
-
}
426
-
427
-
reader, err := mgr.StreamBundleRaw(context.Background(), bundleNum)
428
-
if err != nil {
429
-
if strings.Contains(err.Error(), "not in index") || strings.Contains(err.Error(), "not found") {
430
-
sendJSON(w, 404, map[string]string{"error": "Bundle not found"})
431
-
} else {
432
-
sendJSON(w, 500, map[string]string{"error": err.Error()})
433
-
}
434
-
return
435
-
}
436
-
defer reader.Close()
437
-
438
-
w.Header().Set("Content-Type", "application/zstd")
439
-
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%06d.jsonl.zst", bundleNum))
440
-
441
-
io.Copy(w, reader)
442
-
}
443
-
}
444
-
445
-
func handleBundleJSONLNative(mgr *bundle.Manager) http.HandlerFunc {
446
-
return func(w http.ResponseWriter, r *http.Request) {
447
-
bundleNum, err := strconv.Atoi(r.PathValue("number"))
448
-
if err != nil {
449
-
sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"})
450
-
return
451
-
}
452
-
453
-
reader, err := mgr.StreamBundleDecompressed(context.Background(), bundleNum)
454
-
if err != nil {
455
-
if strings.Contains(err.Error(), "not in index") || strings.Contains(err.Error(), "not found") {
456
-
sendJSON(w, 404, map[string]string{"error": "Bundle not found"})
457
-
} else {
458
-
sendJSON(w, 500, map[string]string{"error": err.Error()})
459
-
}
460
-
return
461
-
}
462
-
defer reader.Close()
463
-
464
-
w.Header().Set("Content-Type", "application/x-ndjson")
465
-
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%06d.jsonl", bundleNum))
466
-
467
-
io.Copy(w, reader)
468
-
}
469
-
}
470
-
471
-
func handleStatusNative(mgr *bundle.Manager, syncMode bool, wsEnabled bool) http.HandlerFunc {
472
-
return func(w http.ResponseWriter, r *http.Request) {
473
-
index := mgr.GetIndex()
474
-
indexStats := index.GetStats()
475
-
476
-
response := StatusResponse{
477
-
Server: ServerStatus{
478
-
Version: version,
479
-
UptimeSeconds: int(time.Since(serverStartTime).Seconds()),
480
-
SyncMode: syncMode,
481
-
WebSocketEnabled: wsEnabled,
482
-
Origin: mgr.GetPLCOrigin(),
483
-
},
484
-
Bundles: BundleStatus{
485
-
Count: indexStats["bundle_count"].(int),
486
-
TotalSize: indexStats["total_size"].(int64),
487
-
UncompressedSize: indexStats["total_uncompressed_size"].(int64),
488
-
},
489
-
}
490
-
491
-
if syncMode && syncInterval > 0 {
492
-
response.Server.SyncIntervalSeconds = int(syncInterval.Seconds())
493
-
}
494
-
495
-
if bundleCount := response.Bundles.Count; bundleCount > 0 {
496
-
firstBundle := indexStats["first_bundle"].(int)
497
-
lastBundle := indexStats["last_bundle"].(int)
498
-
499
-
response.Bundles.FirstBundle = firstBundle
500
-
response.Bundles.LastBundle = lastBundle
501
-
response.Bundles.StartTime = indexStats["start_time"].(time.Time)
502
-
response.Bundles.EndTime = indexStats["end_time"].(time.Time)
503
-
504
-
if firstMeta, err := index.GetBundle(firstBundle); err == nil {
505
-
response.Bundles.RootHash = firstMeta.Hash
506
-
}
507
-
508
-
if lastMeta, err := index.GetBundle(lastBundle); err == nil {
509
-
response.Bundles.HeadHash = lastMeta.Hash
510
-
response.Bundles.HeadAgeSeconds = int(time.Since(lastMeta.EndTime).Seconds())
511
-
}
512
-
513
-
if gaps, ok := indexStats["gaps"].(int); ok {
514
-
response.Bundles.Gaps = gaps
515
-
response.Bundles.HasGaps = gaps > 0
516
-
if gaps > 0 {
517
-
response.Bundles.GapNumbers = index.FindGaps()
518
-
}
519
-
}
520
-
521
-
totalOps := bundleCount * types.BUNDLE_SIZE
522
-
response.Bundles.TotalOperations = totalOps
523
-
524
-
duration := response.Bundles.EndTime.Sub(response.Bundles.StartTime)
525
-
if duration.Hours() > 0 {
526
-
response.Bundles.AvgOpsPerHour = int(float64(totalOps) / duration.Hours())
527
-
}
528
-
}
529
-
530
-
if syncMode {
531
-
mempoolStats := mgr.GetMempoolStats()
532
-
533
-
if count, ok := mempoolStats["count"].(int); ok {
534
-
mempool := &MempoolStatus{
535
-
Count: count,
536
-
TargetBundle: mempoolStats["target_bundle"].(int),
537
-
CanCreateBundle: mempoolStats["can_create_bundle"].(bool),
538
-
MinTimestamp: mempoolStats["min_timestamp"].(time.Time),
539
-
Validated: mempoolStats["validated"].(bool),
540
-
ProgressPercent: float64(count) / float64(types.BUNDLE_SIZE) * 100,
541
-
BundleSize: types.BUNDLE_SIZE,
542
-
OperationsNeeded: types.BUNDLE_SIZE - count,
543
-
}
544
-
545
-
if firstTime, ok := mempoolStats["first_time"].(time.Time); ok {
546
-
mempool.FirstTime = firstTime
547
-
mempool.TimespanSeconds = int(time.Since(firstTime).Seconds())
548
-
}
549
-
if lastTime, ok := mempoolStats["last_time"].(time.Time); ok {
550
-
mempool.LastTime = lastTime
551
-
mempool.LastOpAgeSeconds = int(time.Since(lastTime).Seconds())
552
-
}
553
-
554
-
if count > 100 && count < types.BUNDLE_SIZE {
555
-
if !mempool.FirstTime.IsZero() && !mempool.LastTime.IsZero() {
556
-
timespan := mempool.LastTime.Sub(mempool.FirstTime)
557
-
if timespan.Seconds() > 0 {
558
-
opsPerSec := float64(count) / timespan.Seconds()
559
-
remaining := types.BUNDLE_SIZE - count
560
-
mempool.EtaNextBundleSeconds = int(float64(remaining) / opsPerSec)
561
-
}
562
-
}
563
-
}
564
-
565
-
response.Mempool = mempool
566
-
}
567
-
}
568
-
569
-
sendJSON(w, 200, response)
570
-
}
571
-
}
572
-
573
-
func handleMempoolNative(mgr *bundle.Manager) http.HandlerFunc {
574
-
return func(w http.ResponseWriter, r *http.Request) {
575
-
ops, err := mgr.GetMempoolOperations()
576
-
if err != nil {
577
-
sendJSON(w, 500, map[string]string{"error": err.Error()})
578
-
return
579
-
}
580
-
581
-
w.Header().Set("Content-Type", "application/x-ndjson")
582
-
583
-
if len(ops) == 0 {
584
-
return
585
-
}
586
-
587
-
for _, op := range ops {
588
-
if len(op.RawJSON) > 0 {
589
-
w.Write(op.RawJSON)
590
-
} else {
591
-
data, _ := json.Marshal(op)
592
-
w.Write(data)
593
-
}
594
-
w.Write([]byte("\n"))
595
-
}
596
-
}
597
-
}
598
-
599
-
func handleDebugMemoryNative(mgr *bundle.Manager) http.HandlerFunc {
600
-
return func(w http.ResponseWriter, r *http.Request) {
601
-
var m runtime.MemStats
602
-
runtime.ReadMemStats(&m)
603
-
604
-
didStats := mgr.GetDIDIndexStats()
605
-
606
-
beforeAlloc := m.Alloc / 1024 / 1024
607
-
608
-
runtime.GC()
609
-
runtime.ReadMemStats(&m)
610
-
afterAlloc := m.Alloc / 1024 / 1024
611
-
612
-
response := fmt.Sprintf(`Memory Stats:
613
-
Alloc: %d MB
614
-
TotalAlloc: %d MB
615
-
Sys: %d MB
616
-
NumGC: %d
617
-
618
-
DID Index:
619
-
Cached shards: %d/%d
620
-
621
-
After GC:
622
-
Alloc: %d MB
623
-
`,
624
-
beforeAlloc,
625
-
m.TotalAlloc/1024/1024,
626
-
m.Sys/1024/1024,
627
-
m.NumGC,
628
-
didStats["cached_shards"],
629
-
didStats["cache_limit"],
630
-
afterAlloc)
631
-
632
-
w.Header().Set("Content-Type", "text/plain")
633
-
w.Write([]byte(response))
634
-
}
635
-
}
636
-
637
-
func handleWebSocketNative(mgr *bundle.Manager) http.HandlerFunc {
638
-
return func(w http.ResponseWriter, r *http.Request) {
639
-
cursorStr := r.URL.Query().Get("cursor")
640
-
var cursor int
641
-
642
-
if cursorStr == "" {
643
-
cursor = mgr.GetCurrentCursor()
644
-
} else {
645
-
var err error
646
-
cursor, err = strconv.Atoi(cursorStr)
647
-
if err != nil || cursor < 0 {
648
-
http.Error(w, "Invalid cursor: must be non-negative integer", 400)
649
-
return
650
-
}
651
-
}
652
-
653
-
conn, err := upgrader.Upgrade(w, r, nil)
654
-
if err != nil {
655
-
fmt.Fprintf(os.Stderr, "WebSocket upgrade failed: %v\n", err)
656
-
return
657
-
}
658
-
defer conn.Close()
659
-
660
-
conn.SetPongHandler(func(string) error {
661
-
conn.SetReadDeadline(time.Now().Add(60 * time.Second))
662
-
return nil
663
-
})
664
-
665
-
done := make(chan struct{})
666
-
667
-
go func() {
668
-
defer close(done)
669
-
for {
670
-
_, _, err := conn.ReadMessage()
671
-
if err != nil {
672
-
if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
673
-
fmt.Fprintf(os.Stderr, "WebSocket: client closed connection\n")
674
-
}
675
-
return
676
-
}
677
-
}
678
-
}()
679
-
680
-
bgCtx := context.Background()
681
-
682
-
if err := streamLive(bgCtx, conn, mgr, cursor, done); err != nil {
683
-
fmt.Fprintf(os.Stderr, "WebSocket stream error: %v\n", err)
684
-
}
685
-
}
686
-
}
687
-
688
-
func handleDIDDocumentLatestNative(mgr *bundle.Manager, did string) http.HandlerFunc {
689
-
return func(w http.ResponseWriter, r *http.Request) {
690
-
op, err := mgr.GetLatestDIDOperation(context.Background(), did)
691
-
if err != nil {
692
-
sendJSON(w, 500, map[string]string{"error": err.Error()})
693
-
return
694
-
}
695
-
696
-
doc, err := plcclient.ResolveDIDDocument(did, []plcclient.PLCOperation{*op})
697
-
if err != nil {
698
-
if strings.Contains(err.Error(), "deactivated") {
699
-
sendJSON(w, 410, map[string]string{"error": "DID has been deactivated"})
700
-
} else {
701
-
sendJSON(w, 500, map[string]string{"error": fmt.Sprintf("Resolution failed: %v", err)})
702
-
}
703
-
return
704
-
}
705
-
706
-
w.Header().Set("Content-Type", "application/did+ld+json")
707
-
sendJSON(w, 200, doc)
708
-
}
709
-
}
710
-
711
-
func handleDIDDataNative(mgr *bundle.Manager, did string) http.HandlerFunc {
712
-
return func(w http.ResponseWriter, r *http.Request) {
713
-
if err := plcclient.ValidateDIDFormat(did); err != nil {
714
-
sendJSON(w, 400, map[string]string{"error": "Invalid DID format"})
715
-
return
716
-
}
717
-
718
-
operations, err := mgr.GetDIDOperations(context.Background(), did, false)
719
-
if err != nil {
720
-
sendJSON(w, 500, map[string]string{"error": err.Error()})
721
-
return
722
-
}
723
-
724
-
if len(operations) == 0 {
725
-
sendJSON(w, 404, map[string]string{"error": "DID not found"})
726
-
return
727
-
}
728
-
729
-
state, err := plcclient.BuildDIDState(did, operations)
730
-
if err != nil {
731
-
if strings.Contains(err.Error(), "deactivated") {
732
-
sendJSON(w, 410, map[string]string{"error": "DID has been deactivated"})
733
-
} else {
734
-
sendJSON(w, 500, map[string]string{"error": err.Error()})
735
-
}
736
-
return
737
-
}
738
-
739
-
sendJSON(w, 200, state)
740
-
}
741
-
}
742
-
743
-
func handleDIDAuditLogNative(mgr *bundle.Manager, did string) http.HandlerFunc {
744
-
return func(w http.ResponseWriter, r *http.Request) {
745
-
if err := plcclient.ValidateDIDFormat(did); err != nil {
746
-
sendJSON(w, 400, map[string]string{"error": "Invalid DID format"})
747
-
return
748
-
}
749
-
750
-
operations, err := mgr.GetDIDOperations(context.Background(), did, false)
751
-
if err != nil {
752
-
sendJSON(w, 500, map[string]string{"error": err.Error()})
753
-
return
754
-
}
755
-
756
-
if len(operations) == 0 {
757
-
sendJSON(w, 404, map[string]string{"error": "DID not found"})
758
-
return
759
-
}
760
-
761
-
auditLog := plcclient.FormatAuditLog(operations)
762
-
sendJSON(w, 200, auditLog)
763
-
}
764
-
}
765
-
766
-
// WebSocket streaming functions (unchanged from your original)
767
-
768
-
func streamLive(ctx context.Context, conn *websocket.Conn, mgr *bundle.Manager, startCursor int, done chan struct{}) error {
769
-
index := mgr.GetIndex()
770
-
bundles := index.GetBundles()
771
-
currentRecord := startCursor
772
-
773
-
if len(bundles) > 0 {
774
-
startBundleIdx := startCursor / types.BUNDLE_SIZE
775
-
startPosition := startCursor % types.BUNDLE_SIZE
776
-
777
-
if startBundleIdx < len(bundles) {
778
-
for i := startBundleIdx; i < len(bundles); i++ {
779
-
skipUntil := 0
780
-
if i == startBundleIdx {
781
-
skipUntil = startPosition
782
-
}
783
-
784
-
newRecordCount, err := streamBundle(ctx, conn, mgr, bundles[i].BundleNumber, skipUntil, done)
785
-
if err != nil {
786
-
return err
787
-
}
788
-
currentRecord += newRecordCount
789
-
}
790
-
}
791
-
}
792
-
793
-
lastSeenMempoolCount := 0
794
-
if err := streamMempool(conn, mgr, startCursor, len(bundles)*types.BUNDLE_SIZE, ¤tRecord, &lastSeenMempoolCount, done); err != nil {
795
-
return err
796
-
}
797
-
798
-
ticker := time.NewTicker(500 * time.Millisecond)
799
-
defer ticker.Stop()
800
-
801
-
lastBundleCount := len(bundles)
802
-
if verboseMode {
803
-
fmt.Fprintf(os.Stderr, "WebSocket: entering live mode at cursor %d\n", currentRecord)
804
-
}
805
-
806
-
for {
807
-
select {
808
-
case <-done:
809
-
if verboseMode {
810
-
fmt.Fprintf(os.Stderr, "WebSocket: client disconnected, stopping stream\n")
811
-
}
812
-
return nil
813
-
814
-
case <-ticker.C:
815
-
index = mgr.GetIndex()
816
-
bundles = index.GetBundles()
817
-
818
-
if len(bundles) > lastBundleCount {
819
-
newBundleCount := len(bundles) - lastBundleCount
820
-
821
-
if verboseMode {
822
-
fmt.Fprintf(os.Stderr, "WebSocket: %d new bundle(s) created (operations already streamed from mempool)\n", newBundleCount)
823
-
}
824
-
825
-
currentRecord += newBundleCount * types.BUNDLE_SIZE
826
-
lastBundleCount = len(bundles)
827
-
lastSeenMempoolCount = 0
828
-
}
829
-
830
-
if err := streamMempool(conn, mgr, startCursor, len(bundles)*types.BUNDLE_SIZE, ¤tRecord, &lastSeenMempoolCount, done); err != nil {
831
-
return err
832
-
}
833
-
834
-
if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil {
835
-
return err
836
-
}
837
-
}
838
-
}
839
-
}
840
-
841
-
func streamBundle(ctx context.Context, conn *websocket.Conn, mgr *bundle.Manager, bundleNumber int, skipUntil int, done chan struct{}) (int, error) {
842
-
reader, err := mgr.StreamBundleDecompressed(ctx, bundleNumber)
843
-
if err != nil {
844
-
fmt.Fprintf(os.Stderr, "Failed to stream bundle %d: %v\n", bundleNumber, err)
845
-
return 0, nil
846
-
}
847
-
defer reader.Close()
848
-
849
-
scanner := bufio.NewScanner(reader)
850
-
buf := make([]byte, 0, 64*1024)
851
-
scanner.Buffer(buf, 1024*1024)
852
-
853
-
position := 0
854
-
streamed := 0
855
-
856
-
for scanner.Scan() {
857
-
line := scanner.Bytes()
858
-
if len(line) == 0 {
859
-
continue
860
-
}
861
-
862
-
if position < skipUntil {
863
-
position++
864
-
continue
865
-
}
866
-
867
-
select {
868
-
case <-done:
869
-
return streamed, nil
870
-
default:
871
-
}
872
-
873
-
if err := conn.WriteMessage(websocket.TextMessage, line); err != nil {
874
-
return streamed, err
875
-
}
876
-
877
-
position++
878
-
streamed++
879
-
880
-
if streamed%1000 == 0 {
881
-
conn.WriteMessage(websocket.PingMessage, nil)
882
-
}
883
-
}
884
-
885
-
if err := scanner.Err(); err != nil {
886
-
return streamed, fmt.Errorf("scanner error on bundle %d: %w", bundleNumber, err)
887
-
}
888
-
889
-
return streamed, nil
890
-
}
891
-
892
-
func streamMempool(conn *websocket.Conn, mgr *bundle.Manager, startCursor int, bundleRecordBase int, currentRecord *int, lastSeenCount *int, done chan struct{}) error {
893
-
mempoolOps, err := mgr.GetMempoolOperations()
894
-
if err != nil {
895
-
return nil
896
-
}
897
-
898
-
if len(mempoolOps) <= *lastSeenCount {
899
-
return nil
900
-
}
901
-
902
-
newOps := len(mempoolOps) - *lastSeenCount
903
-
if newOps > 0 && verboseMode {
904
-
fmt.Fprintf(os.Stderr, "WebSocket: streaming %d new mempool operation(s)\n", newOps)
905
-
}
906
-
907
-
for i := *lastSeenCount; i < len(mempoolOps); i++ {
908
-
recordNum := bundleRecordBase + i
909
-
if recordNum < startCursor {
910
-
continue
911
-
}
912
-
913
-
select {
914
-
case <-done:
915
-
return nil
916
-
default:
917
-
}
918
-
919
-
if err := sendOperation(conn, mempoolOps[i]); err != nil {
920
-
return err
921
-
}
922
-
*currentRecord++
923
-
}
924
-
925
-
*lastSeenCount = len(mempoolOps)
926
-
return nil
927
-
}
928
-
929
-
func sendOperation(conn *websocket.Conn, op plcclient.PLCOperation) error {
930
-
var data []byte
931
-
var err error
932
-
933
-
if len(op.RawJSON) > 0 {
934
-
data = op.RawJSON
935
-
} else {
936
-
data, err = json.Marshal(op)
937
-
if err != nil {
938
-
fmt.Fprintf(os.Stderr, "Failed to marshal operation: %v\n", err)
939
-
return nil
940
-
}
941
-
}
942
-
943
-
if err := conn.WriteMessage(websocket.TextMessage, data); err != nil {
944
-
if !websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
945
-
fmt.Fprintf(os.Stderr, "WebSocket write error: %v\n", err)
946
-
}
947
-
return err
948
-
}
949
-
950
-
return nil
951
-
}
952
-
953
-
// Helper functions
954
-
955
-
func getScheme(r *http.Request) string {
956
-
if r.TLS != nil {
957
-
return "https"
958
-
}
959
-
960
-
if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" {
961
-
return proto
962
-
}
963
-
964
-
if r.Header.Get("X-Forwarded-Ssl") == "on" {
965
-
return "https"
966
-
}
967
-
968
-
return "http"
969
-
}
970
-
971
-
func getWSScheme(r *http.Request) string {
972
-
if getScheme(r) == "https" {
973
-
return "wss"
974
-
}
975
-
return "ws"
976
-
}
977
-
978
-
func getBaseURL(r *http.Request) string {
979
-
scheme := getScheme(r)
980
-
host := r.Host
981
-
return fmt.Sprintf("%s://%s", scheme, host)
982
-
}
983
-
984
-
func getWSURL(r *http.Request) string {
985
-
scheme := getWSScheme(r)
986
-
host := r.Host
987
-
return fmt.Sprintf("%s://%s", scheme, host)
988
-
}
989
-
990
-
// Response types (unchanged)
991
-
992
-
type StatusResponse struct {
993
-
Bundles BundleStatus `json:"bundles"`
994
-
Mempool *MempoolStatus `json:"mempool,omitempty"`
995
-
Server ServerStatus `json:"server"`
996
-
}
997
-
998
-
type ServerStatus struct {
999
-
Version string `json:"version"`
1000
-
UptimeSeconds int `json:"uptime_seconds"`
1001
-
SyncMode bool `json:"sync_mode"`
1002
-
SyncIntervalSeconds int `json:"sync_interval_seconds,omitempty"`
1003
-
WebSocketEnabled bool `json:"websocket_enabled"`
1004
-
Origin string `json:"origin,omitempty"`
1005
-
}
1006
-
1007
-
type BundleStatus struct {
1008
-
Count int `json:"count"`
1009
-
FirstBundle int `json:"first_bundle,omitempty"`
1010
-
LastBundle int `json:"last_bundle,omitempty"`
1011
-
TotalSize int64 `json:"total_size"`
1012
-
UncompressedSize int64 `json:"uncompressed_size,omitempty"`
1013
-
CompressionRatio float64 `json:"compression_ratio,omitempty"`
1014
-
TotalOperations int `json:"total_operations,omitempty"`
1015
-
AvgOpsPerHour int `json:"avg_ops_per_hour,omitempty"`
1016
-
StartTime time.Time `json:"start_time,omitempty"`
1017
-
EndTime time.Time `json:"end_time,omitempty"`
1018
-
UpdatedAt time.Time `json:"updated_at"`
1019
-
HeadAgeSeconds int `json:"head_age_seconds,omitempty"`
1020
-
RootHash string `json:"root_hash,omitempty"`
1021
-
HeadHash string `json:"head_hash,omitempty"`
1022
-
Gaps int `json:"gaps,omitempty"`
1023
-
HasGaps bool `json:"has_gaps"`
1024
-
GapNumbers []int `json:"gap_numbers,omitempty"`
1025
-
}
1026
-
1027
-
type MempoolStatus struct {
1028
-
Count int `json:"count"`
1029
-
TargetBundle int `json:"target_bundle"`
1030
-
CanCreateBundle bool `json:"can_create_bundle"`
1031
-
MinTimestamp time.Time `json:"min_timestamp"`
1032
-
Validated bool `json:"validated"`
1033
-
ProgressPercent float64 `json:"progress_percent"`
1034
-
BundleSize int `json:"bundle_size"`
1035
-
OperationsNeeded int `json:"operations_needed"`
1036
-
FirstTime time.Time `json:"first_time,omitempty"`
1037
-
LastTime time.Time `json:"last_time,omitempty"`
1038
-
TimespanSeconds int `json:"timespan_seconds,omitempty"`
1039
-
LastOpAgeSeconds int `json:"last_op_age_seconds,omitempty"`
1040
-
EtaNextBundleSeconds int `json:"eta_next_bundle_seconds,omitempty"`
1041
-
}
1042
-
1043
-
// Background sync (unchanged)
1044
-
1045
-
func runSync(ctx context.Context, mgr *bundle.Manager, interval time.Duration, verbose bool, resolverEnabled bool) {
1046
-
syncBundles(ctx, mgr, verbose, resolverEnabled)
1047
-
1048
-
fmt.Fprintf(os.Stderr, "[Sync] Starting sync loop (interval: %s)\n", interval)
1049
-
1050
-
ticker := time.NewTicker(interval)
1051
-
defer ticker.Stop()
1052
-
1053
-
saveTicker := time.NewTicker(5 * time.Minute)
1054
-
defer saveTicker.Stop()
1055
-
1056
-
for {
1057
-
select {
1058
-
case <-ctx.Done():
1059
-
if err := mgr.SaveMempool(); err != nil {
1060
-
fmt.Fprintf(os.Stderr, "[Sync] Failed to save mempool: %v\n", err)
1061
-
}
1062
-
fmt.Fprintf(os.Stderr, "[Sync] Stopped\n")
1063
-
return
1064
-
1065
-
case <-ticker.C:
1066
-
syncBundles(ctx, mgr, verbose, resolverEnabled)
1067
-
1068
-
case <-saveTicker.C:
1069
-
stats := mgr.GetMempoolStats()
1070
-
if stats["count"].(int) > 0 && verbose {
1071
-
fmt.Fprintf(os.Stderr, "[Sync] Saving mempool (%d ops)\n", stats["count"])
1072
-
mgr.SaveMempool()
1073
-
}
1074
-
}
1075
-
}
1076
-
}
1077
-
1078
-
func syncBundles(ctx context.Context, mgr *bundle.Manager, verbose bool, resolverEnabled bool) {
1079
-
cycleStart := time.Now()
1080
-
1081
-
index := mgr.GetIndex()
1082
-
lastBundle := index.GetLastBundle()
1083
-
startBundle := 1
1084
-
if lastBundle != nil {
1085
-
startBundle = lastBundle.BundleNumber + 1
1086
-
}
1087
-
1088
-
isInitialSync := (lastBundle == nil || lastBundle.BundleNumber < 10)
1089
-
1090
-
if isInitialSync && !verbose {
1091
-
fmt.Fprintf(os.Stderr, "[Sync] Initial sync - fast loading mode (bundle %06d → ...)\n", startBundle)
1092
-
} else if verbose {
1093
-
fmt.Fprintf(os.Stderr, "[Sync] Checking for new bundles (current: %06d)...\n", startBundle-1)
1094
-
}
1095
-
1096
-
mempoolBefore := mgr.GetMempoolStats()["count"].(int)
1097
-
fetchedCount := 0
1098
-
consecutiveErrors := 0
1099
-
1100
-
for {
1101
-
currentBundle := startBundle + fetchedCount
1102
-
1103
-
b, err := mgr.FetchNextBundle(ctx, !verbose)
1104
-
if err != nil {
1105
-
if isEndOfDataError(err) {
1106
-
mempoolAfter := mgr.GetMempoolStats()["count"].(int)
1107
-
addedOps := mempoolAfter - mempoolBefore
1108
-
duration := time.Since(cycleStart)
1109
-
1110
-
if fetchedCount > 0 {
1111
-
fmt.Fprintf(os.Stderr, "[Sync] ✓ Bundle %06d | Synced: %d | Mempool: %d (+%d) | %dms\n",
1112
-
currentBundle-1, fetchedCount, mempoolAfter, addedOps, duration.Milliseconds())
1113
-
} else if !isInitialSync {
1114
-
fmt.Fprintf(os.Stderr, "[Sync] ✓ Bundle %06d | Up to date | Mempool: %d (+%d) | %dms\n",
1115
-
startBundle-1, mempoolAfter, addedOps, duration.Milliseconds())
1116
-
}
1117
-
break
1118
-
}
1119
-
1120
-
consecutiveErrors++
1121
-
if verbose {
1122
-
fmt.Fprintf(os.Stderr, "[Sync] Error fetching bundle %06d: %v\n", currentBundle, err)
1123
-
}
1124
-
1125
-
if consecutiveErrors >= 3 {
1126
-
fmt.Fprintf(os.Stderr, "[Sync] Too many errors, stopping\n")
1127
-
break
1128
-
}
1129
-
1130
-
time.Sleep(5 * time.Second)
1131
-
continue
1132
-
}
1133
-
1134
-
consecutiveErrors = 0
1135
-
1136
-
if err := mgr.SaveBundle(ctx, b, !verbose); err != nil {
1137
-
fmt.Fprintf(os.Stderr, "[Sync] Error saving bundle %06d: %v\n", b.BundleNumber, err)
1138
-
break
1139
-
}
1140
-
1141
-
fetchedCount++
1142
-
1143
-
if !verbose {
1144
-
fmt.Fprintf(os.Stderr, "[Sync] ✓ %06d | hash=%s | content=%s | %d ops, %d DIDs\n",
1145
-
b.BundleNumber,
1146
-
b.Hash[:16]+"...",
1147
-
b.ContentHash[:16]+"...",
1148
-
len(b.Operations),
1149
-
b.DIDCount)
1150
-
}
1151
-
1152
-
time.Sleep(500 * time.Millisecond)
1153
-
}
1154
-
}
1155
-
1156
-
func isEndOfDataError(err error) bool {
1157
-
if err == nil {
1158
-
return false
1159
-
}
1160
-
1161
-
errMsg := err.Error()
1162
-
return strings.Contains(errMsg, "insufficient operations") ||
1163
-
strings.Contains(errMsg, "no more operations available") ||
1164
-
strings.Contains(errMsg, "reached latest data")
1165
-
}
···
+132
cmd/plcbundle/ui/progress.go
+132
cmd/plcbundle/ui/progress.go
···
···
1
+
package ui
2
+
3
+
import (
4
+
"fmt"
5
+
"os"
6
+
"strings"
7
+
"sync"
8
+
"time"
9
+
)
10
+
11
+
// ProgressBar shows progress of an operation
12
+
type ProgressBar struct {
13
+
total int
14
+
current int
15
+
totalBytes int64
16
+
currentBytes int64
17
+
startTime time.Time
18
+
mu sync.Mutex
19
+
width int
20
+
lastPrint time.Time
21
+
showBytes bool
22
+
}
23
+
24
+
// NewProgressBar creates a new progress bar
25
+
func NewProgressBar(total int) *ProgressBar {
26
+
return &ProgressBar{
27
+
total: total,
28
+
startTime: time.Now(),
29
+
width: 40,
30
+
lastPrint: time.Now(),
31
+
showBytes: false,
32
+
}
33
+
}
34
+
35
+
// NewProgressBarWithBytes creates a new progress bar that tracks bytes
36
+
func NewProgressBarWithBytes(total int, totalBytes int64) *ProgressBar {
37
+
return &ProgressBar{
38
+
total: total,
39
+
totalBytes: totalBytes,
40
+
startTime: time.Now(),
41
+
width: 40,
42
+
lastPrint: time.Now(),
43
+
showBytes: true,
44
+
}
45
+
}
46
+
47
+
// Set sets the current progress
48
+
func (pb *ProgressBar) Set(current int) {
49
+
pb.mu.Lock()
50
+
defer pb.mu.Unlock()
51
+
pb.current = current
52
+
pb.print()
53
+
}
54
+
55
+
// SetWithBytes sets progress with byte tracking
56
+
func (pb *ProgressBar) SetWithBytes(current int, bytesProcessed int64) {
57
+
pb.mu.Lock()
58
+
defer pb.mu.Unlock()
59
+
pb.current = current
60
+
pb.currentBytes = bytesProcessed
61
+
pb.showBytes = true
62
+
pb.print()
63
+
}
64
+
65
+
// Finish completes the progress bar
66
+
func (pb *ProgressBar) Finish() {
67
+
pb.mu.Lock()
68
+
defer pb.mu.Unlock()
69
+
pb.current = pb.total
70
+
pb.currentBytes = pb.totalBytes
71
+
pb.print()
72
+
fmt.Fprintf(os.Stderr, "\n")
73
+
}
74
+
75
+
// print renders the progress bar
76
+
func (pb *ProgressBar) print() {
77
+
if time.Since(pb.lastPrint) < 100*time.Millisecond && pb.current < pb.total {
78
+
return
79
+
}
80
+
pb.lastPrint = time.Now()
81
+
82
+
percent := 0.0
83
+
if pb.total > 0 {
84
+
percent = float64(pb.current) / float64(pb.total) * 100
85
+
}
86
+
87
+
filled := 0
88
+
if pb.total > 0 {
89
+
filled = int(float64(pb.width) * float64(pb.current) / float64(pb.total))
90
+
if filled > pb.width {
91
+
filled = pb.width
92
+
}
93
+
}
94
+
95
+
bar := strings.Repeat("█", filled) + strings.Repeat("░", pb.width-filled)
96
+
97
+
elapsed := time.Since(pb.startTime)
98
+
speed := 0.0
99
+
if elapsed.Seconds() > 0 {
100
+
speed = float64(pb.current) / elapsed.Seconds()
101
+
}
102
+
103
+
remaining := pb.total - pb.current
104
+
var eta time.Duration
105
+
if speed > 0 {
106
+
eta = time.Duration(float64(remaining)/speed) * time.Second
107
+
}
108
+
109
+
if pb.showBytes && pb.currentBytes > 0 {
110
+
mbProcessed := float64(pb.currentBytes) / (1000 * 1000)
111
+
mbPerSec := mbProcessed / elapsed.Seconds()
112
+
113
+
fmt.Fprintf(os.Stderr, "\r [%s] %6.2f%% | %d/%d | %.1f/s | %.1f MB/s | ETA: %s ",
114
+
bar, percent, pb.current, pb.total, speed, mbPerSec, formatETA(eta))
115
+
} else {
116
+
fmt.Fprintf(os.Stderr, "\r [%s] %6.2f%% | %d/%d | %.1f/s | ETA: %s ",
117
+
bar, percent, pb.current, pb.total, speed, formatETA(eta))
118
+
}
119
+
}
120
+
121
+
func formatETA(d time.Duration) string {
122
+
if d == 0 {
123
+
return "calculating..."
124
+
}
125
+
if d < time.Minute {
126
+
return fmt.Sprintf("%ds", int(d.Seconds()))
127
+
}
128
+
if d < time.Hour {
129
+
return fmt.Sprintf("%dm %ds", int(d.Minutes()), int(d.Seconds())%60)
130
+
}
131
+
return fmt.Sprintf("%dh %dm", int(d.Hours()), int(d.Minutes())%60)
132
+
}
+28
-6
internal/bundle/manager.go
+28
-6
internal/bundle/manager.go
···
1358
1359
if lastBundle != nil {
1360
nextBundleNum = lastBundle.BundleNumber + 1
1361
-
afterTime = lastBundle.EndTime.Format(time.RFC3339Nano)
1362
prevBundleHash = lastBundle.Hash
1363
1364
prevBundle, err := m.LoadBundle(ctx, lastBundle.BundleNumber)
1365
if err == nil {
1366
_, prevBoundaryCIDs = m.operations.GetBoundaryCIDs(prevBundle.Operations)
···
1371
m.logger.Printf("Preparing bundle %06d (mempool: %d ops)...", nextBundleNum, m.mempool.Count())
1372
}
1373
1374
-
// Fetch operations using syncer
1375
-
for m.mempool.Count() < types.BUNDLE_SIZE {
1376
newOps, err := m.syncer.FetchToMempool(
1377
ctx,
1378
afterTime,
···
1394
return nil, fmt.Errorf("chronological validation failed: %w", err)
1395
}
1396
1397
-
if !quiet {
1398
m.logger.Printf("Added %d new operations (mempool now: %d)", added, m.mempool.Count())
1399
}
1400
1401
-
if len(newOps) == 0 {
1402
break
1403
}
1404
}
1405
1406
if m.mempool.Count() < types.BUNDLE_SIZE {
1407
m.mempool.Save()
1408
-
return nil, fmt.Errorf("insufficient operations: have %d, need %d", m.mempool.Count(), types.BUNDLE_SIZE)
1409
}
1410
1411
// Create bundle
···
1358
1359
if lastBundle != nil {
1360
nextBundleNum = lastBundle.BundleNumber + 1
1361
prevBundleHash = lastBundle.Hash
1362
1363
+
// ✨ FIX: Use mempool's last operation time if available
1364
+
// This prevents re-fetching operations already in mempool
1365
+
mempoolLastTime := m.mempool.GetLastTime()
1366
+
if mempoolLastTime != "" {
1367
+
afterTime = mempoolLastTime
1368
+
if !quiet {
1369
+
m.logger.Printf("Using mempool cursor: %s", afterTime)
1370
+
}
1371
+
} else {
1372
+
// No mempool operations yet, use last bundle
1373
+
afterTime = lastBundle.EndTime.Format(time.RFC3339Nano)
1374
+
}
1375
+
1376
prevBundle, err := m.LoadBundle(ctx, lastBundle.BundleNumber)
1377
if err == nil {
1378
_, prevBoundaryCIDs = m.operations.GetBoundaryCIDs(prevBundle.Operations)
···
1383
m.logger.Printf("Preparing bundle %06d (mempool: %d ops)...", nextBundleNum, m.mempool.Count())
1384
}
1385
1386
+
// Fetch in a loop until we have enough OR hit end-of-data
1387
+
maxAttempts := 10
1388
+
attemptCount := 0
1389
+
1390
+
for m.mempool.Count() < types.BUNDLE_SIZE && attemptCount < maxAttempts {
1391
+
attemptCount++
1392
+
1393
newOps, err := m.syncer.FetchToMempool(
1394
ctx,
1395
afterTime,
···
1411
return nil, fmt.Errorf("chronological validation failed: %w", err)
1412
}
1413
1414
+
if !quiet && added > 0 {
1415
m.logger.Printf("Added %d new operations (mempool now: %d)", added, m.mempool.Count())
1416
}
1417
1418
+
// ✨ Update cursor to last operation in mempool
1419
+
afterTime = m.mempool.GetLastTime()
1420
+
1421
+
// If we got no new operations, we've caught up
1422
+
if len(newOps) == 0 || added == 0 {
1423
break
1424
}
1425
}
1426
1427
if m.mempool.Count() < types.BUNDLE_SIZE {
1428
m.mempool.Save()
1429
+
return nil, fmt.Errorf("insufficient operations: have %d, need %d (reached latest data)",
1430
+
m.mempool.Count(), types.BUNDLE_SIZE)
1431
}
1432
1433
// Create bundle
+6
-11
internal/sync/fetcher.go
+6
-11
internal/sync/fetcher.go
···
27
}
28
29
// FetchToMempool fetches operations and returns them
30
-
// Returns: operations, error
31
func (f *Fetcher) FetchToMempool(
32
ctx context.Context,
33
afterTime string,
···
49
var allNewOps []plcclient.PLCOperation
50
51
for fetchNum := 0; fetchNum < maxFetches; fetchNum++ {
52
-
// Calculate batch size
53
remaining := target - len(allNewOps)
54
if remaining <= 0 {
55
break
···
61
}
62
63
if !quiet {
64
-
f.logger.Printf(" Fetch #%d: requesting %d operations",
65
-
fetchNum+1, batchSize)
66
}
67
68
batch, err := f.plcClient.Export(ctx, plcclient.ExportOptions{
···
77
if !quiet {
78
f.logger.Printf(" No more operations available from PLC")
79
}
80
-
if len(allNewOps) > 0 {
81
-
return allNewOps, nil
82
-
}
83
-
return nil, fmt.Errorf("no operations available")
84
}
85
86
-
// Deduplicate
87
for _, op := range batch {
88
if !seenCIDs[op.CID] {
89
seenCIDs[op.CID] = true
···
96
currentAfter = batch[len(batch)-1].CreatedAt.Format(time.RFC3339Nano)
97
}
98
99
-
// Stop if we got less than requested
100
if len(batch) < batchSize {
101
if !quiet {
102
f.logger.Printf(" Received incomplete batch (%d/%d), caught up to latest", len(batch), batchSize)
···
112
return allNewOps, nil
113
}
114
115
-
return nil, fmt.Errorf("no new operations added")
116
}
···
27
}
28
29
// FetchToMempool fetches operations and returns them
30
func (f *Fetcher) FetchToMempool(
31
ctx context.Context,
32
afterTime string,
···
48
var allNewOps []plcclient.PLCOperation
49
50
for fetchNum := 0; fetchNum < maxFetches; fetchNum++ {
51
remaining := target - len(allNewOps)
52
if remaining <= 0 {
53
break
···
59
}
60
61
if !quiet {
62
+
f.logger.Printf(" Fetch #%d: requesting %d operations", fetchNum+1, batchSize)
63
}
64
65
batch, err := f.plcClient.Export(ctx, plcclient.ExportOptions{
···
74
if !quiet {
75
f.logger.Printf(" No more operations available from PLC")
76
}
77
+
break
78
}
79
80
+
// Deduplicate against boundary CIDs only
81
+
// Mempool will handle deduplication of operations already in mempool
82
for _, op := range batch {
83
if !seenCIDs[op.CID] {
84
seenCIDs[op.CID] = true
···
91
currentAfter = batch[len(batch)-1].CreatedAt.Format(time.RFC3339Nano)
92
}
93
94
+
// Stop if we got less than requested (caught up)
95
if len(batch) < batchSize {
96
if !quiet {
97
f.logger.Printf(" Received incomplete batch (%d/%d), caught up to latest", len(batch), batchSize)
···
107
return allNewOps, nil
108
}
109
110
+
return nil, fmt.Errorf("no operations available (reached latest data)")
111
}
+54
options.go
+54
options.go
···
···
1
+
package plcbundle
2
+
3
+
import (
4
+
"tangled.org/atscan.net/plcbundle/internal/bundle"
5
+
"tangled.org/atscan.net/plcbundle/plcclient"
6
+
)
7
+
8
+
type config struct {
9
+
bundleConfig *bundle.Config
10
+
plcClient *plcclient.Client
11
+
}
12
+
13
+
func defaultConfig() *config {
14
+
return &config{
15
+
bundleConfig: bundle.DefaultConfig("./plc_bundles"),
16
+
}
17
+
}
18
+
19
+
// Option configures the Manager
20
+
type Option func(*config)
21
+
22
+
// WithDirectory sets the bundle storage directory
23
+
func WithDirectory(dir string) Option {
24
+
return func(c *config) {
25
+
c.bundleConfig.BundleDir = dir
26
+
}
27
+
}
28
+
29
+
// WithPLCDirectory sets the PLC directory URL
30
+
func WithPLCDirectory(url string) Option {
31
+
return func(c *config) {
32
+
c.plcClient = plcclient.NewClient(url)
33
+
}
34
+
}
35
+
36
+
// WithVerifyOnLoad enables/disables hash verification when loading bundles
37
+
func WithVerifyOnLoad(verify bool) Option {
38
+
return func(c *config) {
39
+
c.bundleConfig.VerifyOnLoad = verify
40
+
}
41
+
}
42
+
43
+
// WithLogger sets a custom logger
44
+
func WithLogger(logger Logger) Option {
45
+
return func(c *config) {
46
+
c.bundleConfig.Logger = logger
47
+
}
48
+
}
49
+
50
+
// Logger interface
51
+
type Logger interface {
52
+
Printf(format string, v ...interface{})
53
+
Println(v ...interface{})
54
+
}
+595
server/handlers.go
+595
server/handlers.go
···
···
1
+
package server
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"io"
7
+
"net/http"
8
+
"runtime"
9
+
"strconv"
10
+
"strings"
11
+
"time"
12
+
13
+
"github.com/goccy/go-json"
14
+
"tangled.org/atscan.net/plcbundle/internal/types"
15
+
"tangled.org/atscan.net/plcbundle/plcclient"
16
+
)
17
+
18
+
func (s *Server) handleRoot() http.HandlerFunc {
19
+
return func(w http.ResponseWriter, r *http.Request) {
20
+
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
21
+
22
+
index := s.manager.GetIndex()
23
+
stats := index.GetStats()
24
+
bundleCount := stats["bundle_count"].(int)
25
+
26
+
baseURL := getBaseURL(r)
27
+
wsURL := getWSURL(r)
28
+
29
+
var sb strings.Builder
30
+
31
+
sb.WriteString(`
32
+
33
+
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠄⠀⡀⠀⠀⠀⠀⠀⠀⢀⠀⠀⡀⠀⢀⠀⢀⡀⣤⡢⣤⡤⡀⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
34
+
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⡄⡄⠐⡀⠈⣀⠀⡠⡠⠀⣢⣆⢌⡾⢙⠺⢽⠾⡋⣻⡷⡫⢵⣭⢦⣴⠦⠀⢠⠀⠀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
35
+
⠀⠀⠀⠀⠀⠀⠀⠀⢠⣤⣽⣥⡈⠧⣂⢧⢾⠕⠞⠡⠊⠁⣐⠉⠀⠉⢍⠀⠉⠌⡉⠀⠂⠁⠱⠉⠁⢝⠻⠎⣬⢌⡌⣬⣡⣀⣢⣄⡄⠀⡀⠀⠀⠀⠀⠀⠀⠀⠀
36
+
⠀⠀⠀⠀⠀⠀⠀⢀⢸⣿⣿⢿⣾⣯⣑⢄⡂⠀⠄⠂⠀⠀⢀⠀⠀⠐⠀⠀⠀⠀⠀⠀⠀⠀⠄⠐⠀⠀⠀⠀⣄⠭⠂⠈⠜⣩⣿⢝⠃⠀⠁⠀⠀⠀⠀⠀⠀⠀⠀
37
+
⠀⠀⠀⠀⠀⠀⠀⢀⣻⡟⠏⠀⠚⠈⠚⡉⡝⢶⣱⢤⣅⠈⠀⠄⠀⠀⠀⠀⠀⠠⠀⠀⡂⠐⣤⢕⡪⢼⣈⡹⡇⠏⠏⠋⠅⢃⣪⡏⡇⡍⠀⠀⠀⠀⠀⠀⠀⠀⠀
38
+
⠀⠀⠀⠀⠀⠀⠀⠀⠺⣻⡄⠀⠀⠀⢠⠌⠃⠐⠉⢡⠱⠧⠝⡯⣮⢶⣴⣤⡆⢐⣣⢅⣮⡟⠦⠍⠉⠀⠁⠐⠀⠀⠀⠄⠐⠡⣽⡸⣎⢁⠀⠀⠀⠀⠀⠀⠀⠀⠀
39
+
⠀⠀⠀⠀⠀⠀⠀⢈⡻⣧⠀⠁⠐⠀⠀⠀⠀⠀⠀⠊⠀⠕⢀⡉⠈⡫⠽⡿⡟⠿⠟⠁⠀⠀⠄⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠬⠥⣋⡯⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
40
+
⠀⠀⠀⠀⠀⠀⠀⡀⣾⡍⠕⡀⠀⠀⠀⠄⠠⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠥⣤⢌⠀⠀⠀⠀⠀⠀⠀⠀⠀⠁⠀⠀⠄⢀⠀⢝⢞⣫⡆⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀
41
+
⠀⠀⠀⠀⠀⠀⠀⠀⣽⡶⡄⠐⡀⠀⠀⠀⠀⠀⠀⢀⠀⠄⠀⠀⠀⠄⠁⠇⣷⡆⠀⠀⠀⢀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⡸⢝⣮⠍⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
42
+
⠀⠀⠀⠀⠀⠀⢀⠀⢾⣷⠀⠠⡀⠀⠀⠀⠀⢀⠀⠀⠀⠀⠀⠁⡁⠀⠀⣾⡥⠖⠀⠀⠀⠂⠀⠀⠀⠀⠀⠁⠀⡀⠁⠀⠀⠻⢳⣻⢄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
43
+
⠀⠀⠀⠀⠀⠀⠀⠀⣞⡙⠨⣀⠠⠄⠀⠂⠀⠀⠀⠈⢀⠀⠀⠀⠀⠀⠤⢚⢢⣟⠀⠀⠀⠀⡐⠀⠀⡀⠀⠀⠀⠀⠁⠈⠌⠊⣯⣮⡏⠡⠂⠀⠀⠀⠀⠀⠀⠀⠀
44
+
⠀⠀⠀⠀⠀⠀⠀⠀⣻⡟⡄⡡⣄⠀⠠⠀⠀⡅⠀⠐⠀⡀⠀⡀⠀⠄⠈⠃⠳⠪⠤⠀⠀⠀⠀⡀⠀⠂⠀⠀⠀⠁⠈⢠⣠⠒⠻⣻⡧⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
45
+
⠀⠀⠀⠀⠀⠀⠀⠀⠪⡎⠠⢌⠑⡀⠂⠀⠄⠠⠀⠠⠀⠁⡀⠠⠠⡀⣀⠜⢏⡅⠀⠀⡀⠁⠀⠀⠁⠁⠐⠄⡀⢀⠂⠀⠄⢑⣿⣿⣿⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀
46
+
⠀⠀⠀⠀⠀⠀⠀⠀⠼⣻⠧⣣⣀⠐⠨⠁⠕⢈⢀⢀⡁⠀⠈⠠⢀⠀⠐⠜⣽⡗⡤⠀⠂⠀⠠⠀⢂⠠⠀⠁⠄⠀⠔⠀⠑⣨⣿⢯⠋⡅⠀⠀⠀⠀⠀⠀⠀⠀⠀
47
+
⠀⠀⠀⠀⠀⠀⠀⠀⡚⣷⣭⠎⢃⡗⠄⡄⢀⠁⠀⠅⢀⢅⡀⠠⠀⢠⡀⡩⠷⢇⠀⡀⠄⡠⠤⠆⣀⡀⠄⠉⣠⠃⠴⠀⠈⢁⣿⡛⡯⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
48
+
⠀⠀⠀⠀⠀⠀⠀⠘⡬⡿⣿⡏⡻⡯⠌⢁⢛⠠⠓⠐⠐⠐⠌⠃⠋⠂⡢⢰⣈⢏⣰⠂⠈⠀⠠⠒⠡⠌⠫⠭⠩⠢⡬⠆⠿⢷⢿⡽⡧⠉⠊⠀⠀⠀⠀⠀⠀⠀⠀
49
+
⠀⠀⠀⠀⠀⠀⠀⠀⠺⣷⣺⣗⣿⡶⡎⡅⣣⢎⠠⡅⣢⡖⠴⠬⡈⠂⡨⢡⠾⣣⣢⠀⠀⡹⠄⡄⠄⡇⣰⡖⡊⠔⢹⣄⣿⣭⣵⣿⢷⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀
50
+
⠀⠀⠀⠀⠀⠀⠀⠀⠩⣿⣿⣲⣿⣷⣟⣼⠟⣬⢉⡠⣪⢜⣂⣁⠥⠓⠚⡁⢶⣷⣠⠂⡄⡢⣀⡐⠧⢆⣒⡲⡳⡫⢟⡃⢪⡧⣟⡟⣯⠐⠀⠀⠀⠀⠀⠀⠀⠀⠀
51
+
⠀⠀⠀⠀⠀⠀⠀⠀⢺⠟⢿⢟⢻⡗⡮⡿⣲⢷⣆⣏⣇⡧⣄⢖⠾⡷⣿⣤⢳⢷⣣⣦⡜⠗⣭⢂⠩⣹⢿⡲⢎⡧⣕⣖⣓⣽⡿⡖⡿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
52
+
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠉⠂⠂⠏⠿⢻⣥⡪⢽⣳⣳⣥⡶⣫⣍⢐⣥⣻⣾⡻⣅⢭⡴⢭⣿⠕⣧⡭⣞⣻⣣⣻⢿⠟⠛⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
53
+
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠄⠋⠫⠯⣍⢻⣿⣿⣷⣕⣵⣹⣽⣿⣷⣇⡏⣿⡿⣍⡝⠵⠯⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
54
+
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠐⠠⠁⠋⢣⠓⡍⣫⠹⣿⣿⣷⡿⠯⠺⠁⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
55
+
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠋⢀⠋⢈⡿⠿⠁⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
56
+
57
+
plcbundle server
58
+
59
+
*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*
60
+
| ⚠️ Preview Version – Do Not Use In Production! |
61
+
*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*
62
+
| This project and plcbundle specification is currently |
63
+
| unstable and under heavy development. Things can break at |
64
+
| any time. Do not use this for production systems. |
65
+
| Please wait for the 1.0 release. |
66
+
|________________________________________________________________|
67
+
68
+
`)
69
+
70
+
sb.WriteString("\nplcbundle server\n\n")
71
+
sb.WriteString("What is PLC Bundle?\n")
72
+
sb.WriteString("━━━━━━━━━━━━━━━━━━━━\n")
73
+
sb.WriteString("plcbundle archives AT Protocol's DID PLC Directory operations into\n")
74
+
sb.WriteString("immutable, cryptographically-chained bundles of 10,000 operations.\n\n")
75
+
sb.WriteString("More info: https://tangled.org/@atscan.net/plcbundle\n\n")
76
+
77
+
if bundleCount > 0 {
78
+
sb.WriteString("Bundles\n")
79
+
sb.WriteString("━━━━━━━\n")
80
+
sb.WriteString(fmt.Sprintf(" Bundle count: %d\n", bundleCount))
81
+
82
+
firstBundle := stats["first_bundle"].(int)
83
+
lastBundle := stats["last_bundle"].(int)
84
+
totalSize := stats["total_size"].(int64)
85
+
totalUncompressed := stats["total_uncompressed_size"].(int64)
86
+
87
+
sb.WriteString(fmt.Sprintf(" Last bundle: %d (%s)\n", lastBundle,
88
+
stats["updated_at"].(time.Time).Format("2006-01-02 15:04:05")))
89
+
sb.WriteString(fmt.Sprintf(" Range: %06d - %06d\n", firstBundle, lastBundle))
90
+
sb.WriteString(fmt.Sprintf(" Total size: %.2f MB\n", float64(totalSize)/(1000*1000)))
91
+
sb.WriteString(fmt.Sprintf(" Uncompressed: %.2f MB (%.2fx)\n",
92
+
float64(totalUncompressed)/(1000*1000),
93
+
float64(totalUncompressed)/float64(totalSize)))
94
+
95
+
if gaps, ok := stats["gaps"].(int); ok && gaps > 0 {
96
+
sb.WriteString(fmt.Sprintf(" ⚠ Gaps: %d missing bundles\n", gaps))
97
+
}
98
+
99
+
firstMeta, err := index.GetBundle(firstBundle)
100
+
if err == nil {
101
+
sb.WriteString(fmt.Sprintf("\n Root: %s\n", firstMeta.Hash))
102
+
}
103
+
104
+
lastMeta, err := index.GetBundle(lastBundle)
105
+
if err == nil {
106
+
sb.WriteString(fmt.Sprintf(" Head: %s\n", lastMeta.Hash))
107
+
}
108
+
}
109
+
110
+
if s.config.SyncMode {
111
+
mempoolStats := s.manager.GetMempoolStats()
112
+
count := mempoolStats["count"].(int)
113
+
targetBundle := mempoolStats["target_bundle"].(int)
114
+
canCreate := mempoolStats["can_create_bundle"].(bool)
115
+
116
+
sb.WriteString("\nMempool Stats\n")
117
+
sb.WriteString("━━━━━━━━━━━━━\n")
118
+
sb.WriteString(fmt.Sprintf(" Target bundle: %d\n", targetBundle))
119
+
sb.WriteString(fmt.Sprintf(" Operations: %d / %d\n", count, types.BUNDLE_SIZE))
120
+
sb.WriteString(fmt.Sprintf(" Can create bundle: %v\n", canCreate))
121
+
122
+
if count > 0 {
123
+
progress := float64(count) / float64(types.BUNDLE_SIZE) * 100
124
+
sb.WriteString(fmt.Sprintf(" Progress: %.1f%%\n", progress))
125
+
126
+
barWidth := 50
127
+
filled := int(float64(barWidth) * float64(count) / float64(types.BUNDLE_SIZE))
128
+
if filled > barWidth {
129
+
filled = barWidth
130
+
}
131
+
bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled)
132
+
sb.WriteString(fmt.Sprintf(" [%s]\n", bar))
133
+
134
+
if firstTime, ok := mempoolStats["first_time"].(time.Time); ok {
135
+
sb.WriteString(fmt.Sprintf(" First op: %s\n", firstTime.Format("2006-01-02 15:04:05")))
136
+
}
137
+
if lastTime, ok := mempoolStats["last_time"].(time.Time); ok {
138
+
sb.WriteString(fmt.Sprintf(" Last op: %s\n", lastTime.Format("2006-01-02 15:04:05")))
139
+
}
140
+
} else {
141
+
sb.WriteString(" (empty)\n")
142
+
}
143
+
}
144
+
145
+
if didStats := s.manager.GetDIDIndexStats(); didStats["exists"].(bool) {
146
+
sb.WriteString("\nDID Index\n")
147
+
sb.WriteString("━━━━━━━━━\n")
148
+
sb.WriteString(" Status: enabled\n")
149
+
150
+
indexedDIDs := didStats["indexed_dids"].(int64)
151
+
mempoolDIDs := didStats["mempool_dids"].(int64)
152
+
totalDIDs := didStats["total_dids"].(int64)
153
+
154
+
if mempoolDIDs > 0 {
155
+
sb.WriteString(fmt.Sprintf(" Total DIDs: %s (%s indexed + %s mempool)\n",
156
+
formatNumber(int(totalDIDs)),
157
+
formatNumber(int(indexedDIDs)),
158
+
formatNumber(int(mempoolDIDs))))
159
+
} else {
160
+
sb.WriteString(fmt.Sprintf(" Total DIDs: %s\n", formatNumber(int(totalDIDs))))
161
+
}
162
+
163
+
sb.WriteString(fmt.Sprintf(" Cached shards: %d / %d\n",
164
+
didStats["cached_shards"], didStats["cache_limit"]))
165
+
sb.WriteString("\n")
166
+
}
167
+
168
+
sb.WriteString("Server Stats\n")
169
+
sb.WriteString("━━━━━━━━━━━━\n")
170
+
sb.WriteString(fmt.Sprintf(" Version: %s\n", s.config.Version))
171
+
if origin := s.manager.GetPLCOrigin(); origin != "" {
172
+
sb.WriteString(fmt.Sprintf(" Origin: %s\n", origin))
173
+
}
174
+
sb.WriteString(fmt.Sprintf(" Sync mode: %v\n", s.config.SyncMode))
175
+
sb.WriteString(fmt.Sprintf(" WebSocket: %v\n", s.config.EnableWebSocket))
176
+
sb.WriteString(fmt.Sprintf(" Resolver: %v\n", s.config.EnableResolver))
177
+
sb.WriteString(fmt.Sprintf(" Uptime: %s\n", time.Since(s.startTime).Round(time.Second)))
178
+
179
+
sb.WriteString("\n\nAPI Endpoints\n")
180
+
sb.WriteString("━━━━━━━━━━━━━\n")
181
+
sb.WriteString(" GET / This info page\n")
182
+
sb.WriteString(" GET /index.json Full bundle index\n")
183
+
sb.WriteString(" GET /bundle/:number Bundle metadata (JSON)\n")
184
+
sb.WriteString(" GET /data/:number Raw bundle (zstd compressed)\n")
185
+
sb.WriteString(" GET /jsonl/:number Decompressed JSONL stream\n")
186
+
sb.WriteString(" GET /status Server status\n")
187
+
sb.WriteString(" GET /mempool Mempool operations (JSONL)\n")
188
+
189
+
if s.config.EnableResolver {
190
+
sb.WriteString("\nDID Resolution\n")
191
+
sb.WriteString("━━━━━━━━━━━━━━\n")
192
+
sb.WriteString(" GET /:did DID Document (W3C format)\n")
193
+
sb.WriteString(" GET /:did/data PLC State (raw format)\n")
194
+
sb.WriteString(" GET /:did/log/audit Operation history\n")
195
+
196
+
didStats := s.manager.GetDIDIndexStats()
197
+
if didStats["exists"].(bool) {
198
+
sb.WriteString(fmt.Sprintf("\n Index: %s DIDs indexed\n",
199
+
formatNumber(int(didStats["total_dids"].(int64)))))
200
+
} else {
201
+
sb.WriteString("\n ⚠️ Index: not built (will use slow scan)\n")
202
+
}
203
+
sb.WriteString("\n")
204
+
}
205
+
206
+
if s.config.EnableWebSocket {
207
+
sb.WriteString("\nWebSocket Endpoints\n")
208
+
sb.WriteString("━━━━━━━━━━━━━━━━━━━\n")
209
+
sb.WriteString(" WS /ws Live stream (new operations only)\n")
210
+
sb.WriteString(" WS /ws?cursor=0 Stream all from beginning\n")
211
+
sb.WriteString(" WS /ws?cursor=N Stream from cursor N\n\n")
212
+
sb.WriteString("Cursor Format:\n")
213
+
sb.WriteString(" Global record number: (bundleNumber × 10,000) + position\n")
214
+
sb.WriteString(" Example: 88410345 = bundle 8841, position 345\n")
215
+
sb.WriteString(" Default: starts from latest (skips all historical data)\n")
216
+
217
+
latestCursor := s.manager.GetCurrentCursor()
218
+
bundledOps := len(index.GetBundles()) * types.BUNDLE_SIZE
219
+
mempoolOps := latestCursor - bundledOps
220
+
221
+
if s.config.SyncMode && mempoolOps > 0 {
222
+
sb.WriteString(fmt.Sprintf(" Current latest: %d (%d bundled + %d mempool)\n",
223
+
latestCursor, bundledOps, mempoolOps))
224
+
} else {
225
+
sb.WriteString(fmt.Sprintf(" Current latest: %d (%d bundles)\n",
226
+
latestCursor, len(index.GetBundles())))
227
+
}
228
+
}
229
+
230
+
sb.WriteString("\nExamples\n")
231
+
sb.WriteString("━━━━━━━━\n")
232
+
sb.WriteString(fmt.Sprintf(" curl %s/bundle/1\n", baseURL))
233
+
sb.WriteString(fmt.Sprintf(" curl %s/data/42 -o 000042.jsonl.zst\n", baseURL))
234
+
sb.WriteString(fmt.Sprintf(" curl %s/jsonl/1\n", baseURL))
235
+
236
+
if s.config.EnableWebSocket {
237
+
sb.WriteString(fmt.Sprintf(" websocat %s/ws\n", wsURL))
238
+
sb.WriteString(fmt.Sprintf(" websocat '%s/ws?cursor=0'\n", wsURL))
239
+
}
240
+
241
+
if s.config.SyncMode {
242
+
sb.WriteString(fmt.Sprintf(" curl %s/status\n", baseURL))
243
+
sb.WriteString(fmt.Sprintf(" curl %s/mempool\n", baseURL))
244
+
}
245
+
246
+
sb.WriteString("\n────────────────────────────────────────────────────────────────\n")
247
+
sb.WriteString("https://tangled.org/@atscan.net/plcbundle\n")
248
+
249
+
w.Write([]byte(sb.String()))
250
+
}
251
+
}
252
+
253
+
func (s *Server) handleIndexJSON() http.HandlerFunc {
254
+
return func(w http.ResponseWriter, r *http.Request) {
255
+
index := s.manager.GetIndex()
256
+
sendJSON(w, 200, index)
257
+
}
258
+
}
259
+
260
+
func (s *Server) handleBundle() http.HandlerFunc {
261
+
return func(w http.ResponseWriter, r *http.Request) {
262
+
bundleNum, err := strconv.Atoi(r.PathValue("number"))
263
+
if err != nil {
264
+
sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"})
265
+
return
266
+
}
267
+
268
+
meta, err := s.manager.GetIndex().GetBundle(bundleNum)
269
+
if err != nil {
270
+
sendJSON(w, 404, map[string]string{"error": "Bundle not found"})
271
+
return
272
+
}
273
+
274
+
sendJSON(w, 200, meta)
275
+
}
276
+
}
277
+
278
+
func (s *Server) handleBundleData() http.HandlerFunc {
279
+
return func(w http.ResponseWriter, r *http.Request) {
280
+
bundleNum, err := strconv.Atoi(r.PathValue("number"))
281
+
if err != nil {
282
+
sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"})
283
+
return
284
+
}
285
+
286
+
reader, err := s.manager.StreamBundleRaw(context.Background(), bundleNum)
287
+
if err != nil {
288
+
if strings.Contains(err.Error(), "not in index") || strings.Contains(err.Error(), "not found") {
289
+
sendJSON(w, 404, map[string]string{"error": "Bundle not found"})
290
+
} else {
291
+
sendJSON(w, 500, map[string]string{"error": err.Error()})
292
+
}
293
+
return
294
+
}
295
+
defer reader.Close()
296
+
297
+
w.Header().Set("Content-Type", "application/zstd")
298
+
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%06d.jsonl.zst", bundleNum))
299
+
300
+
io.Copy(w, reader)
301
+
}
302
+
}
303
+
304
+
func (s *Server) handleBundleJSONL() http.HandlerFunc {
305
+
return func(w http.ResponseWriter, r *http.Request) {
306
+
bundleNum, err := strconv.Atoi(r.PathValue("number"))
307
+
if err != nil {
308
+
sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"})
309
+
return
310
+
}
311
+
312
+
reader, err := s.manager.StreamBundleDecompressed(context.Background(), bundleNum)
313
+
if err != nil {
314
+
if strings.Contains(err.Error(), "not in index") || strings.Contains(err.Error(), "not found") {
315
+
sendJSON(w, 404, map[string]string{"error": "Bundle not found"})
316
+
} else {
317
+
sendJSON(w, 500, map[string]string{"error": err.Error()})
318
+
}
319
+
return
320
+
}
321
+
defer reader.Close()
322
+
323
+
w.Header().Set("Content-Type", "application/x-ndjson")
324
+
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%06d.jsonl", bundleNum))
325
+
326
+
io.Copy(w, reader)
327
+
}
328
+
}
329
+
330
+
func (s *Server) handleStatus() http.HandlerFunc {
331
+
return func(w http.ResponseWriter, r *http.Request) {
332
+
index := s.manager.GetIndex()
333
+
indexStats := index.GetStats()
334
+
335
+
response := StatusResponse{
336
+
Server: ServerStatus{
337
+
Version: s.config.Version,
338
+
UptimeSeconds: int(time.Since(s.startTime).Seconds()),
339
+
SyncMode: s.config.SyncMode,
340
+
WebSocketEnabled: s.config.EnableWebSocket,
341
+
Origin: s.manager.GetPLCOrigin(),
342
+
},
343
+
Bundles: BundleStatus{
344
+
Count: indexStats["bundle_count"].(int),
345
+
TotalSize: indexStats["total_size"].(int64),
346
+
UncompressedSize: indexStats["total_uncompressed_size"].(int64),
347
+
UpdatedAt: indexStats["updated_at"].(time.Time),
348
+
},
349
+
}
350
+
351
+
if s.config.SyncMode && s.config.SyncInterval > 0 {
352
+
response.Server.SyncIntervalSeconds = int(s.config.SyncInterval.Seconds())
353
+
}
354
+
355
+
if bundleCount := response.Bundles.Count; bundleCount > 0 {
356
+
firstBundle := indexStats["first_bundle"].(int)
357
+
lastBundle := indexStats["last_bundle"].(int)
358
+
359
+
response.Bundles.FirstBundle = firstBundle
360
+
response.Bundles.LastBundle = lastBundle
361
+
response.Bundles.StartTime = indexStats["start_time"].(time.Time)
362
+
response.Bundles.EndTime = indexStats["end_time"].(time.Time)
363
+
364
+
if firstMeta, err := index.GetBundle(firstBundle); err == nil {
365
+
response.Bundles.RootHash = firstMeta.Hash
366
+
}
367
+
368
+
if lastMeta, err := index.GetBundle(lastBundle); err == nil {
369
+
response.Bundles.HeadHash = lastMeta.Hash
370
+
response.Bundles.HeadAgeSeconds = int(time.Since(lastMeta.EndTime).Seconds())
371
+
}
372
+
373
+
if gaps, ok := indexStats["gaps"].(int); ok {
374
+
response.Bundles.Gaps = gaps
375
+
response.Bundles.HasGaps = gaps > 0
376
+
if gaps > 0 {
377
+
response.Bundles.GapNumbers = index.FindGaps()
378
+
}
379
+
}
380
+
381
+
totalOps := bundleCount * types.BUNDLE_SIZE
382
+
response.Bundles.TotalOperations = totalOps
383
+
384
+
duration := response.Bundles.EndTime.Sub(response.Bundles.StartTime)
385
+
if duration.Hours() > 0 {
386
+
response.Bundles.AvgOpsPerHour = int(float64(totalOps) / duration.Hours())
387
+
}
388
+
}
389
+
390
+
if s.config.SyncMode {
391
+
mempoolStats := s.manager.GetMempoolStats()
392
+
393
+
if count, ok := mempoolStats["count"].(int); ok {
394
+
mempool := &MempoolStatus{
395
+
Count: count,
396
+
TargetBundle: mempoolStats["target_bundle"].(int),
397
+
CanCreateBundle: mempoolStats["can_create_bundle"].(bool),
398
+
MinTimestamp: mempoolStats["min_timestamp"].(time.Time),
399
+
Validated: mempoolStats["validated"].(bool),
400
+
ProgressPercent: float64(count) / float64(types.BUNDLE_SIZE) * 100,
401
+
BundleSize: types.BUNDLE_SIZE,
402
+
OperationsNeeded: types.BUNDLE_SIZE - count,
403
+
}
404
+
405
+
if firstTime, ok := mempoolStats["first_time"].(time.Time); ok {
406
+
mempool.FirstTime = firstTime
407
+
mempool.TimespanSeconds = int(time.Since(firstTime).Seconds())
408
+
}
409
+
if lastTime, ok := mempoolStats["last_time"].(time.Time); ok {
410
+
mempool.LastTime = lastTime
411
+
mempool.LastOpAgeSeconds = int(time.Since(lastTime).Seconds())
412
+
}
413
+
414
+
if count > 100 && count < types.BUNDLE_SIZE {
415
+
if !mempool.FirstTime.IsZero() && !mempool.LastTime.IsZero() {
416
+
timespan := mempool.LastTime.Sub(mempool.FirstTime)
417
+
if timespan.Seconds() > 0 {
418
+
opsPerSec := float64(count) / timespan.Seconds()
419
+
remaining := types.BUNDLE_SIZE - count
420
+
mempool.EtaNextBundleSeconds = int(float64(remaining) / opsPerSec)
421
+
}
422
+
}
423
+
}
424
+
425
+
response.Mempool = mempool
426
+
}
427
+
}
428
+
429
+
sendJSON(w, 200, response)
430
+
}
431
+
}
432
+
433
+
func (s *Server) handleMempool() http.HandlerFunc {
434
+
return func(w http.ResponseWriter, r *http.Request) {
435
+
ops, err := s.manager.GetMempoolOperations()
436
+
if err != nil {
437
+
sendJSON(w, 500, map[string]string{"error": err.Error()})
438
+
return
439
+
}
440
+
441
+
w.Header().Set("Content-Type", "application/x-ndjson")
442
+
443
+
if len(ops) == 0 {
444
+
return
445
+
}
446
+
447
+
for _, op := range ops {
448
+
if len(op.RawJSON) > 0 {
449
+
w.Write(op.RawJSON)
450
+
} else {
451
+
data, _ := json.Marshal(op)
452
+
w.Write(data)
453
+
}
454
+
w.Write([]byte("\n"))
455
+
}
456
+
}
457
+
}
458
+
459
+
func (s *Server) handleDebugMemory() http.HandlerFunc {
460
+
return func(w http.ResponseWriter, r *http.Request) {
461
+
var m runtime.MemStats
462
+
runtime.ReadMemStats(&m)
463
+
464
+
didStats := s.manager.GetDIDIndexStats()
465
+
466
+
beforeAlloc := m.Alloc / 1024 / 1024
467
+
468
+
runtime.GC()
469
+
runtime.ReadMemStats(&m)
470
+
afterAlloc := m.Alloc / 1024 / 1024
471
+
472
+
response := fmt.Sprintf(`Memory Stats:
473
+
Alloc: %d MB
474
+
TotalAlloc: %d MB
475
+
Sys: %d MB
476
+
NumGC: %d
477
+
478
+
DID Index:
479
+
Cached shards: %d/%d
480
+
481
+
After GC:
482
+
Alloc: %d MB
483
+
`,
484
+
beforeAlloc,
485
+
m.TotalAlloc/1024/1024,
486
+
m.Sys/1024/1024,
487
+
m.NumGC,
488
+
didStats["cached_shards"],
489
+
didStats["cache_limit"],
490
+
afterAlloc)
491
+
492
+
w.Header().Set("Content-Type", "text/plain")
493
+
w.Write([]byte(response))
494
+
}
495
+
}
496
+
497
+
func (s *Server) handleDIDRouting(w http.ResponseWriter, r *http.Request) {
498
+
path := strings.TrimPrefix(r.URL.Path, "/")
499
+
500
+
parts := strings.SplitN(path, "/", 2)
501
+
did := parts[0]
502
+
503
+
if !strings.HasPrefix(did, "did:plc:") {
504
+
sendJSON(w, 404, map[string]string{"error": "not found"})
505
+
return
506
+
}
507
+
508
+
if len(parts) == 1 {
509
+
s.handleDIDDocument(did)(w, r)
510
+
} else if parts[1] == "data" {
511
+
s.handleDIDData(did)(w, r)
512
+
} else if parts[1] == "log/audit" {
513
+
s.handleDIDAuditLog(did)(w, r)
514
+
} else {
515
+
sendJSON(w, 404, map[string]string{"error": "not found"})
516
+
}
517
+
}
518
+
519
+
func (s *Server) handleDIDDocument(did string) http.HandlerFunc {
520
+
return func(w http.ResponseWriter, r *http.Request) {
521
+
op, err := s.manager.GetLatestDIDOperation(context.Background(), did)
522
+
if err != nil {
523
+
sendJSON(w, 500, map[string]string{"error": err.Error()})
524
+
return
525
+
}
526
+
527
+
doc, err := plcclient.ResolveDIDDocument(did, []plcclient.PLCOperation{*op})
528
+
if err != nil {
529
+
if strings.Contains(err.Error(), "deactivated") {
530
+
sendJSON(w, 410, map[string]string{"error": "DID has been deactivated"})
531
+
} else {
532
+
sendJSON(w, 500, map[string]string{"error": fmt.Sprintf("Resolution failed: %v", err)})
533
+
}
534
+
return
535
+
}
536
+
537
+
w.Header().Set("Content-Type", "application/did+ld+json")
538
+
sendJSON(w, 200, doc)
539
+
}
540
+
}
541
+
542
+
func (s *Server) handleDIDData(did string) http.HandlerFunc {
543
+
return func(w http.ResponseWriter, r *http.Request) {
544
+
if err := plcclient.ValidateDIDFormat(did); err != nil {
545
+
sendJSON(w, 400, map[string]string{"error": "Invalid DID format"})
546
+
return
547
+
}
548
+
549
+
operations, err := s.manager.GetDIDOperations(context.Background(), did, false)
550
+
if err != nil {
551
+
sendJSON(w, 500, map[string]string{"error": err.Error()})
552
+
return
553
+
}
554
+
555
+
if len(operations) == 0 {
556
+
sendJSON(w, 404, map[string]string{"error": "DID not found"})
557
+
return
558
+
}
559
+
560
+
state, err := plcclient.BuildDIDState(did, operations)
561
+
if err != nil {
562
+
if strings.Contains(err.Error(), "deactivated") {
563
+
sendJSON(w, 410, map[string]string{"error": "DID has been deactivated"})
564
+
} else {
565
+
sendJSON(w, 500, map[string]string{"error": err.Error()})
566
+
}
567
+
return
568
+
}
569
+
570
+
sendJSON(w, 200, state)
571
+
}
572
+
}
573
+
574
+
func (s *Server) handleDIDAuditLog(did string) http.HandlerFunc {
575
+
return func(w http.ResponseWriter, r *http.Request) {
576
+
if err := plcclient.ValidateDIDFormat(did); err != nil {
577
+
sendJSON(w, 400, map[string]string{"error": "Invalid DID format"})
578
+
return
579
+
}
580
+
581
+
operations, err := s.manager.GetDIDOperations(context.Background(), did, false)
582
+
if err != nil {
583
+
sendJSON(w, 500, map[string]string{"error": err.Error()})
584
+
return
585
+
}
586
+
587
+
if len(operations) == 0 {
588
+
sendJSON(w, 404, map[string]string{"error": "DID not found"})
589
+
return
590
+
}
591
+
592
+
auditLog := plcclient.FormatAuditLog(operations)
593
+
sendJSON(w, 200, auditLog)
594
+
}
595
+
}
+58
server/helpers.go
+58
server/helpers.go
···
···
1
+
package server
2
+
3
+
import (
4
+
"fmt"
5
+
"net/http"
6
+
)
7
+
8
+
// getScheme determines the HTTP scheme
9
+
func getScheme(r *http.Request) string {
10
+
if r.TLS != nil {
11
+
return "https"
12
+
}
13
+
14
+
if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" {
15
+
return proto
16
+
}
17
+
18
+
if r.Header.Get("X-Forwarded-Ssl") == "on" {
19
+
return "https"
20
+
}
21
+
22
+
return "http"
23
+
}
24
+
25
+
// getWSScheme determines the WebSocket scheme
26
+
func getWSScheme(r *http.Request) string {
27
+
if getScheme(r) == "https" {
28
+
return "wss"
29
+
}
30
+
return "ws"
31
+
}
32
+
33
+
// getBaseURL returns the base URL for HTTP
34
+
func getBaseURL(r *http.Request) string {
35
+
scheme := getScheme(r)
36
+
host := r.Host
37
+
return fmt.Sprintf("%s://%s", scheme, host)
38
+
}
39
+
40
+
// getWSURL returns the base URL for WebSocket
41
+
func getWSURL(r *http.Request) string {
42
+
scheme := getWSScheme(r)
43
+
host := r.Host
44
+
return fmt.Sprintf("%s://%s", scheme, host)
45
+
}
46
+
47
+
// formatNumber formats numbers with thousand separators
48
+
func formatNumber(n int) string {
49
+
s := fmt.Sprintf("%d", n)
50
+
var result []byte
51
+
for i, c := range s {
52
+
if i > 0 && (len(s)-i)%3 == 0 {
53
+
result = append(result, ',')
54
+
}
55
+
result = append(result, byte(c))
56
+
}
57
+
return string(result)
58
+
}
+52
server/middleware.go
+52
server/middleware.go
···
···
1
+
package server
2
+
3
+
import (
4
+
"net/http"
5
+
6
+
"github.com/goccy/go-json"
7
+
)
8
+
9
+
// corsMiddleware adds CORS headers
10
+
func corsMiddleware(next http.Handler) http.Handler {
11
+
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
12
+
// Skip CORS for WebSocket upgrade requests
13
+
if r.Header.Get("Upgrade") == "websocket" {
14
+
next.ServeHTTP(w, r)
15
+
return
16
+
}
17
+
18
+
// Normal CORS handling
19
+
w.Header().Set("Access-Control-Allow-Origin", "*")
20
+
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
21
+
22
+
if requestedHeaders := r.Header.Get("Access-Control-Request-Headers"); requestedHeaders != "" {
23
+
w.Header().Set("Access-Control-Allow-Headers", requestedHeaders)
24
+
} else {
25
+
w.Header().Set("Access-Control-Allow-Headers", "*")
26
+
}
27
+
28
+
w.Header().Set("Access-Control-Max-Age", "86400")
29
+
30
+
if r.Method == "OPTIONS" {
31
+
w.WriteHeader(204)
32
+
return
33
+
}
34
+
35
+
next.ServeHTTP(w, r)
36
+
})
37
+
}
38
+
39
+
// sendJSON sends a JSON response
40
+
func sendJSON(w http.ResponseWriter, statusCode int, data interface{}) {
41
+
w.Header().Set("Content-Type", "application/json")
42
+
43
+
jsonData, err := json.Marshal(data)
44
+
if err != nil {
45
+
w.WriteHeader(500)
46
+
w.Write([]byte(`{"error":"failed to marshal JSON"}`))
47
+
return
48
+
}
49
+
50
+
w.WriteHeader(statusCode)
51
+
w.Write(jsonData)
52
+
}
+108
server/server.go
+108
server/server.go
···
···
1
+
package server
2
+
3
+
import (
4
+
"context"
5
+
"net/http"
6
+
"time"
7
+
8
+
"tangled.org/atscan.net/plcbundle/internal/bundle"
9
+
)
10
+
11
+
// Server serves bundle data over HTTP
12
+
type Server struct {
13
+
manager *bundle.Manager
14
+
addr string
15
+
config *Config
16
+
startTime time.Time
17
+
httpServer *http.Server
18
+
}
19
+
20
+
// Config configures the server
21
+
type Config struct {
22
+
Addr string
23
+
SyncMode bool
24
+
SyncInterval time.Duration
25
+
EnableWebSocket bool
26
+
EnableResolver bool
27
+
Version string
28
+
}
29
+
30
+
// New creates a new HTTP server
31
+
func New(manager *bundle.Manager, config *Config) *Server {
32
+
if config.Version == "" {
33
+
config.Version = "dev"
34
+
}
35
+
36
+
s := &Server{
37
+
manager: manager,
38
+
addr: config.Addr,
39
+
config: config,
40
+
startTime: time.Now(),
41
+
}
42
+
43
+
handler := s.createHandler()
44
+
45
+
s.httpServer = &http.Server{
46
+
Addr: config.Addr,
47
+
Handler: handler,
48
+
}
49
+
50
+
return s
51
+
}
52
+
53
+
// ListenAndServe starts the HTTP server
54
+
func (s *Server) ListenAndServe() error {
55
+
return s.httpServer.ListenAndServe()
56
+
}
57
+
58
+
// Shutdown gracefully shuts down the server
59
+
func (s *Server) Shutdown(ctx context.Context) error {
60
+
return s.httpServer.Shutdown(ctx)
61
+
}
62
+
63
+
// createHandler creates the HTTP handler with all routes
64
+
func (s *Server) createHandler() http.Handler {
65
+
mux := http.NewServeMux()
66
+
67
+
// Specific routes first
68
+
mux.HandleFunc("GET /index.json", s.handleIndexJSON())
69
+
mux.HandleFunc("GET /bundle/{number}", s.handleBundle())
70
+
mux.HandleFunc("GET /data/{number}", s.handleBundleData())
71
+
mux.HandleFunc("GET /jsonl/{number}", s.handleBundleJSONL())
72
+
mux.HandleFunc("GET /status", s.handleStatus())
73
+
mux.HandleFunc("GET /debug/memory", s.handleDebugMemory())
74
+
75
+
// WebSocket
76
+
if s.config.EnableWebSocket {
77
+
mux.HandleFunc("GET /ws", s.handleWebSocket())
78
+
}
79
+
80
+
// Sync mode endpoints
81
+
if s.config.SyncMode {
82
+
mux.HandleFunc("GET /mempool", s.handleMempool())
83
+
}
84
+
85
+
// Root and DID resolver
86
+
mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) {
87
+
path := r.URL.Path
88
+
89
+
if path == "/" {
90
+
s.handleRoot()(w, r)
91
+
return
92
+
}
93
+
94
+
if s.config.EnableResolver {
95
+
s.handleDIDRouting(w, r)
96
+
return
97
+
}
98
+
99
+
sendJSON(w, 404, map[string]string{"error": "not found"})
100
+
})
101
+
102
+
return corsMiddleware(mux)
103
+
}
104
+
105
+
// GetStartTime returns when the server started
106
+
func (s *Server) GetStartTime() time.Time {
107
+
return s.startTime
108
+
}
+58
server/types.go
+58
server/types.go
···
···
1
+
package server
2
+
3
+
import "time"
4
+
5
+
// StatusResponse is the /status endpoint response
6
+
type StatusResponse struct {
7
+
Bundles BundleStatus `json:"bundles"`
8
+
Mempool *MempoolStatus `json:"mempool,omitempty"`
9
+
Server ServerStatus `json:"server"`
10
+
}
11
+
12
+
// ServerStatus contains server information
13
+
type ServerStatus struct {
14
+
Version string `json:"version"`
15
+
UptimeSeconds int `json:"uptime_seconds"`
16
+
SyncMode bool `json:"sync_mode"`
17
+
SyncIntervalSeconds int `json:"sync_interval_seconds,omitempty"`
18
+
WebSocketEnabled bool `json:"websocket_enabled"`
19
+
Origin string `json:"origin,omitempty"`
20
+
}
21
+
22
+
// BundleStatus contains bundle statistics
23
+
type BundleStatus struct {
24
+
Count int `json:"count"`
25
+
FirstBundle int `json:"first_bundle,omitempty"`
26
+
LastBundle int `json:"last_bundle,omitempty"`
27
+
TotalSize int64 `json:"total_size"`
28
+
UncompressedSize int64 `json:"uncompressed_size,omitempty"`
29
+
CompressionRatio float64 `json:"compression_ratio,omitempty"`
30
+
TotalOperations int `json:"total_operations,omitempty"`
31
+
AvgOpsPerHour int `json:"avg_ops_per_hour,omitempty"`
32
+
StartTime time.Time `json:"start_time,omitempty"`
33
+
EndTime time.Time `json:"end_time,omitempty"`
34
+
UpdatedAt time.Time `json:"updated_at"`
35
+
HeadAgeSeconds int `json:"head_age_seconds,omitempty"`
36
+
RootHash string `json:"root_hash,omitempty"`
37
+
HeadHash string `json:"head_hash,omitempty"`
38
+
Gaps int `json:"gaps,omitempty"`
39
+
HasGaps bool `json:"has_gaps"`
40
+
GapNumbers []int `json:"gap_numbers,omitempty"`
41
+
}
42
+
43
+
// MempoolStatus contains mempool statistics
44
+
type MempoolStatus struct {
45
+
Count int `json:"count"`
46
+
TargetBundle int `json:"target_bundle"`
47
+
CanCreateBundle bool `json:"can_create_bundle"`
48
+
MinTimestamp time.Time `json:"min_timestamp"`
49
+
Validated bool `json:"validated"`
50
+
ProgressPercent float64 `json:"progress_percent"`
51
+
BundleSize int `json:"bundle_size"`
52
+
OperationsNeeded int `json:"operations_needed"`
53
+
FirstTime time.Time `json:"first_time,omitempty"`
54
+
LastTime time.Time `json:"last_time,omitempty"`
55
+
TimespanSeconds int `json:"timespan_seconds,omitempty"`
56
+
LastOpAgeSeconds int `json:"last_op_age_seconds,omitempty"`
57
+
EtaNextBundleSeconds int `json:"eta_next_bundle_seconds,omitempty"`
58
+
}
+243
server/websocket.go
+243
server/websocket.go
···
···
1
+
package server
2
+
3
+
import (
4
+
"bufio"
5
+
"context"
6
+
"fmt"
7
+
"net/http"
8
+
"os"
9
+
"strconv"
10
+
"time"
11
+
12
+
"github.com/goccy/go-json"
13
+
"github.com/gorilla/websocket"
14
+
"tangled.org/atscan.net/plcbundle/internal/types"
15
+
"tangled.org/atscan.net/plcbundle/plcclient"
16
+
)
17
+
18
+
var upgrader = websocket.Upgrader{
19
+
ReadBufferSize: 1024,
20
+
WriteBufferSize: 1024,
21
+
CheckOrigin: func(r *http.Request) bool {
22
+
return true
23
+
},
24
+
}
25
+
26
+
func (s *Server) handleWebSocket() http.HandlerFunc {
27
+
return func(w http.ResponseWriter, r *http.Request) {
28
+
cursorStr := r.URL.Query().Get("cursor")
29
+
var cursor int
30
+
31
+
if cursorStr == "" {
32
+
cursor = s.manager.GetCurrentCursor()
33
+
} else {
34
+
var err error
35
+
cursor, err = strconv.Atoi(cursorStr)
36
+
if err != nil || cursor < 0 {
37
+
http.Error(w, "Invalid cursor: must be non-negative integer", 400)
38
+
return
39
+
}
40
+
}
41
+
42
+
conn, err := upgrader.Upgrade(w, r, nil)
43
+
if err != nil {
44
+
fmt.Fprintf(os.Stderr, "WebSocket upgrade failed: %v\n", err)
45
+
return
46
+
}
47
+
defer conn.Close()
48
+
49
+
conn.SetPongHandler(func(string) error {
50
+
conn.SetReadDeadline(time.Now().Add(60 * time.Second))
51
+
return nil
52
+
})
53
+
54
+
done := make(chan struct{})
55
+
56
+
go func() {
57
+
defer close(done)
58
+
for {
59
+
_, _, err := conn.ReadMessage()
60
+
if err != nil {
61
+
if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
62
+
fmt.Fprintf(os.Stderr, "WebSocket: client closed connection\n")
63
+
}
64
+
return
65
+
}
66
+
}
67
+
}()
68
+
69
+
bgCtx := context.Background()
70
+
71
+
if err := s.streamLive(bgCtx, conn, cursor, done); err != nil {
72
+
fmt.Fprintf(os.Stderr, "WebSocket stream error: %v\n", err)
73
+
}
74
+
}
75
+
}
76
+
77
+
func (s *Server) streamLive(ctx context.Context, conn *websocket.Conn, startCursor int, done chan struct{}) error {
78
+
index := s.manager.GetIndex()
79
+
bundles := index.GetBundles()
80
+
currentRecord := startCursor
81
+
82
+
// Stream existing bundles
83
+
if len(bundles) > 0 {
84
+
startBundleIdx := startCursor / types.BUNDLE_SIZE
85
+
startPosition := startCursor % types.BUNDLE_SIZE
86
+
87
+
if startBundleIdx < len(bundles) {
88
+
for i := startBundleIdx; i < len(bundles); i++ {
89
+
skipUntil := 0
90
+
if i == startBundleIdx {
91
+
skipUntil = startPosition
92
+
}
93
+
94
+
newRecordCount, err := s.streamBundle(ctx, conn, bundles[i].BundleNumber, skipUntil, done)
95
+
if err != nil {
96
+
return err
97
+
}
98
+
currentRecord += newRecordCount
99
+
}
100
+
}
101
+
}
102
+
103
+
lastSeenMempoolCount := 0
104
+
if err := s.streamMempool(conn, startCursor, len(bundles)*types.BUNDLE_SIZE, ¤tRecord, &lastSeenMempoolCount, done); err != nil {
105
+
return err
106
+
}
107
+
108
+
ticker := time.NewTicker(500 * time.Millisecond)
109
+
defer ticker.Stop()
110
+
111
+
lastBundleCount := len(bundles)
112
+
113
+
for {
114
+
select {
115
+
case <-done:
116
+
return nil
117
+
118
+
case <-ticker.C:
119
+
index = s.manager.GetIndex()
120
+
bundles = index.GetBundles()
121
+
122
+
if len(bundles) > lastBundleCount {
123
+
newBundleCount := len(bundles) - lastBundleCount
124
+
currentRecord += newBundleCount * types.BUNDLE_SIZE
125
+
lastBundleCount = len(bundles)
126
+
lastSeenMempoolCount = 0
127
+
}
128
+
129
+
if err := s.streamMempool(conn, startCursor, len(bundles)*types.BUNDLE_SIZE, ¤tRecord, &lastSeenMempoolCount, done); err != nil {
130
+
return err
131
+
}
132
+
133
+
if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil {
134
+
return err
135
+
}
136
+
}
137
+
}
138
+
}
139
+
140
+
func (s *Server) streamBundle(ctx context.Context, conn *websocket.Conn, bundleNumber int, skipUntil int, done chan struct{}) (int, error) {
141
+
reader, err := s.manager.StreamBundleDecompressed(ctx, bundleNumber)
142
+
if err != nil {
143
+
return 0, nil
144
+
}
145
+
defer reader.Close()
146
+
147
+
scanner := bufio.NewScanner(reader)
148
+
buf := make([]byte, 0, 64*1024)
149
+
scanner.Buffer(buf, 1024*1024)
150
+
151
+
position := 0
152
+
streamed := 0
153
+
154
+
for scanner.Scan() {
155
+
line := scanner.Bytes()
156
+
if len(line) == 0 {
157
+
continue
158
+
}
159
+
160
+
if position < skipUntil {
161
+
position++
162
+
continue
163
+
}
164
+
165
+
select {
166
+
case <-done:
167
+
return streamed, nil
168
+
default:
169
+
}
170
+
171
+
if err := conn.WriteMessage(websocket.TextMessage, line); err != nil {
172
+
return streamed, err
173
+
}
174
+
175
+
position++
176
+
streamed++
177
+
178
+
if streamed%1000 == 0 {
179
+
conn.WriteMessage(websocket.PingMessage, nil)
180
+
}
181
+
}
182
+
183
+
if err := scanner.Err(); err != nil {
184
+
return streamed, fmt.Errorf("scanner error on bundle %d: %w", bundleNumber, err)
185
+
}
186
+
187
+
return streamed, nil
188
+
}
189
+
190
+
func (s *Server) streamMempool(conn *websocket.Conn, startCursor int, bundleRecordBase int, currentRecord *int, lastSeenCount *int, done chan struct{}) error {
191
+
mempoolOps, err := s.manager.GetMempoolOperations()
192
+
if err != nil {
193
+
return nil
194
+
}
195
+
196
+
if len(mempoolOps) <= *lastSeenCount {
197
+
return nil
198
+
}
199
+
200
+
for i := *lastSeenCount; i < len(mempoolOps); i++ {
201
+
recordNum := bundleRecordBase + i
202
+
if recordNum < startCursor {
203
+
continue
204
+
}
205
+
206
+
select {
207
+
case <-done:
208
+
return nil
209
+
default:
210
+
}
211
+
212
+
if err := sendOperation(conn, mempoolOps[i]); err != nil {
213
+
return err
214
+
}
215
+
*currentRecord++
216
+
}
217
+
218
+
*lastSeenCount = len(mempoolOps)
219
+
return nil
220
+
}
221
+
222
+
func sendOperation(conn *websocket.Conn, op plcclient.PLCOperation) error {
223
+
var data []byte
224
+
var err error
225
+
226
+
if len(op.RawJSON) > 0 {
227
+
data = op.RawJSON
228
+
} else {
229
+
data, err = json.Marshal(op)
230
+
if err != nil {
231
+
return nil
232
+
}
233
+
}
234
+
235
+
if err := conn.WriteMessage(websocket.TextMessage, data); err != nil {
236
+
if !websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
237
+
fmt.Fprintf(os.Stderr, "WebSocket write error: %v\n", err)
238
+
}
239
+
return err
240
+
}
241
+
242
+
return nil
243
+
}
+46
types.go
+46
types.go
···
···
1
+
package plcbundle
2
+
3
+
import (
4
+
"time"
5
+
6
+
"tangled.org/atscan.net/plcbundle/plcclient"
7
+
)
8
+
9
+
// Bundle represents a PLC bundle (public version)
10
+
type Bundle struct {
11
+
BundleNumber int
12
+
StartTime time.Time
13
+
EndTime time.Time
14
+
Operations []plcclient.PLCOperation
15
+
DIDCount int
16
+
Hash string
17
+
CompressedSize int64
18
+
UncompressedSize int64
19
+
}
20
+
21
+
// BundleInfo provides metadata about a bundle
22
+
type BundleInfo struct {
23
+
BundleNumber int `json:"bundle_number"`
24
+
StartTime time.Time `json:"start_time"`
25
+
EndTime time.Time `json:"end_time"`
26
+
OperationCount int `json:"operation_count"`
27
+
DIDCount int `json:"did_count"`
28
+
Hash string `json:"hash"`
29
+
CompressedSize int64 `json:"compressed_size"`
30
+
UncompressedSize int64 `json:"uncompressed_size"`
31
+
}
32
+
33
+
// IndexStats provides statistics about the bundle index
34
+
type IndexStats struct {
35
+
BundleCount int
36
+
FirstBundle int
37
+
LastBundle int
38
+
TotalSize int64
39
+
MissingBundles []int
40
+
}
41
+
42
+
// Helper to convert internal bundle to public
43
+
func toBundlePublic(b interface{}) *Bundle {
44
+
// Implement conversion from internal bundle to public Bundle
45
+
return &Bundle{} // placeholder
46
+
}