bnewbold.net
go scratch code for atproto
refactor some duplicated glot code
+356
-333
+3
-3
cmd/glot/breaking.go
+3
-3
cmd/glot/breaking.go
···
22
22
Flags: []cli.Flag{
23
23
&cli.StringFlag{
24
24
Name: "lexicons-dir",
25
-
Value: "./lexicons/",
25
+
Value: "lexicons/",
26
26
Usage: "base directory for project Lexicon files",
27
27
Sources: cli.EnvVars("LEXICONS_DIR"),
28
28
},
···
35
35
}
36
36
37
37
func runBreaking(ctx context.Context, cmd *cli.Command) error {
38
-
return compareSchemas(ctx, cmd, breakingCompare)
38
+
return runComparisons(ctx, cmd, compareBreaking)
39
39
}
40
40
41
-
func breakingCompare(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error {
41
+
func compareBreaking(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error {
42
42
43
43
// skip schemas which aren't in both locations
44
44
if localJSON == nil || remoteJSON == nil {
+3
-3
cmd/glot/diff.go
+3
-3
cmd/glot/diff.go
···
21
21
Flags: []cli.Flag{
22
22
&cli.StringFlag{
23
23
Name: "lexicons-dir",
24
-
Value: "./lexicons/",
24
+
Value: "lexicons/",
25
25
Usage: "base directory for project Lexicon files",
26
26
Sources: cli.EnvVars("LEXICONS_DIR"),
27
27
},
···
30
30
}
31
31
32
32
func runDiff(ctx context.Context, cmd *cli.Command) error {
33
-
return compareSchemas(ctx, cmd, diffCompare)
33
+
return runComparisons(ctx, cmd, compareDiff)
34
34
}
35
35
36
-
func diffCompare(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error {
36
+
func compareDiff(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error {
37
37
38
38
// skip schemas which aren't in both locations
39
39
if localJSON == nil || remoteJSON == nil {
+4
-43
cmd/glot/dns.go
cmd/glot/check_dns.go
+4
-43
cmd/glot/dns.go
cmd/glot/check_dns.go
···
2
2
3
3
import (
4
4
"context"
5
-
"encoding/json"
6
5
"fmt"
7
-
"io/fs"
8
-
"os"
9
-
"path"
10
-
"path/filepath"
11
6
"sort"
12
7
13
8
"github.com/bluesky-social/indigo/atproto/identity"
···
24
19
Flags: []cli.Flag{
25
20
&cli.StringFlag{
26
21
Name: "lexicons-dir",
27
-
Value: "./lexicons/",
22
+
Value: "lexicons/",
28
23
Usage: "base directory for project Lexicon files",
29
24
Sources: cli.EnvVars("LEXICONS_DIR"),
30
25
},
···
44
39
*/
45
40
func runCheckDNS(ctx context.Context, cmd *cli.Command) error {
46
41
47
-
paths := cmd.Args().Slice()
48
-
if !cmd.Args().Present() {
49
-
paths = []string{cmd.String("lexicons-dir")}
50
-
_, err := os.Stat(paths[0])
51
-
if err != nil {
52
-
return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err)
53
-
}
54
-
}
55
-
56
42
// collect all NSID/path mappings
57
-
localSchemas := map[syntax.NSID]json.RawMessage{}
58
-
59
-
for _, p := range paths {
60
-
finfo, err := os.Stat(p)
61
-
if err != nil {
62
-
return fmt.Errorf("failed loading %s: %w", p, err)
63
-
}
64
-
if finfo.IsDir() {
65
-
if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error {
66
-
if d.IsDir() || path.Ext(fp) != ".json" {
67
-
return nil
68
-
}
69
-
nsid, rec, err := loadSchemaPath(fp)
70
-
if err != nil {
71
-
return err
72
-
}
73
-
localSchemas[nsid] = *rec
74
-
return nil
75
-
}); err != nil {
76
-
return err
77
-
}
78
-
continue
79
-
}
80
-
nsid, rec, err := loadSchemaPath(p)
81
-
if err != nil {
82
-
return err
83
-
}
84
-
localSchemas[nsid] = *rec
43
+
localSchemas, err := collectSchemaJSON(cmd)
44
+
if err != nil {
45
+
return err
85
46
}
86
47
87
48
localGroups := map[string]bool{}
+8
-32
cmd/glot/lint.go
+8
-32
cmd/glot/lint.go
···
6
6
"encoding/json"
7
7
"errors"
8
8
"fmt"
9
-
"io/fs"
10
9
"log/slog"
11
10
"os"
12
-
"path"
13
-
"path/filepath"
14
11
"regexp"
15
12
"slices"
16
13
···
33
30
Flags: []cli.Flag{
34
31
&cli.StringFlag{
35
32
Name: "lexicons-dir",
36
-
Value: "./lexicons/",
33
+
Value: "lexicons/",
37
34
Usage: "base directory for project Lexicon files",
38
35
Sources: cli.EnvVars("LEXICONS_DIR"),
39
36
},
···
46
43
}
47
44
48
45
func runLint(ctx context.Context, cmd *cli.Command) error {
49
-
paths := cmd.Args().Slice()
50
-
if !cmd.Args().Present() {
51
-
paths = []string{cmd.String("lexicons-dir")}
52
-
_, err := os.Stat(paths[0])
53
-
if err != nil {
54
-
return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err)
55
-
}
46
+
47
+
// enumerate lexicon JSON file paths
48
+
filePaths, err := collectPaths(cmd)
49
+
if err != nil {
50
+
return err
56
51
}
57
52
58
53
// TODO: load up entire directory in to a catalog? or have a "linter" struct?
59
54
60
55
slog.Debug("starting lint run")
61
56
anyFailures := false
62
-
for _, p := range paths {
63
-
finfo, err := os.Stat(p)
57
+
for _, fp := range filePaths {
58
+
err = lintFilePath(ctx, cmd, fp)
64
59
if err != nil {
65
-
return fmt.Errorf("failed loading %s: %w", p, err)
66
-
}
67
-
if finfo.IsDir() {
68
-
if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error {
69
-
if d.IsDir() || path.Ext(fp) != ".json" {
70
-
return nil
71
-
}
72
-
err = lintFilePath(ctx, cmd, fp)
73
-
if err == ErrLintFailures {
74
-
anyFailures = true
75
-
return nil
76
-
}
77
-
return err
78
-
}); err != nil {
79
-
return err
80
-
}
81
-
continue
82
-
}
83
-
if err := lintFilePath(ctx, cmd, p); err != nil {
84
60
if err == ErrLintFailures {
85
61
anyFailures = true
86
62
} else {
+1
-1
cmd/glot/new.go
+1
-1
cmd/glot/new.go
+6
-44
cmd/glot/publish.go
+6
-44
cmd/glot/publish.go
···
4
4
"context"
5
5
"encoding/json"
6
6
"fmt"
7
-
"io/fs"
8
-
"os"
9
-
"path"
10
-
"path/filepath"
11
7
"reflect"
12
8
"sort"
13
9
···
28
24
Flags: []cli.Flag{
29
25
&cli.StringFlag{
30
26
Name: "lexicons-dir",
31
-
Value: "./lexicons/",
27
+
Value: "lexicons/",
32
28
Usage: "base directory for project Lexicon files",
33
29
Sources: cli.EnvVars("LEXICONS_DIR"),
34
30
},
···
86
82
return fmt.Errorf("require API client to have DID configured")
87
83
}
88
84
89
-
paths := cmd.Args().Slice()
90
-
if !cmd.Args().Present() {
91
-
paths = []string{cmd.String("lexicons-dir")}
92
-
_, err := os.Stat(paths[0])
93
-
if err != nil {
94
-
return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err)
95
-
}
96
-
}
97
-
98
85
// collect all NSID/path mappings
99
-
localSchemas := map[syntax.NSID]json.RawMessage{}
100
-
remoteSchemas := map[syntax.NSID]json.RawMessage{}
101
-
102
-
for _, p := range paths {
103
-
finfo, err := os.Stat(p)
104
-
if err != nil {
105
-
return fmt.Errorf("failed loading %s: %w", p, err)
106
-
}
107
-
if finfo.IsDir() {
108
-
if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error {
109
-
if d.IsDir() || path.Ext(fp) != ".json" {
110
-
return nil
111
-
}
112
-
nsid, rec, err := loadSchemaPath(fp)
113
-
if err != nil {
114
-
return err
115
-
}
116
-
localSchemas[nsid] = *rec
117
-
return nil
118
-
}); err != nil {
119
-
return err
120
-
}
121
-
continue
122
-
}
123
-
nsid, rec, err := loadSchemaPath(p)
124
-
if err != nil {
125
-
return err
126
-
}
127
-
localSchemas[nsid] = *rec
86
+
localSchemas, err := collectSchemaJSON(cmd)
87
+
if err != nil {
88
+
return err
128
89
}
90
+
remoteSchemas := map[syntax.NSID]json.RawMessage{}
129
91
130
92
localGroups := map[string]bool{}
131
93
allNSIDMap := map[syntax.NSID]bool{}
···
136
98
}
137
99
138
100
for g := range localGroups {
139
-
if err := fetchLexiconGroup(ctx, cmd, g, &remoteSchemas); err != nil {
101
+
if err := resolveLexiconGroup(ctx, cmd, g, &remoteSchemas); err != nil {
140
102
return err
141
103
}
142
104
}
+1
-1
cmd/glot/pull.go
+1
-1
cmd/glot/pull.go
+3
-191
cmd/glot/status.go
+3
-191
cmd/glot/status.go
···
4
4
"context"
5
5
"encoding/json"
6
6
"fmt"
7
-
"io/fs"
8
-
"log/slog"
9
-
"os"
10
-
"path"
11
-
"path/filepath"
12
7
"reflect"
13
-
"sort"
14
8
15
-
"github.com/bluesky-social/indigo/api/agnostic"
16
-
"github.com/bluesky-social/indigo/atproto/atclient"
17
9
"github.com/bluesky-social/indigo/atproto/atdata"
18
-
"github.com/bluesky-social/indigo/atproto/identity"
19
-
"github.com/bluesky-social/indigo/atproto/lexicon"
20
10
"github.com/bluesky-social/indigo/atproto/syntax"
21
11
22
12
"github.com/urfave/cli/v3"
···
30
20
Flags: []cli.Flag{
31
21
&cli.StringFlag{
32
22
Name: "lexicons-dir",
33
-
Value: "./lexicons/",
23
+
Value: "lexicons/",
34
24
Usage: "base directory for project Lexicon files",
35
25
Sources: cli.EnvVars("LEXICONS_DIR"),
36
26
},
···
38
28
Action: runStatus,
39
29
}
40
30
41
-
func loadSchemaPath(fpath string) (syntax.NSID, *json.RawMessage, error) {
42
-
b, err := os.ReadFile(fpath)
43
-
if err != nil {
44
-
return "", nil, err
45
-
}
46
-
47
-
// parse file to check for errors
48
-
// TODO: use json/v2 when available for case-sensitivity
49
-
var sf lexicon.SchemaFile
50
-
err = json.Unmarshal(b, &sf)
51
-
if err == nil {
52
-
err = sf.FinishParse()
53
-
}
54
-
if err == nil {
55
-
err = sf.CheckSchema()
56
-
}
57
-
if err != nil {
58
-
return "", nil, err
59
-
}
60
-
61
-
var rec json.RawMessage
62
-
if err := json.Unmarshal(b, &rec); err != nil {
63
-
return "", nil, err
64
-
}
65
-
return syntax.NSID(sf.ID), &rec, nil
66
-
}
67
-
68
31
func runStatus(ctx context.Context, cmd *cli.Command) error {
69
-
return compareSchemas(ctx, cmd, statusCompare)
32
+
return runComparisons(ctx, cmd, compareStatus)
70
33
}
71
34
72
-
func statusCompare(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error {
35
+
func compareStatus(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error {
73
36
74
37
// new remote schema (missing local)
75
38
if localJSON == nil {
···
100
63
}
101
64
return nil
102
65
}
103
-
104
-
func compareSchemas(ctx context.Context, cmd *cli.Command, comp func(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error) error {
105
-
paths := cmd.Args().Slice()
106
-
if !cmd.Args().Present() {
107
-
paths = []string{cmd.String("lexicons-dir")}
108
-
_, err := os.Stat(paths[0])
109
-
if err != nil {
110
-
return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err)
111
-
}
112
-
}
113
-
114
-
// collect all NSID/path mappings
115
-
localSchemas := map[syntax.NSID]json.RawMessage{}
116
-
remoteSchemas := map[syntax.NSID]json.RawMessage{}
117
-
118
-
for _, p := range paths {
119
-
finfo, err := os.Stat(p)
120
-
if err != nil {
121
-
return fmt.Errorf("failed loading %s: %w", p, err)
122
-
}
123
-
if finfo.IsDir() {
124
-
if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error {
125
-
if d.IsDir() || path.Ext(fp) != ".json" {
126
-
return nil
127
-
}
128
-
nsid, rec, err := loadSchemaPath(fp)
129
-
if err != nil {
130
-
return err
131
-
}
132
-
localSchemas[nsid] = *rec
133
-
return nil
134
-
}); err != nil {
135
-
return err
136
-
}
137
-
continue
138
-
}
139
-
nsid, rec, err := loadSchemaPath(p)
140
-
if err != nil {
141
-
return err
142
-
}
143
-
localSchemas[nsid] = *rec
144
-
}
145
-
146
-
localGroups := map[string]bool{}
147
-
allNSIDMap := map[syntax.NSID]bool{}
148
-
for k := range localSchemas {
149
-
g := nsidGroup(k)
150
-
localGroups[g] = true
151
-
allNSIDMap[k] = true
152
-
}
153
-
154
-
for g := range localGroups {
155
-
if err := fetchLexiconGroup(ctx, cmd, g, &remoteSchemas); err != nil {
156
-
return err
157
-
}
158
-
}
159
-
160
-
for k := range remoteSchemas {
161
-
allNSIDMap[k] = true
162
-
}
163
-
allNSID := []string{}
164
-
for k := range allNSIDMap {
165
-
allNSID = append(allNSID, string(k))
166
-
}
167
-
sort.Strings(allNSID)
168
-
169
-
anyFailures := false
170
-
for _, k := range allNSID {
171
-
nsid := syntax.NSID(k)
172
-
if err := comp(ctx, cmd, nsid, localSchemas[nsid], remoteSchemas[nsid]); err != nil {
173
-
if err != ErrLintFailures {
174
-
return err
175
-
}
176
-
anyFailures = true
177
-
}
178
-
}
179
-
180
-
if anyFailures {
181
-
return ErrLintFailures
182
-
}
183
-
return nil
184
-
}
185
-
186
-
func fetchLexiconGroup(ctx context.Context, cmd *cli.Command, group string, remote *map[syntax.NSID]json.RawMessage) error {
187
-
188
-
slog.Debug("trying to load NSID group", "group", group)
189
-
190
-
// TODO: netclient support for listing records
191
-
dir := identity.BaseDirectory{}
192
-
did, err := dir.ResolveNSID(ctx, syntax.NSID(group+"name"))
193
-
if err != nil {
194
-
// if NSID isn't registered, just skip comparison
195
-
slog.Debug("skipping NSID pattern which did not resolve", "group", group)
196
-
return nil
197
-
}
198
-
ident, err := dir.LookupDID(ctx, did)
199
-
if err != nil {
200
-
return err
201
-
}
202
-
c := atclient.NewAPIClient(ident.PDSEndpoint())
203
-
204
-
cursor := ""
205
-
for {
206
-
// collection string, cursor string, limit int64, repo string, reverse bool
207
-
resp, err := agnostic.RepoListRecords(ctx, c, schemaNSID.String(), cursor, 100, ident.DID.String(), false)
208
-
if err != nil {
209
-
return err
210
-
}
211
-
for _, rec := range resp.Records {
212
-
aturi, err := syntax.ParseATURI(rec.Uri)
213
-
if err != nil {
214
-
return err
215
-
}
216
-
nsid, err := syntax.ParseNSID(aturi.RecordKey().String())
217
-
if err != nil {
218
-
slog.Warn("ignoring invalid schema NSID", "did", ident.DID, "rkey", aturi.RecordKey())
219
-
continue
220
-
}
221
-
if nsidGroup(nsid) != group {
222
-
// ignoring other NSIDs
223
-
continue
224
-
}
225
-
if rec.Value == nil {
226
-
return fmt.Errorf("missing record value: %s", nsid)
227
-
}
228
-
229
-
// parse file to check for errors
230
-
// TODO: use json/v2 when available for case-sensitivity
231
-
var sf lexicon.SchemaFile
232
-
err = json.Unmarshal(*rec.Value, &sf)
233
-
if err == nil {
234
-
err = sf.FinishParse()
235
-
}
236
-
if err == nil {
237
-
err = sf.CheckSchema()
238
-
}
239
-
if err != nil {
240
-
return fmt.Errorf("invalid lexicon schema record (%s): %w", nsid, err)
241
-
}
242
-
243
-
(*remote)[nsid] = *rec.Value
244
-
245
-
}
246
-
if resp.Cursor != nil && *resp.Cursor != "" {
247
-
cursor = *resp.Cursor
248
-
} else {
249
-
break
250
-
}
251
-
}
252
-
return nil
253
-
}
-15
cmd/glot/util.go
-15
cmd/glot/util.go
···
2
2
3
3
import (
4
4
"fmt"
5
-
"path"
6
5
"strings"
7
6
8
7
"github.com/bluesky-social/indigo/atproto/syntax"
9
-
10
-
"github.com/urfave/cli/v3"
11
8
)
12
9
13
10
var (
···
34
31
}
35
32
return raw, nil
36
33
}
37
-
38
-
func pathForNSID(cmd *cli.Command, nsid syntax.NSID) string {
39
-
40
-
odir := cmd.String("output-dir")
41
-
if odir != "" {
42
-
return path.Join(odir, nsid.Name()+".json")
43
-
}
44
-
45
-
base := cmd.String("lexicons-dir")
46
-
sub := strings.ReplaceAll(nsid.String(), ".", "/")
47
-
return path.Join(base, sub+".json")
48
-
}
+60
cmd/glot/util_compare.go
+60
cmd/glot/util_compare.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"sort"
7
+
8
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
+
10
+
"github.com/urfave/cli/v3"
11
+
)
12
+
13
+
func runComparisons(ctx context.Context, cmd *cli.Command, comp func(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error) error {
14
+
15
+
// collect all NSID/path mappings
16
+
localSchemas, err := collectSchemaJSON(cmd)
17
+
if err != nil {
18
+
return err
19
+
}
20
+
remoteSchemas := map[syntax.NSID]json.RawMessage{}
21
+
22
+
localGroups := map[string]bool{}
23
+
allNSIDMap := map[syntax.NSID]bool{}
24
+
for k := range localSchemas {
25
+
g := nsidGroup(k)
26
+
localGroups[g] = true
27
+
allNSIDMap[k] = true
28
+
}
29
+
30
+
for g := range localGroups {
31
+
if err := resolveLexiconGroup(ctx, cmd, g, &remoteSchemas); err != nil {
32
+
return err
33
+
}
34
+
}
35
+
36
+
for k := range remoteSchemas {
37
+
allNSIDMap[k] = true
38
+
}
39
+
allNSID := []string{}
40
+
for k := range allNSIDMap {
41
+
allNSID = append(allNSID, string(k))
42
+
}
43
+
sort.Strings(allNSID)
44
+
45
+
anyFailures := false
46
+
for _, k := range allNSID {
47
+
nsid := syntax.NSID(k)
48
+
if err := comp(ctx, cmd, nsid, localSchemas[nsid], remoteSchemas[nsid]); err != nil {
49
+
if err != ErrLintFailures {
50
+
return err
51
+
}
52
+
anyFailures = true
53
+
}
54
+
}
55
+
56
+
if anyFailures {
57
+
return ErrLintFailures
58
+
}
59
+
return nil
60
+
}
+86
cmd/glot/util_fetch.go
+86
cmd/glot/util_fetch.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"log/slog"
8
+
9
+
"github.com/bluesky-social/indigo/api/agnostic"
10
+
"github.com/bluesky-social/indigo/atproto/atclient"
11
+
"github.com/bluesky-social/indigo/atproto/identity"
12
+
"github.com/bluesky-social/indigo/atproto/lexicon"
13
+
"github.com/bluesky-social/indigo/atproto/syntax"
14
+
15
+
"github.com/urfave/cli/v3"
16
+
)
17
+
18
+
// helper which resolves and fetches all lexicon schemas (as JSON), storing them in provided map
19
+
func resolveLexiconGroup(ctx context.Context, cmd *cli.Command, group string, remote *map[syntax.NSID]json.RawMessage) error {
20
+
21
+
slog.Debug("resolving schemas for NSID group", "group", group)
22
+
23
+
// TODO: netclient support for listing records
24
+
dir := identity.BaseDirectory{}
25
+
did, err := dir.ResolveNSID(ctx, syntax.NSID(group+"name"))
26
+
if err != nil {
27
+
// if NSID isn't registered, just skip comparison
28
+
slog.Warn("skipping NSID pattern which did not resolve", "group", group)
29
+
return nil
30
+
}
31
+
ident, err := dir.LookupDID(ctx, did)
32
+
if err != nil {
33
+
return err
34
+
}
35
+
c := atclient.NewAPIClient(ident.PDSEndpoint())
36
+
37
+
cursor := ""
38
+
for {
39
+
// collection string, cursor string, limit int64, repo string, reverse bool
40
+
resp, err := agnostic.RepoListRecords(ctx, c, schemaNSID.String(), cursor, 100, ident.DID.String(), false)
41
+
if err != nil {
42
+
return err
43
+
}
44
+
for _, rec := range resp.Records {
45
+
aturi, err := syntax.ParseATURI(rec.Uri)
46
+
if err != nil {
47
+
return err
48
+
}
49
+
nsid, err := syntax.ParseNSID(aturi.RecordKey().String())
50
+
if err != nil {
51
+
slog.Warn("ignoring invalid schema NSID", "did", ident.DID, "rkey", aturi.RecordKey())
52
+
continue
53
+
}
54
+
if nsidGroup(nsid) != group {
55
+
// ignoring other NSIDs
56
+
continue
57
+
}
58
+
if rec.Value == nil {
59
+
return fmt.Errorf("missing record value: %s", nsid)
60
+
}
61
+
62
+
// parse file to check for errors
63
+
// TODO: use json/v2 when available for case-sensitivity
64
+
var sf lexicon.SchemaFile
65
+
err = json.Unmarshal(*rec.Value, &sf)
66
+
if err == nil {
67
+
err = sf.FinishParse()
68
+
}
69
+
if err == nil {
70
+
err = sf.CheckSchema()
71
+
}
72
+
if err != nil {
73
+
return fmt.Errorf("invalid lexicon schema record (%s): %w", nsid, err)
74
+
}
75
+
76
+
(*remote)[nsid] = *rec.Value
77
+
78
+
}
79
+
if resp.Cursor != nil && *resp.Cursor != "" {
80
+
cursor = *resp.Cursor
81
+
} else {
82
+
break
83
+
}
84
+
}
85
+
return nil
86
+
}
+181
cmd/glot/util_files.go
+181
cmd/glot/util_files.go
···
1
+
package main
2
+
3
+
import (
4
+
"encoding/json"
5
+
"fmt"
6
+
"io"
7
+
"io/fs"
8
+
"os"
9
+
"path"
10
+
"path/filepath"
11
+
"sort"
12
+
"strings"
13
+
14
+
"github.com/bluesky-social/indigo/atproto/lexicon"
15
+
"github.com/bluesky-social/indigo/atproto/syntax"
16
+
17
+
"github.com/urfave/cli/v3"
18
+
)
19
+
20
+
func pathForNSID(cmd *cli.Command, nsid syntax.NSID) string {
21
+
22
+
odir := cmd.String("output-dir")
23
+
if odir != "" {
24
+
return path.Join(odir, nsid.Name()+".json")
25
+
}
26
+
27
+
base := cmd.String("lexicons-dir")
28
+
sub := strings.ReplaceAll(nsid.String(), ".", "/")
29
+
return path.Join(base, sub+".json")
30
+
}
31
+
32
+
// parses through directories and files provided as CLI args, and returns a list of recursively enumerated .json files
33
+
func collectPaths(cmd *cli.Command) ([]string, error) {
34
+
35
+
paths := cmd.Args().Slice()
36
+
if !cmd.Args().Present() {
37
+
paths = []string{cmd.String("lexicons-dir")}
38
+
_, err := os.Stat(paths[0])
39
+
if err != nil {
40
+
return nil, fmt.Errorf("no path arguments specified and default lexicon directory not found")
41
+
}
42
+
}
43
+
44
+
filePaths := []string{}
45
+
46
+
for _, p := range paths {
47
+
finfo, err := os.Stat(p)
48
+
if err != nil {
49
+
return nil, fmt.Errorf("failed reading path %s: %w", p, err)
50
+
}
51
+
if finfo.IsDir() {
52
+
if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error {
53
+
if d.IsDir() || path.Ext(fp) != ".json" {
54
+
return nil
55
+
}
56
+
filePaths = append(filePaths, fp)
57
+
return nil
58
+
}); err != nil {
59
+
return nil, err
60
+
}
61
+
continue
62
+
}
63
+
filePaths = append(filePaths, p)
64
+
}
65
+
66
+
sort.Strings(filePaths)
67
+
return filePaths, nil
68
+
}
69
+
70
+
// parses through directories and files provided as CLI args, and returns broadly inclusive lexicon catalog.
71
+
//
72
+
// includes 'lexicons-dir', which may be broader than collectPaths() would return
73
+
func collectCatalog(cmd *cli.Command) (lexicon.Catalog, error) {
74
+
75
+
cat := lexicon.NewBaseCatalog()
76
+
77
+
lexDir := cmd.String("lexicons-dir")
78
+
paths := cmd.Args().Slice()
79
+
if !cmd.Args().Present() {
80
+
_, err := os.Stat(lexDir)
81
+
if err != nil {
82
+
return nil, fmt.Errorf("no path arguments specified and default lexicon directory not found")
83
+
}
84
+
}
85
+
86
+
// load lexicon dir (recursively)
87
+
ldinfo, err := os.Stat(lexDir)
88
+
if err == nil && ldinfo.IsDir() {
89
+
if err := cat.LoadDirectory(lexDir); err != nil {
90
+
return nil, err
91
+
}
92
+
}
93
+
94
+
for _, p := range paths {
95
+
96
+
if strings.HasPrefix(p, lexDir) {
97
+
// if path is under lexdir, we have already loaded, so skip
98
+
// NOTE: this isn't particularly reliable
99
+
continue
100
+
}
101
+
102
+
finfo, err := os.Stat(p)
103
+
if err != nil {
104
+
return nil, fmt.Errorf("failed reading path %s: %w", p, err)
105
+
}
106
+
if finfo.IsDir() {
107
+
if p != lexDir {
108
+
if err := cat.LoadDirectory(p); err != nil {
109
+
return nil, err
110
+
}
111
+
}
112
+
continue
113
+
}
114
+
if !finfo.Mode().IsRegular() && path.Ext(p) == ".json" {
115
+
// load schema file in to catalog
116
+
f, err := os.Open(p)
117
+
if err != nil {
118
+
return nil, err
119
+
}
120
+
defer func() { _ = f.Close() }()
121
+
122
+
b, err := io.ReadAll(f)
123
+
if err != nil {
124
+
return nil, err
125
+
}
126
+
var sf lexicon.SchemaFile
127
+
if err := json.Unmarshal(b, &sf); err != nil {
128
+
return nil, err
129
+
}
130
+
if err := cat.AddSchemaFile(sf); err != nil {
131
+
return nil, err
132
+
}
133
+
}
134
+
}
135
+
return &cat, nil
136
+
}
137
+
138
+
func loadSchemaJSON(fpath string) (syntax.NSID, *json.RawMessage, error) {
139
+
b, err := os.ReadFile(fpath)
140
+
if err != nil {
141
+
return "", nil, err
142
+
}
143
+
144
+
// parse file to check for errors
145
+
// TODO: use json/v2 when available for case-sensitivity
146
+
var sf lexicon.SchemaFile
147
+
err = json.Unmarshal(b, &sf)
148
+
if err == nil {
149
+
err = sf.FinishParse()
150
+
}
151
+
if err == nil {
152
+
err = sf.CheckSchema()
153
+
}
154
+
if err != nil {
155
+
return "", nil, err
156
+
}
157
+
158
+
var rec json.RawMessage
159
+
if err := json.Unmarshal(b, &rec); err != nil {
160
+
return "", nil, err
161
+
}
162
+
return syntax.NSID(sf.ID), &rec, nil
163
+
}
164
+
165
+
func collectSchemaJSON(cmd *cli.Command) (map[syntax.NSID]json.RawMessage, error) {
166
+
schemas := map[syntax.NSID]json.RawMessage{}
167
+
168
+
filePaths, err := collectPaths(cmd)
169
+
if err != nil {
170
+
return nil, err
171
+
}
172
+
173
+
for _, fp := range filePaths {
174
+
nsid, rec, err := loadSchemaJSON(fp)
175
+
if err != nil {
176
+
return nil, err
177
+
}
178
+
schemas[nsid] = *rec
179
+
}
180
+
return schemas, nil
181
+
}