bnewbold.net
go scratch code for atproto
1package main
2
3import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "log/slog"
8 "os"
9 "path"
10
11 "github.com/bluesky-social/indigo/api/agnostic"
12 "github.com/bluesky-social/indigo/atproto/atclient"
13 "github.com/bluesky-social/indigo/atproto/atdata"
14 "github.com/bluesky-social/indigo/atproto/identity"
15 "github.com/bluesky-social/indigo/atproto/lexicon"
16 "github.com/bluesky-social/indigo/atproto/syntax"
17 "tangled.org/bnewbold.net/cobalt/netclient"
18
19 "github.com/urfave/cli/v3"
20)
21
22var cmdLexPull = &cli.Command{
23 Name: "pull",
24 Usage: "fetch (or update) lexicon schemas to local directory",
25 Description: "Resolves and downloads lexicons, and saves as JSON files in local directory.\nPatterns can be full NSIDs, or \"groups\" ending in '.' or '.*'. Does not recursively fetch sub-groups.\nUse 'status' command to check for missing or out-of-date lexicons which need fetching.",
26 ArgsUsage: `<nsid-pattern>+`,
27 Flags: []cli.Flag{
28 &cli.StringFlag{
29 Name: "lexicons-dir",
30 Value: "lexicons/",
31 Usage: "base directory for project Lexicon files",
32 Sources: cli.EnvVars("LEXICONS_DIR"),
33 },
34 &cli.BoolFlag{
35 Name: "update",
36 Aliases: []string{"u"},
37 Usage: "overwrite any existing local files",
38 },
39 &cli.StringFlag{
40 Name: "output-dir",
41 Aliases: []string{"o"},
42 Usage: "write schema files to specific directory",
43 Sources: cli.EnvVars("LEXICONS_DIR"),
44 },
45 },
46 Action: runLexPull,
47}
48
49func runLexPull(ctx context.Context, cmd *cli.Command) error {
50 if !cmd.Args().Present() {
51 cli.ShowSubcommandHelpAndExit(cmd, 1)
52 }
53
54 for _, p := range cmd.Args().Slice() {
55
56 group, err := ParseNSIDGroup(p)
57 if nil == err {
58 if err := pullLexiconGroup(ctx, cmd, group); err != nil {
59 return err
60 }
61 continue
62 }
63
64 nsid, err := syntax.ParseNSID(p)
65 if err != nil {
66 return fmt.Errorf("invalid Lexicon NSID pattern: %s", p)
67 }
68 if err := pullLexicon(ctx, cmd, nsid); err != nil {
69 return err
70 }
71 }
72 return nil
73}
74
75func pullLexicon(ctx context.Context, cmd *cli.Command, nsid syntax.NSID) error {
76
77 fpath := pathForNSID(cmd, nsid)
78 if !cmd.Bool("update") {
79 _, err := os.Stat(fpath)
80 if err == nil {
81 fmt.Printf(" 🟣 %s\n", nsid)
82 return nil
83 }
84 }
85
86 // TODO: common net client
87 netc := netclient.NewNetClient()
88 dir := identity.BaseDirectory{}
89 did, err := dir.ResolveNSID(ctx, nsid)
90 if err != nil {
91 return fmt.Errorf("failed to resolve NSID %s: %w", nsid, err)
92 }
93
94 var rec json.RawMessage
95 cid, err := netc.GetRecord(ctx, did, schemaNSID, syntax.RecordKey(nsid), &rec)
96 if err != nil {
97 return err
98 }
99 slog.Debug("fetched NSID schema record", "nsid", nsid, "cid", cid)
100
101 if err := writeLexiconFile(ctx, cmd, nsid, fpath, rec); err != nil {
102 return err
103 }
104 fmt.Printf(" 🟢 %s\n", nsid)
105 return nil
106}
107
108func writeLexiconFile(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, fpath string, rec json.RawMessage) error {
109
110 var sf lexicon.SchemaFile
111 err := json.Unmarshal(rec, &sf)
112 if err == nil {
113 err = sf.FinishParse()
114 }
115 // NOTE: not calling CheckSchema()
116 if err != nil {
117 return fmt.Errorf("schema record syntax invalid (%s): %w", nsid, err)
118 }
119
120 // ensure (nested) directory exists
121 if err := os.MkdirAll(path.Dir(fpath), 0755); err != nil {
122 return err
123 }
124
125 // remove $type (from record)
126 d, err := atdata.UnmarshalJSON(rec)
127 if err != nil {
128 return err
129 }
130 delete(d, "$type")
131
132 b, err := json.MarshalIndent(d, "", " ")
133 if err != nil {
134 return err
135 }
136 b = append(b, '\n')
137
138 if err := os.WriteFile(fpath, b, 0666); err != nil {
139 return err
140 }
141
142 slog.Debug("wrote NSID schema record to disk", "nsid", nsid, "path", fpath)
143 return nil
144}
145
146func pullLexiconGroup(ctx context.Context, cmd *cli.Command, group string) error {
147
148 // TODO: netclient support for listing records
149 dir := identity.BaseDirectory{}
150 did, err := dir.ResolveNSID(ctx, syntax.NSID(group+"name"))
151 if err != nil {
152 return err
153 }
154 ident, err := dir.LookupDID(ctx, did)
155 if err != nil {
156 return err
157 }
158 c := atclient.NewAPIClient(ident.PDSEndpoint())
159
160 cursor := ""
161 for {
162 // collection string, cursor string, limit int64, repo string, reverse bool
163 resp, err := agnostic.RepoListRecords(ctx, c, schemaNSID.String(), cursor, 100, ident.DID.String(), false)
164 if err != nil {
165 return err
166 }
167 for _, rec := range resp.Records {
168 aturi, err := syntax.ParseATURI(rec.Uri)
169 if err != nil {
170 return err
171 }
172 nsid, err := syntax.ParseNSID(aturi.RecordKey().String())
173 if err != nil {
174 slog.Warn("ignoring invalid schema NSID", "did", ident.DID, "rkey", aturi.RecordKey())
175 continue
176 }
177 if nsidGroup(nsid) != group {
178 // ignoring other NSIDs
179 continue
180 }
181 if rec.Value == nil {
182 return fmt.Errorf("missing record value: %s", nsid)
183 }
184
185 fpath := pathForNSID(cmd, nsid)
186 if !cmd.Bool("update") {
187 _, err := os.Stat(fpath)
188 if err == nil {
189 fmt.Printf(" 🟣 %s\n", nsid)
190 continue
191 }
192 }
193 if err := writeLexiconFile(ctx, cmd, nsid, fpath, *rec.Value); err != nil {
194 return nil
195 }
196 fmt.Printf(" 🟢 %s\n", nsid)
197 }
198 if resp.Cursor != nil && *resp.Cursor != "" {
199 cursor = *resp.Cursor
200 } else {
201 break
202 }
203 }
204 return nil
205}