sullen.net
forked from
tangled.org/core
Monorepo for Tangled
[mega-merge]
Signed-off-by: Seongmin Lee <git@boltless.me>
boltless.me
+3536
-1706
+133
appview/db/db.go
+133
appview/db/db.go
···
3
3
import (
4
4
"context"
5
5
"database/sql"
6
+
"fmt"
6
7
"log/slog"
7
8
"strings"
8
9
···
1171
1172
create index if not exists idx_stars_subject_at_created on stars(subject_at, created);
1172
1173
`)
1173
1174
return err
1175
+
})
1176
+
1177
+
// we cannot modify user-owned record on repository delete
1178
+
orm.RunMigration(conn, logger, "remove-foreign-key-profile_pinned_repositories-and-repos", func(tx *sql.Tx) error {
1179
+
_, err := tx.Exec(`
1180
+
create table profile_pinned_repositories_new (
1181
+
did text not null,
1182
+
1183
+
-- data
1184
+
at_uri text not null,
1185
+
1186
+
-- constraints
1187
+
unique(did, at_uri),
1188
+
foreign key (did) references profile(did) on delete cascade
1189
+
);
1190
+
1191
+
insert into profile_pinned_repositories_new (did, at_uri)
1192
+
select did, at_uri from profile_pinned_repositories;
1193
+
1194
+
drop table profile_pinned_repositories;
1195
+
alter table profile_pinned_repositories_new rename to profile_pinned_repositories;
1196
+
`)
1197
+
return err
1198
+
})
1199
+
1200
+
// several changes here
1201
+
// 1. remove autoincrement id for these tables
1202
+
// 2. remove unique constraints other than (did, rkey) to handle non-unique atproto records
1203
+
// 3. add generated at_uri field
1204
+
//
1205
+
// see comments below and commit message for details
1206
+
orm.RunMigration(conn, logger, "flexible-stars-reactions-follows-public_keys", func(tx *sql.Tx) error {
1207
+
// - add at_uri
1208
+
// - remove unique constraint (did, subject_at)
1209
+
if _, err := tx.Exec(`
1210
+
create table stars_new (
1211
+
did text not null,
1212
+
rkey text not null,
1213
+
at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.feed.star' || '/' || rkey) stored,
1214
+
1215
+
subject_at text not null,
1216
+
created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
1217
+
1218
+
unique(did, rkey)
1219
+
);
1220
+
1221
+
insert into stars_new (did, rkey, subject_at, created)
1222
+
select did, rkey, subject_at, created from stars;
1223
+
1224
+
drop table stars;
1225
+
alter table stars_new rename to stars;
1226
+
`); err != nil {
1227
+
return fmt.Errorf("migrating stars: %w", err)
1228
+
}
1229
+
1230
+
// - add at_uri
1231
+
// - reacted_by_did -> did
1232
+
// - thread_at -> subject_at
1233
+
// - remove unique constraint
1234
+
if _, err := tx.Exec(`
1235
+
create table reactions_new (
1236
+
did text not null,
1237
+
rkey text not null,
1238
+
at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.feed.reaction' || '/' || rkey) stored,
1239
+
1240
+
subject_at text not null,
1241
+
kind text not null,
1242
+
created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
1243
+
1244
+
unique(did, rkey)
1245
+
);
1246
+
1247
+
insert into reactions_new (did, rkey, subject_at, kind, created)
1248
+
select reacted_by_did, rkey, thread_at, kind, created from reactions;
1249
+
1250
+
drop table reactions;
1251
+
alter table reactions_new rename to reactions;
1252
+
`); err != nil {
1253
+
return fmt.Errorf("migrating reactions: %w", err)
1254
+
}
1255
+
1256
+
// - add at_uri column
1257
+
// - user_did -> did
1258
+
// - followed_at -> created
1259
+
// - remove unique constraint
1260
+
// - remove check constraint
1261
+
if _, err := tx.Exec(`
1262
+
create table follows_new (
1263
+
did text not null,
1264
+
rkey text not null,
1265
+
at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.graph.follow' || '/' || rkey) stored,
1266
+
1267
+
subject_did text not null,
1268
+
created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
1269
+
1270
+
unique(did, rkey)
1271
+
);
1272
+
1273
+
insert into follows_new (did, rkey, subject_did, created)
1274
+
select user_did, rkey, subject_did, followed_at from follows;
1275
+
1276
+
drop table follows;
1277
+
alter table follows_new rename to follows;
1278
+
`); err != nil {
1279
+
return fmt.Errorf("migrating follows: %w", err)
1280
+
}
1281
+
1282
+
// - add at_uri column
1283
+
// - remove foreign key relationship from repos
1284
+
if _, err := tx.Exec(`
1285
+
create table public_keys_new (
1286
+
did text not null,
1287
+
rkey text not null,
1288
+
at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.publicKey' || '/' || rkey) stored,
1289
+
1290
+
name text not null,
1291
+
key text not null,
1292
+
created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
1293
+
1294
+
unique(did, rkey)
1295
+
);
1296
+
1297
+
insert into public_keys_new (did, rkey, name, key, created)
1298
+
select did, rkey, name, key, created from public_keys;
1299
+
1300
+
drop table public_keys;
1301
+
alter table public_keys_new rename to public_keys;
1302
+
`); err != nil {
1303
+
return fmt.Errorf("migrating public_keys: %w", err)
1304
+
}
1305
+
1306
+
return nil
1174
1307
})
1175
1308
1176
1309
return &DB{
+41
-35
appview/db/follow.go
+41
-35
appview/db/follow.go
···
6
6
"strings"
7
7
"time"
8
8
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
10
"tangled.org/core/appview/models"
10
11
"tangled.org/core/orm"
11
12
)
12
13
13
-
func AddFollow(e Execer, follow *models.Follow) error {
14
-
query := `insert or ignore into follows (user_did, subject_did, rkey) values (?, ?, ?)`
15
-
_, err := e.Exec(query, follow.UserDid, follow.SubjectDid, follow.Rkey)
14
+
func UpsertFollow(e Execer, follow models.Follow) error {
15
+
_, err := e.Exec(
16
+
`insert into follows (did, rkey, subject_did, created)
17
+
values (?, ?, ?, ?)
18
+
on conflict(did, rkey) do update set
19
+
subject_did = excluded.subject_did,
20
+
created = excluded.created`,
21
+
follow.UserDid,
22
+
follow.Rkey,
23
+
follow.SubjectDid,
24
+
follow.FollowedAt.Format(time.RFC3339),
25
+
)
16
26
return err
17
27
}
18
28
19
-
// Get a follow record
20
-
func GetFollow(e Execer, userDid, subjectDid string) (*models.Follow, error) {
21
-
query := `select user_did, subject_did, followed_at, rkey from follows where user_did = ? and subject_did = ?`
22
-
row := e.QueryRow(query, userDid, subjectDid)
23
-
24
-
var follow models.Follow
25
-
var followedAt string
26
-
err := row.Scan(&follow.UserDid, &follow.SubjectDid, &followedAt, &follow.Rkey)
29
+
// Remove a follow
30
+
func DeleteFollow(e Execer, did, subjectDid syntax.DID) ([]syntax.ATURI, error) {
31
+
var deleted []syntax.ATURI
32
+
rows, err := e.Query(
33
+
`delete from follows
34
+
where did = ? and subject_did = ?
35
+
returning at_uri`,
36
+
did,
37
+
subjectDid,
38
+
)
27
39
if err != nil {
28
-
return nil, err
40
+
return nil, fmt.Errorf("deleting stars: %w", err)
29
41
}
42
+
defer rows.Close()
30
43
31
-
followedAtTime, err := time.Parse(time.RFC3339, followedAt)
32
-
if err != nil {
33
-
log.Println("unable to determine followed at time")
34
-
follow.FollowedAt = time.Now()
35
-
} else {
36
-
follow.FollowedAt = followedAtTime
44
+
for rows.Next() {
45
+
var aturi syntax.ATURI
46
+
if err := rows.Scan(&aturi); err != nil {
47
+
return nil, fmt.Errorf("scanning at_uri: %w", err)
48
+
}
49
+
deleted = append(deleted, aturi)
37
50
}
38
-
39
-
return &follow, nil
40
-
}
41
-
42
-
// Remove a follow
43
-
func DeleteFollow(e Execer, userDid, subjectDid string) error {
44
-
_, err := e.Exec(`delete from follows where user_did = ? and subject_did = ?`, userDid, subjectDid)
45
-
return err
51
+
return deleted, nil
46
52
}
47
53
48
54
// Remove a follow
49
55
func DeleteFollowByRkey(e Execer, userDid, rkey string) error {
50
-
_, err := e.Exec(`delete from follows where user_did = ? and rkey = ?`, userDid, rkey)
56
+
_, err := e.Exec(`delete from follows where did = ? and rkey = ?`, userDid, rkey)
51
57
return err
52
58
}
53
59
···
56
62
err := e.QueryRow(
57
63
`SELECT
58
64
COUNT(CASE WHEN subject_did = ? THEN 1 END) AS followers,
59
-
COUNT(CASE WHEN user_did = ? THEN 1 END) AS following
65
+
COUNT(CASE WHEN did = ? THEN 1 END) AS following
60
66
FROM follows;`, did, did).Scan(&followers, &following)
61
67
if err != nil {
62
68
return models.FollowStats{}, err
···
96
102
group by subject_did
97
103
) f
98
104
full outer join (
99
-
select user_did as did, count(*) as following
105
+
select did as did, count(*) as following
100
106
from follows
101
-
where user_did in (%s)
102
-
group by user_did
107
+
where did in (%s)
108
+
group by did
103
109
) g on f.did = g.did`,
104
110
placeholderStr, placeholderStr)
105
111
···
156
162
}
157
163
158
164
query := fmt.Sprintf(
159
-
`select user_did, subject_did, followed_at, rkey
165
+
`select did, subject_did, created, rkey
160
166
from follows
161
167
%s
162
-
order by followed_at desc
168
+
order by created desc
163
169
%s
164
170
`, whereClause, limitClause)
165
171
···
198
204
}
199
205
200
206
func GetFollowing(e Execer, did string) ([]models.Follow, error) {
201
-
return GetFollows(e, 0, orm.FilterEq("user_did", did))
207
+
return GetFollows(e, 0, orm.FilterEq("did", did))
202
208
}
203
209
204
210
func getFollowStatuses(e Execer, userDid string, subjectDids []string) (map[string]models.FollowStatus, error) {
···
239
245
query := fmt.Sprintf(`
240
246
SELECT subject_did
241
247
FROM follows
242
-
WHERE user_did = ? AND subject_did IN (%s)
248
+
WHERE did = ? AND subject_did IN (%s)
243
249
`, strings.Join(placeholders, ","))
244
250
245
251
rows, err := e.Query(query, args...)
+1
-4
appview/db/profile.go
+1
-4
appview/db/profile.go
···
131
131
}
132
132
133
133
func UpsertProfile(tx *sql.Tx, profile *models.Profile) error {
134
-
defer tx.Rollback()
135
-
136
134
// update links
137
135
_, err := tx.Exec(`delete from profile_links where did = ?`, profile.Did)
138
136
if err != nil {
···
226
224
return err
227
225
}
228
226
}
229
-
230
-
return tx.Commit()
227
+
return nil
231
228
}
232
229
233
230
func GetProfiles(e Execer, filters ...orm.Filter) (map[string]*models.Profile, error) {
+17
-7
appview/db/pubkeys.go
+17
-7
appview/db/pubkeys.go
···
5
5
"time"
6
6
)
7
7
8
-
func AddPublicKey(e Execer, did, name, key, rkey string) error {
8
+
func UpsertPublicKey(e Execer, pubKey models.PublicKey) error {
9
9
_, err := e.Exec(
10
-
`insert or ignore into public_keys (did, name, key, rkey)
11
-
values (?, ?, ?, ?)`,
12
-
did, name, key, rkey)
10
+
`insert into public_keys (did, rkey, name, key, created)
11
+
values (?, ?, ?, ?, ?)
12
+
on conflict(did, rkey) do update set
13
+
name = excluded.name,
14
+
key = excluded.key,
15
+
created = excluded.created`,
16
+
pubKey.Did,
17
+
pubKey.Rkey,
18
+
pubKey.Name,
19
+
pubKey.Key,
20
+
pubKey.Created.Format(time.RFC3339),
21
+
)
13
22
return err
14
23
}
15
24
16
-
func DeletePublicKey(e Execer, did, name, key string) error {
25
+
// for public_keys with empty rkey
26
+
func DeletePublicKeyLegacy(e Execer, did, name string) error {
17
27
_, err := e.Exec(`
18
28
delete from public_keys
19
-
where did = ? and name = ? and key = ?`,
20
-
did, name, key)
29
+
where did = ? and name = ? and rkey = ''`,
30
+
did, name)
21
31
return err
22
32
}
23
33
+62
-48
appview/db/reaction.go
+62
-48
appview/db/reaction.go
···
1
1
package db
2
2
3
3
import (
4
-
"log"
4
+
"fmt"
5
5
"time"
6
6
7
7
"github.com/bluesky-social/indigo/atproto/syntax"
8
8
"tangled.org/core/appview/models"
9
9
)
10
10
11
-
func AddReaction(e Execer, reactedByDid string, threadAt syntax.ATURI, kind models.ReactionKind, rkey string) error {
12
-
query := `insert or ignore into reactions (reacted_by_did, thread_at, kind, rkey) values (?, ?, ?, ?)`
13
-
_, err := e.Exec(query, reactedByDid, threadAt, kind, rkey)
11
+
func UpsertReaction(e Execer, reaction models.Reaction) error {
12
+
_, err := e.Exec(
13
+
`insert into reactions (did, rkey, subject_at, kind, created)
14
+
values (?, ?, ?, ?, ?)
15
+
on conflict(did, rkey) do update set
16
+
subject_at = excluded.subject_at,
17
+
kind = excluded.kind,
18
+
created = excluded.created`,
19
+
reaction.ReactedByDid,
20
+
reaction.Rkey,
21
+
reaction.ThreadAt,
22
+
reaction.Kind,
23
+
reaction.Created.Format(time.RFC3339),
24
+
)
14
25
return err
15
26
}
16
27
17
-
// Get a reaction record
18
-
func GetReaction(e Execer, reactedByDid string, threadAt syntax.ATURI, kind models.ReactionKind) (*models.Reaction, error) {
19
-
query := `
20
-
select reacted_by_did, thread_at, created, rkey
21
-
from reactions
22
-
where reacted_by_did = ? and thread_at = ? and kind = ?`
23
-
row := e.QueryRow(query, reactedByDid, threadAt, kind)
24
-
25
-
var reaction models.Reaction
26
-
var created string
27
-
err := row.Scan(&reaction.ReactedByDid, &reaction.ThreadAt, &created, &reaction.Rkey)
28
+
// Remove a reaction
29
+
func DeleteReaction(e Execer, did syntax.DID, subjectAt syntax.ATURI, kind models.ReactionKind) ([]syntax.ATURI, error) {
30
+
var deleted []syntax.ATURI
31
+
rows, err := e.Query(
32
+
`delete from reactions
33
+
where did = ? and subject_at = ? and kind = ?
34
+
returning at_uri`,
35
+
did,
36
+
subjectAt,
37
+
kind,
38
+
)
28
39
if err != nil {
29
-
return nil, err
40
+
return nil, fmt.Errorf("deleting stars: %w", err)
30
41
}
42
+
defer rows.Close()
31
43
32
-
createdAtTime, err := time.Parse(time.RFC3339, created)
33
-
if err != nil {
34
-
log.Println("unable to determine followed at time")
35
-
reaction.Created = time.Now()
36
-
} else {
37
-
reaction.Created = createdAtTime
44
+
for rows.Next() {
45
+
var aturi syntax.ATURI
46
+
if err := rows.Scan(&aturi); err != nil {
47
+
return nil, fmt.Errorf("scanning at_uri: %w", err)
48
+
}
49
+
deleted = append(deleted, aturi)
38
50
}
39
-
40
-
return &reaction, nil
41
-
}
42
-
43
-
// Remove a reaction
44
-
func DeleteReaction(e Execer, reactedByDid string, threadAt syntax.ATURI, kind models.ReactionKind) error {
45
-
_, err := e.Exec(`delete from reactions where reacted_by_did = ? and thread_at = ? and kind = ?`, reactedByDid, threadAt, kind)
46
-
return err
51
+
return deleted, nil
47
52
}
48
53
49
54
// Remove a reaction
50
-
func DeleteReactionByRkey(e Execer, reactedByDid string, rkey string) error {
51
-
_, err := e.Exec(`delete from reactions where reacted_by_did = ? and rkey = ?`, reactedByDid, rkey)
55
+
func DeleteReactionByRkey(e Execer, did string, rkey string) error {
56
+
_, err := e.Exec(`delete from reactions where did = ? and rkey = ?`, did, rkey)
52
57
return err
53
58
}
54
59
55
-
func GetReactionCount(e Execer, threadAt syntax.ATURI, kind models.ReactionKind) (int, error) {
60
+
func GetReactionCount(e Execer, subjectAt syntax.ATURI, kind models.ReactionKind) (int, error) {
56
61
count := 0
57
62
err := e.QueryRow(
58
-
`select count(reacted_by_did) from reactions where thread_at = ? and kind = ?`, threadAt, kind).Scan(&count)
63
+
`select count(did) from reactions where subject_at = ? and kind = ?`, subjectAt, kind).Scan(&count)
59
64
if err != nil {
60
65
return 0, err
61
66
}
62
67
return count, nil
63
68
}
64
69
65
-
func GetReactionMap(e Execer, userLimit int, threadAt syntax.ATURI) (map[models.ReactionKind]models.ReactionDisplayData, error) {
70
+
func GetReactionMap(e Execer, userLimit int, subjectAt syntax.ATURI) (map[models.ReactionKind]models.ReactionDisplayData, error) {
66
71
query := `
67
-
select kind, reacted_by_did,
72
+
select kind, did,
68
73
row_number() over (partition by kind order by created asc) as rn,
69
74
count(*) over (partition by kind) as total
70
75
from reactions
71
-
where thread_at = ?
76
+
where subject_at = ?
72
77
order by kind, created asc`
73
78
74
-
rows, err := e.Query(query, threadAt)
79
+
rows, err := e.Query(query, subjectAt)
75
80
if err != nil {
76
81
return nil, err
77
82
}
···
101
106
return reactionMap, rows.Err()
102
107
}
103
108
104
-
func GetReactionStatus(e Execer, userDid string, threadAt syntax.ATURI, kind models.ReactionKind) bool {
105
-
if _, err := GetReaction(e, userDid, threadAt, kind); err != nil {
106
-
return false
107
-
} else {
108
-
return true
109
-
}
109
+
func GetReactionStatus(e Execer, userDid string, threadAt syntax.ATURI, kind models.ReactionKind) (bool, error) {
110
+
var exists bool
111
+
err := e.QueryRow(
112
+
`select exists (
113
+
select 1 from reactions
114
+
where did = ? and subject_at = ? and kind = ?
115
+
)`,
116
+
userDid,
117
+
threadAt,
118
+
kind,
119
+
).Scan(&exists)
120
+
return exists, err
110
121
}
111
122
112
-
func GetReactionStatusMap(e Execer, userDid string, threadAt syntax.ATURI) map[models.ReactionKind]bool {
123
+
func GetReactionStatusMap(e Execer, userDid string, threadAt syntax.ATURI) (map[models.ReactionKind]bool, error) {
113
124
statusMap := map[models.ReactionKind]bool{}
114
125
for _, kind := range models.OrderedReactionKinds {
115
-
count := GetReactionStatus(e, userDid, threadAt, kind)
116
-
statusMap[kind] = count
126
+
reacted, err := GetReactionStatus(e, userDid, threadAt, kind)
127
+
if err != nil {
128
+
return nil, err
129
+
}
130
+
statusMap[kind] = reacted
117
131
}
118
-
return statusMap
132
+
return statusMap, nil
119
133
}
+27
-31
appview/db/star.go
+27
-31
appview/db/star.go
···
4
4
"database/sql"
5
5
"errors"
6
6
"fmt"
7
-
"log"
8
7
"slices"
9
8
"strings"
10
9
"time"
···
14
13
"tangled.org/core/orm"
15
14
)
16
15
17
-
func AddStar(e Execer, star *models.Star) error {
18
-
query := `insert or ignore into stars (did, subject_at, rkey) values (?, ?, ?)`
16
+
func UpsertStar(e Execer, star models.Star) error {
19
17
_, err := e.Exec(
20
-
query,
18
+
`insert into stars (did, rkey, subject_at, created)
19
+
values (?, ?, ?, ?)
20
+
on conflict(did, rkey) do update set
21
+
subject_at = excluded.subject_at,
22
+
created = excluded.created`,
21
23
star.Did,
22
-
star.RepoAt.String(),
23
24
star.Rkey,
25
+
star.RepoAt,
26
+
star.Created.Format(time.RFC3339),
24
27
)
25
28
return err
26
29
}
27
30
28
-
// Get a star record
29
-
func GetStar(e Execer, did string, subjectAt syntax.ATURI) (*models.Star, error) {
30
-
query := `
31
-
select did, subject_at, created, rkey
32
-
from stars
33
-
where did = ? and subject_at = ?`
34
-
row := e.QueryRow(query, did, subjectAt)
35
-
36
-
var star models.Star
37
-
var created string
38
-
err := row.Scan(&star.Did, &star.RepoAt, &created, &star.Rkey)
31
+
// Remove a star
32
+
func DeleteStar(tx *sql.Tx, did syntax.DID, subjectAt syntax.ATURI) ([]syntax.ATURI, error) {
33
+
var deleted []syntax.ATURI
34
+
rows, err := tx.Query(
35
+
`delete from stars
36
+
where did = ? and subject_at = ?
37
+
returning at_uri`,
38
+
did,
39
+
subjectAt,
40
+
)
39
41
if err != nil {
40
-
return nil, err
42
+
return nil, fmt.Errorf("deleting stars: %w", err)
41
43
}
44
+
defer rows.Close()
42
45
43
-
createdAtTime, err := time.Parse(time.RFC3339, created)
44
-
if err != nil {
45
-
log.Println("unable to determine followed at time")
46
-
star.Created = time.Now()
47
-
} else {
48
-
star.Created = createdAtTime
46
+
for rows.Next() {
47
+
var aturi syntax.ATURI
48
+
if err := rows.Scan(&aturi); err != nil {
49
+
return nil, fmt.Errorf("scanning at_uri: %w", err)
50
+
}
51
+
deleted = append(deleted, aturi)
49
52
}
50
-
51
-
return &star, nil
52
-
}
53
-
54
-
// Remove a star
55
-
func DeleteStar(e Execer, did string, subjectAt syntax.ATURI) error {
56
-
_, err := e.Exec(`delete from stars where did = ? and subject_at = ?`, did, subjectAt)
57
-
return err
53
+
return deleted, nil
58
54
}
59
55
60
56
// Remove a star
+1
-1
appview/db/timeline.go
+1
-1
appview/db/timeline.go
···
183
183
func getTimelineFollows(e Execer, limit int, loggedInUserDid string, userIsFollowing []string) ([]models.TimelineEvent, error) {
184
184
filters := make([]orm.Filter, 0)
185
185
if userIsFollowing != nil {
186
-
filters = append(filters, orm.FilterIn("user_did", userIsFollowing))
186
+
filters = append(filters, orm.FilterIn("did", userIsFollowing))
187
187
}
188
188
189
189
follows, err := GetFollows(e, limit, filters...)
+38
-12
appview/ingester.go
+38
-12
appview/ingester.go
···
19
19
"tangled.org/core/appview/db"
20
20
"tangled.org/core/appview/models"
21
21
"tangled.org/core/appview/serververify"
22
-
"tangled.org/core/appview/validator"
23
22
"tangled.org/core/idresolver"
24
23
"tangled.org/core/orm"
25
24
"tangled.org/core/rbac"
···
31
30
IdResolver *idresolver.Resolver
32
31
Config *config.Config
33
32
Logger *slog.Logger
34
-
Validator *validator.Validator
35
33
}
36
34
37
35
type processFunc func(ctx context.Context, e *jmodels.Event) error
···
121
119
l.Error("invalid record", "err", err)
122
120
return err
123
121
}
124
-
err = db.AddStar(i.Db, &models.Star{
122
+
err = db.UpsertStar(i.Db, models.Star{
125
123
Did: did,
126
124
RepoAt: subjectUri,
127
125
Rkey: e.Commit.RKey,
···
133
131
if err != nil {
134
132
return fmt.Errorf("failed to %s star record: %w", e.Commit.Operation, err)
135
133
}
134
+
l.Info("processed star", "operation", e.Commit.Operation, "rkey", e.Commit.RKey)
136
135
137
136
return nil
138
137
}
···
154
153
return err
155
154
}
156
155
157
-
err = db.AddFollow(i.Db, &models.Follow{
156
+
err = db.UpsertFollow(i.Db, models.Follow{
158
157
UserDid: did,
159
158
SubjectDid: record.Subject,
160
159
Rkey: e.Commit.RKey,
···
166
165
if err != nil {
167
166
return fmt.Errorf("failed to %s follow record: %w", e.Commit.Operation, err)
168
167
}
168
+
l.Info("processed follow", "operation", e.Commit.Operation, "rkey", e.Commit.RKey)
169
169
170
170
return nil
171
171
}
···
187
187
l.Error("invalid record", "err", err)
188
188
return err
189
189
}
190
+
pubKey, err := models.PublicKeyFromRecord(syntax.DID(did), syntax.RecordKey(e.Commit.RKey), record)
191
+
if err != nil {
192
+
l.Error("invalid record", "err", err)
193
+
return err
194
+
}
195
+
if err := pubKey.Validate(); err != nil {
196
+
l.Error("invalid record", "err", err)
197
+
return err
198
+
}
190
199
191
-
name := record.Name
192
-
key := record.Key
193
-
err = db.AddPublicKey(i.Db, did, name, key, e.Commit.RKey)
200
+
err = db.UpsertPublicKey(i.Db, pubKey)
194
201
case jmodels.CommitOperationDelete:
195
202
l.Debug("processing delete of pubkey")
196
203
err = db.DeletePublicKeyByRkey(i.Db, did, e.Commit.RKey)
···
199
206
if err != nil {
200
207
return fmt.Errorf("failed to %s pubkey record: %w", e.Commit.Operation, err)
201
208
}
209
+
l.Info("processed pubkey", "operation", e.Commit.Operation, "rkey", e.Commit.RKey)
202
210
203
211
return nil
204
212
}
···
343
351
if err != nil {
344
352
return fmt.Errorf("failed to start transaction")
345
353
}
354
+
defer tx.Rollback()
346
355
347
356
err = db.ValidateProfile(tx, &profile)
348
357
if err != nil {
···
350
359
}
351
360
352
361
err = db.UpsertProfile(tx, &profile)
362
+
if err != nil {
363
+
return fmt.Errorf("upserting profile: %w", err)
364
+
}
365
+
366
+
err = tx.Commit()
353
367
case jmodels.CommitOperationDelete:
354
368
err = db.DeleteArtifact(i.Db, orm.FilterEq("did", did), orm.FilterEq("rkey", e.Commit.RKey))
355
369
}
···
607
621
608
622
string := models.StringFromRecord(did, rkey, record)
609
623
610
-
if err = i.Validator.ValidateString(&string); err != nil {
624
+
if err = string.Validate(); err != nil {
611
625
l.Error("invalid record", "err", err)
612
626
return err
613
627
}
···
816
830
817
831
issue := models.IssueFromRecord(did, rkey, record)
818
832
819
-
if err := i.Validator.ValidateIssue(&issue); err != nil {
833
+
if err := issue.Validate(); err != nil {
820
834
return fmt.Errorf("failed to validate issue: %w", err)
821
835
}
822
836
···
896
910
return fmt.Errorf("failed to parse comment from record: %w", err)
897
911
}
898
912
899
-
if err := i.Validator.ValidateIssueComment(comment); err != nil {
913
+
if err := comment.Validate(); err != nil {
900
914
return fmt.Errorf("failed to validate comment: %w", err)
901
915
}
902
916
···
956
970
return fmt.Errorf("failed to parse labeldef from record: %w", err)
957
971
}
958
972
959
-
if err := i.Validator.ValidateLabelDefinition(def); err != nil {
973
+
if err := def.Validate(); err != nil {
960
974
return fmt.Errorf("failed to validate labeldef: %w", err)
961
975
}
962
976
···
1032
1046
if !ok {
1033
1047
return fmt.Errorf("failed to find label def for key: %s, expected: %q", o.OperandKey, slices.Collect(maps.Keys(actx.Defs)))
1034
1048
}
1035
-
if err := i.Validator.ValidateLabelOp(def, repo, &o); err != nil {
1049
+
1050
+
// validate permissions: only collaborators can apply labels currently
1051
+
//
1052
+
// TODO: introduce a repo:triage permission
1053
+
ok, err := i.Enforcer.IsPushAllowed(o.Did, repo.Knot, repo.DidSlashRepo())
1054
+
if err != nil {
1055
+
return fmt.Errorf("enforcing permission: %w", err)
1056
+
}
1057
+
if !ok {
1058
+
return fmt.Errorf("unauthorized label operation")
1059
+
}
1060
+
1061
+
if err := def.ValidateOperandValue(&o); err != nil {
1036
1062
return fmt.Errorf("failed to validate labelop: %w", err)
1037
1063
}
1038
1064
}
+7
-8
appview/issues/issues.go
+7
-8
appview/issues/issues.go
···
27
27
"tangled.org/core/appview/pages/repoinfo"
28
28
"tangled.org/core/appview/pagination"
29
29
"tangled.org/core/appview/reporesolver"
30
-
"tangled.org/core/appview/validator"
31
30
"tangled.org/core/idresolver"
32
31
"tangled.org/core/orm"
33
32
"tangled.org/core/rbac"
···
45
44
config *config.Config
46
45
notifier notify.Notifier
47
46
logger *slog.Logger
48
-
validator *validator.Validator
49
47
indexer *issues_indexer.Indexer
50
48
}
51
49
···
59
57
db *db.DB,
60
58
config *config.Config,
61
59
notifier notify.Notifier,
62
-
validator *validator.Validator,
63
60
indexer *issues_indexer.Indexer,
64
61
logger *slog.Logger,
65
62
) *Issues {
···
74
71
config: config,
75
72
notifier: notifier,
76
73
logger: logger,
77
-
validator: validator,
78
74
indexer: indexer,
79
75
}
80
76
}
···
102
98
103
99
userReactions := map[models.ReactionKind]bool{}
104
100
if user != nil {
105
-
userReactions = db.GetReactionStatusMap(rp.db, user.Active.Did, issue.AtUri())
101
+
userReactions, err = db.GetReactionStatusMap(rp.db, user.Active.Did, issue.AtUri())
102
+
if err != nil {
103
+
l.Error("failed to get issue reaction status", "err", err)
104
+
}
106
105
}
107
106
108
107
backlinks, err := db.GetBacklinks(rp.db, issue.AtUri())
···
166
165
newIssue.Body = r.FormValue("body")
167
166
newIssue.Mentions, newIssue.References = rp.mentionsResolver.Resolve(r.Context(), newIssue.Body)
168
167
169
-
if err := rp.validator.ValidateIssue(newIssue); err != nil {
168
+
if err := newIssue.Validate(); err != nil {
170
169
l.Error("validation error", "err", err)
171
170
rp.pages.Notice(w, noticeId, fmt.Sprintf("Failed to edit issue: %s", err))
172
171
return
···
425
424
Mentions: mentions,
426
425
References: references,
427
426
}
428
-
if err = rp.validator.ValidateIssueComment(&comment); err != nil {
427
+
if err = comment.Validate(); err != nil {
429
428
l.Error("failed to validate comment", "err", err)
430
429
rp.pages.Notice(w, "issue-comment", "Failed to create comment.")
431
430
return
···
928
927
Repo: f,
929
928
}
930
929
931
-
if err := rp.validator.ValidateIssue(issue); err != nil {
930
+
if err := issue.Validate(); err != nil {
932
931
l.Error("validation error", "err", err)
933
932
rp.pages.Notice(w, "issues", fmt.Sprintf("Failed to create issue: %s", err))
934
933
return
+27
-15
appview/labels/labels.go
+27
-15
appview/labels/labels.go
···
15
15
"tangled.org/core/appview/models"
16
16
"tangled.org/core/appview/oauth"
17
17
"tangled.org/core/appview/pages"
18
-
"tangled.org/core/appview/validator"
19
18
"tangled.org/core/orm"
20
19
"tangled.org/core/rbac"
21
20
"tangled.org/core/tid"
···
28
27
)
29
28
30
29
type Labels struct {
31
-
oauth *oauth.OAuth
32
-
pages *pages.Pages
33
-
db *db.DB
34
-
logger *slog.Logger
35
-
validator *validator.Validator
36
-
enforcer *rbac.Enforcer
30
+
oauth *oauth.OAuth
31
+
pages *pages.Pages
32
+
db *db.DB
33
+
logger *slog.Logger
34
+
enforcer *rbac.Enforcer
37
35
}
38
36
39
37
func New(
40
38
oauth *oauth.OAuth,
41
39
pages *pages.Pages,
42
40
db *db.DB,
43
-
validator *validator.Validator,
44
41
enforcer *rbac.Enforcer,
45
42
logger *slog.Logger,
46
43
) *Labels {
47
44
return &Labels{
48
-
oauth: oauth,
49
-
pages: pages,
50
-
db: db,
51
-
logger: logger,
52
-
validator: validator,
53
-
enforcer: enforcer,
45
+
oauth: oauth,
46
+
pages: pages,
47
+
db: db,
48
+
logger: logger,
49
+
enforcer: enforcer,
54
50
}
55
51
}
56
52
···
163
159
164
160
for i := range labelOps {
165
161
def := actx.Defs[labelOps[i].OperandKey]
166
-
if err := l.validator.ValidateLabelOp(def, repo, &labelOps[i]); err != nil {
162
+
op := labelOps[i]
163
+
164
+
// validate permissions: only collaborators can apply labels currently
165
+
//
166
+
// TODO: introduce a repo:triage permission
167
+
ok, err := l.enforcer.IsPushAllowed(op.Did, repo.Knot, repo.DidSlashRepo())
168
+
if err != nil {
169
+
fail("Failed to enforce permissions. Please try again later", fmt.Errorf("enforcing permission: %w", err))
170
+
return
171
+
}
172
+
if !ok {
173
+
fail("Unauthorized label operation", fmt.Errorf("unauthorized label operation"))
174
+
return
175
+
}
176
+
177
+
if err := def.ValidateOperandValue(&op); err != nil {
167
178
fail(fmt.Sprintf("Invalid form data: %s", err), err)
168
179
return
169
180
}
181
+
labelOps[i] = op
170
182
}
171
183
172
184
// reduce the opset
+9
appview/models/follow.go
+9
appview/models/follow.go
···
2
2
3
3
import (
4
4
"time"
5
+
6
+
"tangled.org/core/api/tangled"
5
7
)
6
8
7
9
type Follow struct {
···
9
11
SubjectDid string
10
12
FollowedAt time.Time
11
13
Rkey string
14
+
}
15
+
16
+
func (f *Follow) AsRecord() tangled.GraphFollow {
17
+
return tangled.GraphFollow{
18
+
Subject: f.SubjectDid,
19
+
CreatedAt: f.FollowedAt.Format(time.RFC3339),
20
+
}
12
21
}
13
22
14
23
type FollowStats struct {
+32
appview/models/issue.go
+32
appview/models/issue.go
···
3
3
import (
4
4
"fmt"
5
5
"sort"
6
+
"strings"
6
7
"time"
7
8
8
9
"github.com/bluesky-social/indigo/atproto/syntax"
9
10
"tangled.org/core/api/tangled"
11
+
"tangled.org/core/appview/pages/markup/sanitizer"
10
12
)
11
13
12
14
type Issue struct {
···
59
61
return "open"
60
62
}
61
63
return "closed"
64
+
}
65
+
66
+
var _ Validator = new(Issue)
67
+
68
+
func (i *Issue) Validate() error {
69
+
if i.Title == "" {
70
+
return fmt.Errorf("issue title is empty")
71
+
}
72
+
if i.Body == "" {
73
+
return fmt.Errorf("issue body is empty")
74
+
}
75
+
76
+
if st := strings.TrimSpace(sanitizer.SanitizeDescription(i.Title)); st == "" {
77
+
return fmt.Errorf("title is empty after HTML sanitization")
78
+
}
79
+
80
+
if st := strings.TrimSpace(sanitizer.SanitizeDefault(i.Body)); st == "" {
81
+
return fmt.Errorf("body is empty after HTML sanitization")
82
+
}
83
+
return nil
62
84
}
63
85
64
86
type CommentListItem struct {
···
215
237
216
238
func (i *IssueComment) IsReply() bool {
217
239
return i.ReplyTo != nil
240
+
}
241
+
242
+
var _ Validator = new(IssueComment)
243
+
244
+
func (i *IssueComment) Validate() error {
245
+
if sb := strings.TrimSpace(sanitizer.SanitizeDefault(i.Body)); sb == "" {
246
+
return fmt.Errorf("body is empty after HTML sanitization")
247
+
}
248
+
249
+
return nil
218
250
}
219
251
220
252
func IssueCommentFromRecord(did, rkey string, record tangled.RepoIssueComment) (*IssueComment, error) {
+183
-4
appview/models/label.go
+183
-4
appview/models/label.go
···
7
7
"encoding/json"
8
8
"errors"
9
9
"fmt"
10
+
"regexp"
10
11
"slices"
12
+
"strings"
11
13
"time"
12
14
13
15
"github.com/bluesky-social/indigo/api/atproto"
···
120
122
}
121
123
}
122
124
125
+
var (
126
+
// Label name should be alphanumeric with hyphens/underscores, but not start/end with them
127
+
labelNameRegex = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9_-]*[a-zA-Z0-9])?$`)
128
+
// Color should be a valid hex color
129
+
colorRegex = regexp.MustCompile(`^#[a-fA-F0-9]{6}$`)
130
+
// You can only label issues and pulls presently
131
+
validScopes = []string{tangled.RepoIssueNSID, tangled.RepoPullNSID}
132
+
)
133
+
134
+
var _ Validator = new(LabelDefinition)
135
+
136
+
func (l *LabelDefinition) Validate() error {
137
+
if l.Name == "" {
138
+
return fmt.Errorf("label name is empty")
139
+
}
140
+
if len(l.Name) > 40 {
141
+
return fmt.Errorf("label name too long (max 40 graphemes)")
142
+
}
143
+
if len(l.Name) < 1 {
144
+
return fmt.Errorf("label name too short (min 1 grapheme)")
145
+
}
146
+
if !labelNameRegex.MatchString(l.Name) {
147
+
return fmt.Errorf("label name contains invalid characters (use only letters, numbers, hyphens, and underscores)")
148
+
}
149
+
150
+
if !l.ValueType.IsConcreteType() {
151
+
return fmt.Errorf("invalid value type: %q (must be one of: null, boolean, integer, string)", l.ValueType.Type)
152
+
}
153
+
154
+
// null type checks: cannot be enums, multiple or explicit format
155
+
if l.ValueType.IsNull() && l.ValueType.IsEnum() {
156
+
return fmt.Errorf("null type cannot be used in conjunction with enum type")
157
+
}
158
+
if l.ValueType.IsNull() && l.Multiple {
159
+
return fmt.Errorf("null type labels cannot be multiple")
160
+
}
161
+
if l.ValueType.IsNull() && !l.ValueType.IsAnyFormat() {
162
+
return fmt.Errorf("format cannot be used in conjunction with null type")
163
+
}
164
+
165
+
// format checks: cannot be used with enum, or integers
166
+
if !l.ValueType.IsAnyFormat() && l.ValueType.IsEnum() {
167
+
return fmt.Errorf("enum types cannot be used in conjunction with format specification")
168
+
}
169
+
170
+
if !l.ValueType.IsAnyFormat() && !l.ValueType.IsString() {
171
+
return fmt.Errorf("format specifications are only permitted on string types")
172
+
}
173
+
174
+
// validate scope (nsid format)
175
+
if l.Scope == nil {
176
+
return fmt.Errorf("scope is required")
177
+
}
178
+
for _, s := range l.Scope {
179
+
if _, err := syntax.ParseNSID(s); err != nil {
180
+
return fmt.Errorf("failed to parse scope: %w", err)
181
+
}
182
+
if !slices.Contains(validScopes, s) {
183
+
return fmt.Errorf("invalid scope: scope must be present in %q", validScopes)
184
+
}
185
+
}
186
+
187
+
// validate color if provided
188
+
if l.Color != nil {
189
+
color := strings.TrimSpace(*l.Color)
190
+
if color == "" {
191
+
// empty color is fine, set to nil
192
+
l.Color = nil
193
+
} else {
194
+
if !colorRegex.MatchString(color) {
195
+
return fmt.Errorf("color must be a valid hex color (e.g. #79FFE1 or #000)")
196
+
}
197
+
// expand 3-digit hex to 6-digit hex
198
+
if len(color) == 4 { // #ABC
199
+
color = fmt.Sprintf("#%c%c%c%c%c%c", color[1], color[1], color[2], color[2], color[3], color[3])
200
+
}
201
+
// convert to uppercase for consistency
202
+
color = strings.ToUpper(color)
203
+
l.Color = &color
204
+
}
205
+
}
206
+
207
+
return nil
208
+
}
209
+
210
+
// ValidateOperandValue validates the label operation operand value based on
211
+
// label definition.
212
+
//
213
+
// NOTE: This can modify the [LabelOp]
214
+
func (def *LabelDefinition) ValidateOperandValue(op *LabelOp) error {
215
+
expectedKey := def.AtUri().String()
216
+
if op.OperandKey != def.AtUri().String() {
217
+
return fmt.Errorf("operand key %q does not match label definition URI %q", op.OperandKey, expectedKey)
218
+
}
219
+
220
+
valueType := def.ValueType
221
+
222
+
// this is permitted, it "unsets" a label
223
+
if op.OperandValue == "" {
224
+
op.Operation = LabelOperationDel
225
+
return nil
226
+
}
227
+
228
+
switch valueType.Type {
229
+
case ConcreteTypeNull:
230
+
// For null type, value should be empty
231
+
if op.OperandValue != "null" {
232
+
return fmt.Errorf("null type requires empty value, got %q", op.OperandValue)
233
+
}
234
+
235
+
case ConcreteTypeString:
236
+
// For string type, validate enum constraints if present
237
+
if valueType.IsEnum() {
238
+
if !slices.Contains(valueType.Enum, op.OperandValue) {
239
+
return fmt.Errorf("value %q is not in allowed enum values %v", op.OperandValue, valueType.Enum)
240
+
}
241
+
}
242
+
243
+
switch valueType.Format {
244
+
case ValueTypeFormatDid:
245
+
if _, err := syntax.ParseDID(op.OperandValue); err != nil {
246
+
return fmt.Errorf("failed to resolve did/handle: %w", err)
247
+
}
248
+
case ValueTypeFormatAny, "":
249
+
default:
250
+
return fmt.Errorf("unsupported format constraint: %q", valueType.Format)
251
+
}
252
+
253
+
case ConcreteTypeInt:
254
+
if op.OperandValue == "" {
255
+
return fmt.Errorf("integer type requires non-empty value")
256
+
}
257
+
if _, err := fmt.Sscanf(op.OperandValue, "%d", new(int)); err != nil {
258
+
return fmt.Errorf("value %q is not a valid integer", op.OperandValue)
259
+
}
260
+
261
+
if valueType.IsEnum() {
262
+
if !slices.Contains(valueType.Enum, op.OperandValue) {
263
+
return fmt.Errorf("value %q is not in allowed enum values %v", op.OperandValue, valueType.Enum)
264
+
}
265
+
}
266
+
267
+
case ConcreteTypeBool:
268
+
if op.OperandValue != "true" && op.OperandValue != "false" {
269
+
return fmt.Errorf("boolean type requires value to be 'true' or 'false', got %q", op.OperandValue)
270
+
}
271
+
272
+
// validate enum constraints if present (though uncommon for booleans)
273
+
if valueType.IsEnum() {
274
+
if !slices.Contains(valueType.Enum, op.OperandValue) {
275
+
return fmt.Errorf("value %q is not in allowed enum values %v", op.OperandValue, valueType.Enum)
276
+
}
277
+
}
278
+
279
+
default:
280
+
return fmt.Errorf("unsupported value type: %q", valueType.Type)
281
+
}
282
+
283
+
return nil
284
+
}
285
+
123
286
// random color for a given seed
124
287
func randomColor(seed string) string {
125
288
hash := sha1.Sum([]byte(seed))
···
131
294
return fmt.Sprintf("#%s%s%s", r, g, b)
132
295
}
133
296
134
-
func (ld LabelDefinition) GetColor() string {
135
-
if ld.Color == nil {
136
-
seed := fmt.Sprintf("%d:%s:%s", ld.Id, ld.Did, ld.Rkey)
297
+
func (l LabelDefinition) GetColor() string {
298
+
if l.Color == nil {
299
+
seed := fmt.Sprintf("%d:%s:%s", l.Id, l.Did, l.Rkey)
137
300
color := randomColor(seed)
138
301
return color
139
302
}
140
303
141
-
return *ld.Color
304
+
return *l.Color
142
305
}
143
306
144
307
func LabelDefinitionFromRecord(did, rkey string, record tangled.LabelDefinition) (*LabelDefinition, error) {
···
203
366
204
367
// otherwise, createdat is in the future relative to indexedat -> use indexedat
205
368
return indexedAt
369
+
}
370
+
371
+
var _ Validator = new(LabelOp)
372
+
373
+
func (l *LabelOp) Validate() error {
374
+
if _, err := syntax.ParseATURI(string(l.Subject)); err != nil {
375
+
return fmt.Errorf("invalid subject URI: %w", err)
376
+
}
377
+
if l.Operation != LabelOperationAdd && l.Operation != LabelOperationDel {
378
+
return fmt.Errorf("invalid operation: %q (must be 'add' or 'del')", l.Operation)
379
+
}
380
+
// Validate performed time is not zero/invalid
381
+
if l.PerformedAt.IsZero() {
382
+
return fmt.Errorf("performed_at timestamp is required")
383
+
}
384
+
return nil
206
385
}
207
386
208
387
type LabelOperation string
+38
appview/models/pubkey.go
+38
appview/models/pubkey.go
···
2
2
3
3
import (
4
4
"encoding/json"
5
+
"fmt"
5
6
"time"
7
+
8
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
+
"github.com/gliderlabs/ssh"
10
+
"tangled.org/core/api/tangled"
6
11
)
7
12
8
13
type PublicKey struct {
···
23
28
Alias: (*Alias)(&p),
24
29
})
25
30
}
31
+
32
+
func (p *PublicKey) AsRecord() tangled.PublicKey {
33
+
return tangled.PublicKey{
34
+
Name: p.Name,
35
+
Key: p.Key,
36
+
CreatedAt: p.Created.Format(time.RFC3339),
37
+
}
38
+
}
39
+
40
+
var _ Validator = new(PublicKey)
41
+
42
+
func (p *PublicKey) Validate() error {
43
+
if _, _, _, _, err := ssh.ParseAuthorizedKey([]byte(p.Key)); err != nil {
44
+
return fmt.Errorf("invalid ssh key format: %w", err)
45
+
}
46
+
47
+
return nil
48
+
}
49
+
50
+
func PublicKeyFromRecord(did syntax.DID, rkey syntax.RecordKey, record tangled.PublicKey) (PublicKey, error) {
51
+
created, err := time.Parse(time.RFC3339, record.CreatedAt)
52
+
if err != nil {
53
+
return PublicKey{}, fmt.Errorf("invalid time format '%s'", record.CreatedAt)
54
+
}
55
+
56
+
return PublicKey{
57
+
Did: did.String(),
58
+
Rkey: rkey.String(),
59
+
Name: record.Name,
60
+
Key: record.Key,
61
+
Created: &created,
62
+
}, nil
63
+
}
+9
appview/models/reaction.go
+9
appview/models/reaction.go
···
4
4
"time"
5
5
6
6
"github.com/bluesky-social/indigo/atproto/syntax"
7
+
"tangled.org/core/api/tangled"
7
8
)
8
9
9
10
type ReactionKind string
···
54
55
Created time.Time
55
56
Rkey string
56
57
Kind ReactionKind
58
+
}
59
+
60
+
func (r *Reaction) AsRecord() tangled.FeedReaction {
61
+
return tangled.FeedReaction{
62
+
Subject: r.ThreadAt.String(),
63
+
Reaction: r.Kind.String(),
64
+
CreatedAt: r.Created.Format(time.RFC3339),
65
+
}
57
66
}
58
67
59
68
type ReactionDisplayData struct {
+8
appview/models/star.go
+8
appview/models/star.go
···
4
4
"time"
5
5
6
6
"github.com/bluesky-social/indigo/atproto/syntax"
7
+
"tangled.org/core/api/tangled"
7
8
)
8
9
9
10
type Star struct {
···
11
12
RepoAt syntax.ATURI
12
13
Created time.Time
13
14
Rkey string
15
+
}
16
+
17
+
func (s *Star) AsRecord() tangled.FeedStar {
18
+
return tangled.FeedStar{
19
+
Subject: s.RepoAt.String(),
20
+
CreatedAt: s.Created.Format(time.RFC3339),
21
+
}
14
22
}
15
23
16
24
// RepoStar is used for reverse mapping to repos
+21
appview/models/string.go
+21
appview/models/string.go
···
2
2
3
3
import (
4
4
"bytes"
5
+
"errors"
5
6
"fmt"
6
7
"io"
7
8
"strings"
8
9
"time"
10
+
"unicode/utf8"
9
11
10
12
"github.com/bluesky-social/indigo/atproto/syntax"
11
13
"tangled.org/core/api/tangled"
···
33
35
Contents: s.Contents,
34
36
CreatedAt: s.Created.Format(time.RFC3339),
35
37
}
38
+
}
39
+
40
+
var _ Validator = new(String)
41
+
42
+
func (s *String) Validate() error {
43
+
var err error
44
+
if utf8.RuneCountInString(s.Filename) > 140 {
45
+
err = errors.Join(err, fmt.Errorf("filename too long"))
46
+
}
47
+
48
+
if utf8.RuneCountInString(s.Description) > 280 {
49
+
err = errors.Join(err, fmt.Errorf("description too long"))
50
+
}
51
+
52
+
if len(s.Contents) == 0 {
53
+
err = errors.Join(err, fmt.Errorf("contents is empty"))
54
+
}
55
+
56
+
return err
36
57
}
37
58
38
59
func StringFromRecord(did, rkey string, record tangled.String) String {
+6
appview/models/validator.go
+6
appview/models/validator.go
+4
-3
appview/pages/funcmap.go
+4
-3
appview/pages/funcmap.go
···
29
29
"tangled.org/core/appview/models"
30
30
"tangled.org/core/appview/oauth"
31
31
"tangled.org/core/appview/pages/markup"
32
+
"tangled.org/core/appview/pages/markup/sanitizer"
32
33
"tangled.org/core/crypto"
33
34
)
34
35
···
257
258
"markdown": func(text string) template.HTML {
258
259
p.rctx.RendererType = markup.RendererTypeDefault
259
260
htmlString := p.rctx.RenderMarkdown(text)
260
-
sanitized := p.rctx.SanitizeDefault(htmlString)
261
+
sanitized := sanitizer.SanitizeDefault(htmlString)
261
262
return template.HTML(sanitized)
262
263
},
263
264
"description": func(text string) template.HTML {
···
267
268
emoji.Emoji,
268
269
),
269
270
))
270
-
sanitized := p.rctx.SanitizeDescription(htmlString)
271
+
sanitized := sanitizer.SanitizeDescription(htmlString)
271
272
return template.HTML(sanitized)
272
273
},
273
274
"readme": func(text string) template.HTML {
274
275
p.rctx.RendererType = markup.RendererTypeRepoMarkdown
275
276
htmlString := p.rctx.RenderMarkdown(text)
276
-
sanitized := p.rctx.SanitizeDefault(htmlString)
277
+
sanitized := sanitizer.SanitizeDefault(htmlString)
277
278
return template.HTML(sanitized)
278
279
},
279
280
"code": func(content, path string) string {
-9
appview/pages/markup/markdown.go
-9
appview/pages/markup/markdown.go
···
48
48
IsDev bool
49
49
Hostname string
50
50
RendererType RendererType
51
-
Sanitizer Sanitizer
52
51
Files fs.FS
53
52
}
54
53
···
177
176
}
178
177
default:
179
178
}
180
-
}
181
-
182
-
func (rctx *RenderContext) SanitizeDefault(html string) string {
183
-
return rctx.Sanitizer.SanitizeDefault(html)
184
-
}
185
-
186
-
func (rctx *RenderContext) SanitizeDescription(html string) string {
187
-
return rctx.Sanitizer.SanitizeDescription(html)
188
179
}
189
180
190
181
type MarkdownTransformer struct {
+11
-18
appview/pages/markup/sanitizer.go
appview/pages/markup/sanitizer/sanitizer.go
+11
-18
appview/pages/markup/sanitizer.go
appview/pages/markup/sanitizer/sanitizer.go
···
1
-
package markup
1
+
package sanitizer
2
2
3
3
import (
4
4
"maps"
···
10
10
"github.com/microcosm-cc/bluemonday"
11
11
)
12
12
13
-
type Sanitizer struct {
14
-
defaultPolicy *bluemonday.Policy
15
-
descriptionPolicy *bluemonday.Policy
16
-
}
13
+
var (
14
+
defaultPolicy = newDefaultPolicy()
15
+
descriptionPolicy = newDescriptionPolicy()
16
+
)
17
17
18
-
func NewSanitizer() Sanitizer {
19
-
return Sanitizer{
20
-
defaultPolicy: defaultPolicy(),
21
-
descriptionPolicy: descriptionPolicy(),
22
-
}
18
+
func SanitizeDefault(html string) string {
19
+
return defaultPolicy.Sanitize(html)
23
20
}
24
-
25
-
func (s *Sanitizer) SanitizeDefault(html string) string {
26
-
return s.defaultPolicy.Sanitize(html)
27
-
}
28
-
func (s *Sanitizer) SanitizeDescription(html string) string {
29
-
return s.descriptionPolicy.Sanitize(html)
21
+
func SanitizeDescription(html string) string {
22
+
return descriptionPolicy.Sanitize(html)
30
23
}
31
24
32
-
func defaultPolicy() *bluemonday.Policy {
25
+
func newDefaultPolicy() *bluemonday.Policy {
33
26
policy := bluemonday.UGCPolicy()
34
27
35
28
// Allow generally safe attributes
···
123
116
return policy
124
117
}
125
118
126
-
func descriptionPolicy() *bluemonday.Policy {
119
+
func newDescriptionPolicy() *bluemonday.Policy {
127
120
policy := bluemonday.NewPolicy()
128
121
policy.AllowStandardURLs()
129
122
+5
-5
appview/pages/pages.go
+5
-5
appview/pages/pages.go
···
22
22
"tangled.org/core/appview/models"
23
23
"tangled.org/core/appview/oauth"
24
24
"tangled.org/core/appview/pages/markup"
25
+
"tangled.org/core/appview/pages/markup/sanitizer"
25
26
"tangled.org/core/appview/pages/repoinfo"
26
27
"tangled.org/core/appview/pagination"
27
28
"tangled.org/core/idresolver"
···
56
57
Hostname: config.Core.AppviewHost,
57
58
CamoUrl: config.Camo.Host,
58
59
CamoSecret: config.Camo.SharedSecret,
59
-
Sanitizer: markup.NewSanitizer(),
60
60
Files: Files,
61
61
}
62
62
···
271
271
272
272
p.rctx.RendererType = markup.RendererTypeDefault
273
273
htmlString := p.rctx.RenderMarkdown(string(markdownBytes))
274
-
sanitized := p.rctx.SanitizeDefault(htmlString)
274
+
sanitized := sanitizer.SanitizeDefault(htmlString)
275
275
params.Content = template.HTML(sanitized)
276
276
277
277
return p.execute("legal/terms", w, params)
···
299
299
300
300
p.rctx.RendererType = markup.RendererTypeDefault
301
301
htmlString := p.rctx.RenderMarkdown(string(markdownBytes))
302
-
sanitized := p.rctx.SanitizeDefault(htmlString)
302
+
sanitized := sanitizer.SanitizeDefault(htmlString)
303
303
params.Content = template.HTML(sanitized)
304
304
305
305
return p.execute("legal/privacy", w, params)
···
699
699
case ".md", ".markdown", ".mdown", ".mkdn", ".mkd":
700
700
params.Raw = false
701
701
htmlString := p.rctx.RenderMarkdown(params.Readme)
702
-
sanitized := p.rctx.SanitizeDefault(htmlString)
702
+
sanitized := sanitizer.SanitizeDefault(htmlString)
703
703
params.HTMLReadme = template.HTML(sanitized)
704
704
default:
705
705
params.Raw = true
···
790
790
case ".md", ".markdown", ".mdown", ".mkdn", ".mkd":
791
791
params.Raw = false
792
792
htmlString := p.rctx.RenderMarkdown(params.Readme)
793
-
sanitized := p.rctx.SanitizeDefault(htmlString)
793
+
sanitized := sanitizer.SanitizeDefault(htmlString)
794
794
params.HTMLReadme = template.HTML(sanitized)
795
795
default:
796
796
params.Raw = true
+1
-1
appview/pages/templates/strings/fragments/form.html
+1
-1
appview/pages/templates/strings/fragments/form.html
···
31
31
name="content"
32
32
id="content-textarea"
33
33
wrap="off"
34
-
class="w-full dark:bg-gray-700 dark:text-white dark:border-gray-600 dark:placeholder-gray-400 font-mono"
34
+
class="w-full dark:bg-gray-700 dark:text-white dark:border-gray-600 dark:placeholder-gray-400"
35
35
rows="20"
36
36
spellcheck="false"
37
37
placeholder="Paste your string here!"
+1
-1
appview/pages/templates/user/settings/fragments/keyListing.html
+1
-1
appview/pages/templates/user/settings/fragments/keyListing.html
···
19
19
<button
20
20
class="btn text-red-500 hover:text-red-700 dark:text-red-400 dark:hover:text-red-300 gap-2 group"
21
21
title="Delete key"
22
-
hx-delete="/settings/keys?name={{urlquery $key.Name}}&rkey={{urlquery $key.Rkey}}&key={{urlquery $key.Key}}"
22
+
hx-delete="/settings/keys?name={{urlquery $key.Name}}&rkey={{urlquery $key.Rkey}}"
23
23
hx-swap="none"
24
24
hx-confirm="Are you sure you want to delete the key {{ $key.Name }}?"
25
25
>
+28
-13
appview/pulls/pulls.go
+28
-13
appview/pulls/pulls.go
···
27
27
"tangled.org/core/appview/notify"
28
28
"tangled.org/core/appview/oauth"
29
29
"tangled.org/core/appview/pages"
30
-
"tangled.org/core/appview/pages/markup"
30
+
"tangled.org/core/appview/pages/markup/sanitizer"
31
31
"tangled.org/core/appview/pages/repoinfo"
32
32
"tangled.org/core/appview/pagination"
33
33
"tangled.org/core/appview/reporesolver"
34
-
"tangled.org/core/appview/validator"
35
34
"tangled.org/core/appview/xrpcclient"
36
35
"tangled.org/core/idresolver"
37
36
"tangled.org/core/orm"
···
59
58
notifier notify.Notifier
60
59
enforcer *rbac.Enforcer
61
60
logger *slog.Logger
62
-
validator *validator.Validator
63
61
indexer *pulls_indexer.Indexer
64
62
}
65
63
···
73
71
config *config.Config,
74
72
notifier notify.Notifier,
75
73
enforcer *rbac.Enforcer,
76
-
validator *validator.Validator,
77
74
indexer *pulls_indexer.Indexer,
78
75
logger *slog.Logger,
79
76
) *Pulls {
···
88
85
notifier: notifier,
89
86
enforcer: enforcer,
90
87
logger: logger,
91
-
validator: validator,
92
88
indexer: indexer,
93
89
}
94
90
}
···
227
223
228
224
reactionMap, err := db.GetReactionMap(s.db, 20, pull.AtUri())
229
225
if err != nil {
230
-
log.Println("failed to get pull reactions")
226
+
s.logger.Error("failed to get pull reaction status", "err", err)
231
227
s.pages.Notice(w, "pulls", "Failed to load pull. Try again later.")
232
228
}
233
229
234
230
userReactions := map[models.ReactionKind]bool{}
235
231
if user != nil {
236
-
userReactions = db.GetReactionStatusMap(s.db, user.Active.Did, pull.AtUri())
232
+
userReactions, err = db.GetReactionStatusMap(s.db, user.Active.Did, pull.AtUri())
233
+
if err != nil {
234
+
s.logger.Error("failed to get pull reaction status", "err", err)
235
+
}
237
236
}
238
237
239
238
labelDefs, err := db.GetLabelDefinitions(
···
863
862
s.pages.Notice(w, "pull", "Title is required for git-diff patches.")
864
863
return
865
864
}
866
-
sanitizer := markup.NewSanitizer()
867
865
if st := strings.TrimSpace(sanitizer.SanitizeDescription(title)); (st) == "" {
868
866
s.pages.Notice(w, "pull", "Title is empty after HTML sanitization")
869
867
return
···
991
989
patch := comparison.FormatPatchRaw
992
990
combined := comparison.CombinedPatchRaw
993
991
994
-
if err := s.validator.ValidatePatch(&patch); err != nil {
992
+
if err := validatePatch(&patch); err != nil {
995
993
s.logger.Error("failed to validate patch", "err", err)
996
994
s.pages.Notice(w, "pull", "Invalid patch format. Please provide a valid diff.")
997
995
return
···
1009
1007
}
1010
1008
1011
1009
func (s *Pulls) handlePatchBasedPull(w http.ResponseWriter, r *http.Request, repo *models.Repo, user *oauth.MultiAccountUser, title, body, targetBranch, patch string, isStacked bool) {
1012
-
if err := s.validator.ValidatePatch(&patch); err != nil {
1010
+
if err := validatePatch(&patch); err != nil {
1013
1011
s.logger.Error("patch validation failed", "err", err)
1014
1012
s.pages.Notice(w, "pull", "Invalid patch format. Please provide a valid diff.")
1015
1013
return
···
1101
1099
patch := comparison.FormatPatchRaw
1102
1100
combined := comparison.CombinedPatchRaw
1103
1101
1104
-
if err := s.validator.ValidatePatch(&patch); err != nil {
1102
+
if err := validatePatch(&patch); err != nil {
1105
1103
s.logger.Error("failed to validate patch", "err", err)
1106
1104
s.pages.Notice(w, "pull", "Invalid patch format. Please provide a valid diff.")
1107
1105
return
···
1394
1392
return
1395
1393
}
1396
1394
1397
-
if err := s.validator.ValidatePatch(&patch); err != nil {
1395
+
if err := validatePatch(&patch); err != nil {
1398
1396
s.logger.Error("faield to validate patch", "err", err)
1399
1397
s.pages.Notice(w, "patch-error", "Invalid patch format. Please provide a valid git diff or format-patch.")
1400
1398
return
···
1815
1813
return
1816
1814
}
1817
1815
1818
-
if err := s.validator.ValidatePatch(&patch); err != nil {
1816
+
if err := validatePatch(&patch); err != nil {
1819
1817
s.pages.Notice(w, "resubmit-error", err.Error())
1820
1818
return
1821
1819
}
···
2441
2439
w.Close()
2442
2440
return &b
2443
2441
}
2442
+
2443
+
func validatePatch(patch *string) error {
2444
+
if patch == nil || *patch == "" {
2445
+
return fmt.Errorf("patch is empty")
2446
+
}
2447
+
2448
+
// add newline if not present to diff style patches
2449
+
if !patchutil.IsFormatPatch(*patch) && !strings.HasSuffix(*patch, "\n") {
2450
+
*patch = *patch + "\n"
2451
+
}
2452
+
2453
+
if err := patchutil.IsPatchValid(*patch); err != nil {
2454
+
return err
2455
+
}
2456
+
2457
+
return nil
2458
+
}
+1
-5
appview/repo/repo.go
+1
-5
appview/repo/repo.go
···
20
20
"tangled.org/core/appview/oauth"
21
21
"tangled.org/core/appview/pages"
22
22
"tangled.org/core/appview/reporesolver"
23
-
"tangled.org/core/appview/validator"
24
23
xrpcclient "tangled.org/core/appview/xrpcclient"
25
24
"tangled.org/core/eventconsumer"
26
25
"tangled.org/core/idresolver"
···
49
48
notifier notify.Notifier
50
49
logger *slog.Logger
51
50
serviceAuth *serviceauth.ServiceAuth
52
-
validator *validator.Validator
53
51
}
54
52
55
53
func New(
···
63
61
notifier notify.Notifier,
64
62
enforcer *rbac.Enforcer,
65
63
logger *slog.Logger,
66
-
validator *validator.Validator,
67
64
) *Repo {
68
65
return &Repo{oauth: oauth,
69
66
repoResolver: repoResolver,
···
75
72
notifier: notifier,
76
73
enforcer: enforcer,
77
74
logger: logger,
78
-
validator: validator,
79
75
}
80
76
}
81
77
···
225
221
Multiple: multiple,
226
222
Created: time.Now(),
227
223
}
228
-
if err := rp.validator.ValidateLabelDefinition(&label); err != nil {
224
+
if err := label.Validate(); err != nil {
229
225
fail(err.Error(), err)
230
226
return
231
227
}
+66
-6
appview/repo/settings.go
+66
-6
appview/repo/settings.go
···
4
4
"encoding/json"
5
5
"fmt"
6
6
"net/http"
7
+
"net/url"
8
+
"regexp"
7
9
"slices"
8
10
"strings"
9
11
"time"
···
15
17
"tangled.org/core/appview/pages"
16
18
xrpcclient "tangled.org/core/appview/xrpcclient"
17
19
"tangled.org/core/orm"
20
+
"tangled.org/core/sets"
18
21
"tangled.org/core/types"
19
22
20
23
comatproto "github.com/bluesky-social/indigo/api/atproto"
···
402
405
topicStr = r.FormValue("topics")
403
406
)
404
407
405
-
err = rp.validator.ValidateURI(website)
406
-
if website != "" && err != nil {
407
-
l.Error("invalid uri", "err", err)
408
-
rp.pages.Notice(w, noticeId, err.Error())
409
-
return
408
+
if website != "" {
409
+
if err := validateURI(website); err != nil {
410
+
l.Error("invalid uri", "err", err)
411
+
rp.pages.Notice(w, noticeId, err.Error())
412
+
return
413
+
}
410
414
}
411
415
412
-
topics, err := rp.validator.ValidateRepoTopicStr(topicStr)
416
+
topics, err := parseRepoTopicStr(topicStr)
413
417
if err != nil {
414
418
l.Error("invalid topics", "err", err)
415
419
rp.pages.Notice(w, noticeId, err.Error())
···
469
473
470
474
rp.pages.HxRefresh(w)
471
475
}
476
+
477
+
const (
478
+
maxTopicLen = 50
479
+
maxTopics = 20
480
+
)
481
+
482
+
var (
483
+
topicRE = regexp.MustCompile(`\A[a-z0-9-]+\z`)
484
+
)
485
+
486
+
// parseRepoTopicStr parses and validates whitespace-separated topic string.
487
+
//
488
+
// Rules:
489
+
// - topics are separated by whitespace
490
+
// - each topic may contain lowercase letters, digits, and hyphens only
491
+
// - each topic must be <= 50 characters long
492
+
// - no more than 20 topics allowed
493
+
// - duplicates are removed
494
+
func parseRepoTopicStr(topicStr string) ([]string, error) {
495
+
topicStr = strings.TrimSpace(topicStr)
496
+
if topicStr == "" {
497
+
return nil, nil
498
+
}
499
+
parts := strings.Fields(topicStr)
500
+
if len(parts) > maxTopics {
501
+
return nil, fmt.Errorf("too many topics: %d (maximum %d)", len(parts), maxTopics)
502
+
}
503
+
504
+
topicSet := sets.New[string]()
505
+
506
+
for _, t := range parts {
507
+
if topicSet.Contains(t) {
508
+
continue
509
+
}
510
+
if len(t) > maxTopicLen {
511
+
return nil, fmt.Errorf("topic '%s' is too long (maximum %d characters)", t, maxTopics)
512
+
}
513
+
if !topicRE.MatchString(t) {
514
+
return nil, fmt.Errorf("topic '%s' contains invalid characters (allowed: lowercase letters, digits, hyphens)", t)
515
+
}
516
+
topicSet.Insert(t)
517
+
}
518
+
return slices.Collect(topicSet.All()), nil
519
+
}
520
+
521
+
// TODO(boltless): move this to models.Repo instead
522
+
func validateURI(uri string) error {
523
+
parsed, err := url.Parse(uri)
524
+
if err != nil {
525
+
return fmt.Errorf("invalid uri format")
526
+
}
527
+
if parsed.Scheme == "" {
528
+
return fmt.Errorf("uri scheme missing")
529
+
}
530
+
return nil
531
+
}
+39
-36
appview/settings/settings.go
+39
-36
appview/settings/settings.go
···
24
24
comatproto "github.com/bluesky-social/indigo/api/atproto"
25
25
"github.com/bluesky-social/indigo/atproto/syntax"
26
26
lexutil "github.com/bluesky-social/indigo/lex/util"
27
-
"github.com/gliderlabs/ssh"
28
27
"github.com/google/uuid"
29
28
)
30
29
···
439
438
log.Println("unimplemented")
440
439
return
441
440
case http.MethodPut:
442
-
did := s.OAuth.GetDid(r)
443
-
key := r.FormValue("key")
444
-
key = strings.TrimSpace(key)
445
-
name := r.FormValue("name")
446
-
client, err := s.OAuth.AuthorizedClient(r)
447
-
if err != nil {
448
-
s.Pages.Notice(w, "settings-keys", "Failed to authorize. Try again later.")
449
-
return
441
+
created := time.Now()
442
+
pubKey := models.PublicKey{
443
+
Did: s.OAuth.GetDid(r),
444
+
Rkey: tid.TID(),
445
+
Name: r.FormValue("name"),
446
+
Key: strings.TrimSpace(r.FormValue("key")),
447
+
Created: &created,
450
448
}
451
449
452
-
_, _, _, _, err = ssh.ParseAuthorizedKey([]byte(key))
453
-
if err != nil {
450
+
if err := pubKey.Validate(); err != nil {
454
451
log.Printf("parsing public key: %s", err)
455
452
s.Pages.Notice(w, "settings-keys", "That doesn't look like a valid public key. Make sure it's a <strong>public</strong> key.")
456
453
return
457
454
}
458
455
459
-
rkey := tid.TID()
460
-
461
456
tx, err := s.Db.Begin()
462
457
if err != nil {
463
458
log.Printf("failed to start tx; adding public key: %s", err)
···
466
461
}
467
462
defer tx.Rollback()
468
463
469
-
if err := db.AddPublicKey(tx, did, name, key, rkey); err != nil {
464
+
if err = db.UpsertPublicKey(s.Db, pubKey); err != nil {
470
465
log.Printf("adding public key: %s", err)
471
466
s.Pages.Notice(w, "settings-keys", "Failed to add public key.")
467
+
return
468
+
}
469
+
470
+
client, err := s.OAuth.AuthorizedClient(r)
471
+
if err != nil {
472
+
s.Pages.Notice(w, "settings-keys", "Failed to authorize. Try again later.")
472
473
return
473
474
}
474
475
475
476
// store in pds too
477
+
record := pubKey.AsRecord()
476
478
resp, err := comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{
477
479
Collection: tangled.PublicKeyNSID,
478
-
Repo: did,
479
-
Rkey: rkey,
480
+
Repo: pubKey.Did,
481
+
Rkey: pubKey.Rkey,
480
482
Record: &lexutil.LexiconTypeDecoder{
481
-
Val: &tangled.PublicKey{
482
-
CreatedAt: time.Now().Format(time.RFC3339),
483
-
Key: key,
484
-
Name: name,
485
-
}},
483
+
Val: &record,
484
+
},
486
485
})
487
486
// invalid record
488
487
if err != nil {
···
509
508
510
509
name := q.Get("name")
511
510
rkey := q.Get("rkey")
512
-
key := q.Get("key")
513
511
514
512
log.Println(name)
515
513
log.Println(rkey)
516
-
log.Println(key)
517
514
518
-
client, err := s.OAuth.AuthorizedClient(r)
519
-
if err != nil {
520
-
log.Printf("failed to authorize client: %s", err)
521
-
s.Pages.Notice(w, "settings-keys", "Failed to authorize client.")
522
-
return
523
-
}
515
+
if rkey == "" {
516
+
if err := db.DeletePublicKeyLegacy(s.Db, did, name); err != nil {
517
+
log.Printf("removing public key: %s", err)
518
+
s.Pages.Notice(w, "settings-keys", "Failed to remove public key.")
519
+
return
520
+
}
521
+
} else {
522
+
if err := db.DeletePublicKeyByRkey(s.Db, did, rkey); err != nil {
523
+
log.Printf("removing public key: %s", err)
524
+
s.Pages.Notice(w, "settings-keys", "Failed to remove public key.")
525
+
return
526
+
}
524
527
525
-
if err := db.DeletePublicKey(s.Db, did, name, key); err != nil {
526
-
log.Printf("removing public key: %s", err)
527
-
s.Pages.Notice(w, "settings-keys", "Failed to remove public key.")
528
-
return
529
-
}
528
+
client, err := s.OAuth.AuthorizedClient(r)
529
+
if err != nil {
530
+
log.Printf("failed to authorize client: %s", err)
531
+
s.Pages.Notice(w, "settings-keys", "Failed to authorize client.")
532
+
return
533
+
}
530
534
531
-
if rkey != "" {
532
535
// remove from pds too
533
-
_, err := comatproto.RepoDeleteRecord(r.Context(), client, &comatproto.RepoDeleteRecord_Input{
536
+
_, err = comatproto.RepoDeleteRecord(r.Context(), client, &comatproto.RepoDeleteRecord_Input{
534
537
Collection: tangled.PublicKeyNSID,
535
538
Repo: did,
536
539
Rkey: rkey,
+59
-34
appview/state/follow.go
+59
-34
appview/state/follow.go
···
6
6
"time"
7
7
8
8
comatproto "github.com/bluesky-social/indigo/api/atproto"
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
10
lexutil "github.com/bluesky-social/indigo/lex/util"
10
11
"tangled.org/core/api/tangled"
11
12
"tangled.org/core/appview/db"
···
42
43
43
44
switch r.Method {
44
45
case http.MethodPost:
45
-
createdAt := time.Now().Format(time.RFC3339)
46
-
rkey := tid.TID()
46
+
follow := models.Follow{
47
+
UserDid: currentUser.Active.Did,
48
+
SubjectDid: subjectIdent.DID.String(),
49
+
Rkey: tid.TID(),
50
+
FollowedAt: time.Now(),
51
+
}
52
+
53
+
tx, err := s.db.BeginTx(r.Context(), nil)
54
+
if err != nil {
55
+
s.logger.Error("failed to start transaction", "err", err)
56
+
return
57
+
}
58
+
defer tx.Rollback()
59
+
60
+
if err := db.UpsertFollow(tx, follow); err != nil {
61
+
s.logger.Error("failed to follow", "err", err)
62
+
return
63
+
}
64
+
65
+
record := follow.AsRecord()
47
66
resp, err := comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{
48
67
Collection: tangled.GraphFollowNSID,
49
68
Repo: currentUser.Active.Did,
50
-
Rkey: rkey,
69
+
Rkey: follow.Rkey,
51
70
Record: &lexutil.LexiconTypeDecoder{
52
-
Val: &tangled.GraphFollow{
53
-
Subject: subjectIdent.DID.String(),
54
-
CreatedAt: createdAt,
55
-
}},
71
+
Val: &record,
72
+
},
56
73
})
57
74
if err != nil {
58
75
log.Println("failed to create atproto record", err)
59
76
return
60
77
}
61
-
62
78
log.Println("created atproto record: ", resp.Uri)
63
79
64
-
follow := &models.Follow{
65
-
UserDid: currentUser.Active.Did,
66
-
SubjectDid: subjectIdent.DID.String(),
67
-
Rkey: rkey,
80
+
if err := tx.Commit(); err != nil {
81
+
s.logger.Error("failed to commit transaction", "err", err)
82
+
// DB op failed but record is created in PDS. Ingester will backfill the missed operation
68
83
}
69
84
70
-
err = db.AddFollow(s.db, follow)
71
-
if err != nil {
72
-
log.Println("failed to follow", err)
73
-
return
74
-
}
75
-
76
-
s.notifier.NewFollow(r.Context(), follow)
85
+
s.notifier.NewFollow(r.Context(), &follow)
77
86
78
87
followStats, err := db.GetFollowerFollowingCount(s.db, subjectIdent.DID.String())
79
88
if err != nil {
···
88
97
89
98
return
90
99
case http.MethodDelete:
91
-
// find the record in the db
92
-
follow, err := db.GetFollow(s.db, currentUser.Active.Did, subjectIdent.DID.String())
100
+
tx, err := s.db.BeginTx(r.Context(), nil)
93
101
if err != nil {
94
-
log.Println("failed to get follow relationship")
102
+
s.logger.Error("failed to start transaction", "err", err)
103
+
}
104
+
defer tx.Rollback()
105
+
106
+
follows, err := db.DeleteFollow(tx, syntax.DID(currentUser.Active.Did), subjectIdent.DID)
107
+
if err != nil {
108
+
s.logger.Error("failed to delete follows from db", "err", err)
95
109
return
96
110
}
97
111
98
-
_, err = comatproto.RepoDeleteRecord(r.Context(), client, &comatproto.RepoDeleteRecord_Input{
99
-
Collection: tangled.GraphFollowNSID,
100
-
Repo: currentUser.Active.Did,
101
-
Rkey: follow.Rkey,
112
+
var writes []*comatproto.RepoApplyWrites_Input_Writes_Elem
113
+
for _, followAt := range follows {
114
+
writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
115
+
RepoApplyWrites_Delete: &comatproto.RepoApplyWrites_Delete{
116
+
Collection: tangled.GraphFollowNSID,
117
+
Rkey: followAt.RecordKey().String(),
118
+
},
119
+
})
120
+
}
121
+
_, err = comatproto.RepoApplyWrites(r.Context(), client, &comatproto.RepoApplyWrites_Input{
122
+
Repo: currentUser.Active.Did,
123
+
Writes: writes,
102
124
})
103
-
104
125
if err != nil {
105
-
log.Println("failed to unfollow")
126
+
s.logger.Error("failed to delete follows from PDS", "err", err)
106
127
return
107
128
}
108
129
109
-
err = db.DeleteFollowByRkey(s.db, currentUser.Active.Did, follow.Rkey)
110
-
if err != nil {
111
-
log.Println("failed to delete follow from DB")
112
-
// this is not an issue, the firehose event might have already done this
130
+
if err := tx.Commit(); err != nil {
131
+
s.logger.Error("failed to commit transaction", "err", err)
132
+
// DB op failed but record is created in PDS. Ingester will backfill the missed operation
113
133
}
134
+
135
+
s.notifier.DeleteFollow(r.Context(), &models.Follow{
136
+
UserDid: currentUser.Active.Did,
137
+
SubjectDid: subjectIdent.DID.String(),
138
+
// Rkey
139
+
// FollowedAt
140
+
})
114
141
115
142
followStats, err := db.GetFollowerFollowingCount(s.db, subjectIdent.DID.String())
116
143
if err != nil {
···
122
149
FollowStatus: models.IsNotFollowing,
123
150
FollowersCount: followStats.Followers,
124
151
})
125
-
126
-
s.notifier.DeleteFollow(r.Context(), follow)
127
152
128
153
return
129
154
}
-86
appview/state/knotstream.go
-86
appview/state/knotstream.go
···
18
18
"tangled.org/core/log"
19
19
"tangled.org/core/orm"
20
20
"tangled.org/core/rbac"
21
-
"tangled.org/core/workflow"
22
21
23
-
"github.com/bluesky-social/indigo/atproto/syntax"
24
22
"github.com/go-git/go-git/v5/plumbing"
25
23
"github.com/posthog/posthog-go"
26
24
)
···
67
65
switch msg.Nsid {
68
66
case tangled.GitRefUpdateNSID:
69
67
return ingestRefUpdate(d, enforcer, posthog, dev, source, msg)
70
-
case tangled.PipelineNSID:
71
-
return ingestPipeline(d, source, msg)
72
68
}
73
69
74
70
return nil
···
190
186
191
187
return tx.Commit()
192
188
}
193
-
194
-
func ingestPipeline(d *db.DB, source ec.Source, msg ec.Message) error {
195
-
var record tangled.Pipeline
196
-
err := json.Unmarshal(msg.EventJson, &record)
197
-
if err != nil {
198
-
return err
199
-
}
200
-
201
-
if record.TriggerMetadata == nil {
202
-
return fmt.Errorf("empty trigger metadata: nsid %s, rkey %s", msg.Nsid, msg.Rkey)
203
-
}
204
-
205
-
if record.TriggerMetadata.Repo == nil {
206
-
return fmt.Errorf("empty repo: nsid %s, rkey %s", msg.Nsid, msg.Rkey)
207
-
}
208
-
209
-
// does this repo have a spindle configured?
210
-
repos, err := db.GetRepos(
211
-
d,
212
-
0,
213
-
orm.FilterEq("did", record.TriggerMetadata.Repo.Did),
214
-
orm.FilterEq("name", record.TriggerMetadata.Repo.Repo),
215
-
)
216
-
if err != nil {
217
-
return fmt.Errorf("failed to look for repo in DB: nsid %s, rkey %s, %w", msg.Nsid, msg.Rkey, err)
218
-
}
219
-
if len(repos) != 1 {
220
-
return fmt.Errorf("incorrect number of repos returned: %d (expected 1)", len(repos))
221
-
}
222
-
if repos[0].Spindle == "" {
223
-
return fmt.Errorf("repo does not have a spindle configured yet: nsid %s, rkey %s", msg.Nsid, msg.Rkey)
224
-
}
225
-
226
-
// trigger info
227
-
var trigger models.Trigger
228
-
var sha string
229
-
trigger.Kind = workflow.TriggerKind(record.TriggerMetadata.Kind)
230
-
switch trigger.Kind {
231
-
case workflow.TriggerKindPush:
232
-
trigger.PushRef = &record.TriggerMetadata.Push.Ref
233
-
trigger.PushNewSha = &record.TriggerMetadata.Push.NewSha
234
-
trigger.PushOldSha = &record.TriggerMetadata.Push.OldSha
235
-
sha = *trigger.PushNewSha
236
-
case workflow.TriggerKindPullRequest:
237
-
trigger.PRSourceBranch = &record.TriggerMetadata.PullRequest.SourceBranch
238
-
trigger.PRTargetBranch = &record.TriggerMetadata.PullRequest.TargetBranch
239
-
trigger.PRSourceSha = &record.TriggerMetadata.PullRequest.SourceSha
240
-
trigger.PRAction = &record.TriggerMetadata.PullRequest.Action
241
-
sha = *trigger.PRSourceSha
242
-
}
243
-
244
-
tx, err := d.Begin()
245
-
if err != nil {
246
-
return fmt.Errorf("failed to start txn: %w", err)
247
-
}
248
-
249
-
triggerId, err := db.AddTrigger(tx, trigger)
250
-
if err != nil {
251
-
return fmt.Errorf("failed to add trigger entry: %w", err)
252
-
}
253
-
254
-
pipeline := models.Pipeline{
255
-
Rkey: msg.Rkey,
256
-
Knot: source.Key(),
257
-
RepoOwner: syntax.DID(record.TriggerMetadata.Repo.Did),
258
-
RepoName: record.TriggerMetadata.Repo.Repo,
259
-
TriggerId: int(triggerId),
260
-
Sha: sha,
261
-
}
262
-
263
-
err = db.AddPipeline(tx, pipeline)
264
-
if err != nil {
265
-
return fmt.Errorf("failed to add pipeline: %w", err)
266
-
}
267
-
268
-
err = tx.Commit()
269
-
if err != nil {
270
-
return fmt.Errorf("failed to commit txn: %w", err)
271
-
}
272
-
273
-
return nil
274
-
}
+12
-5
appview/state/profile.go
+12
-5
appview/state/profile.go
···
613
613
s.pages.Notice(w, "update-profile", "Failed to update profile, try again later.")
614
614
return
615
615
}
616
+
defer tx.Rollback()
617
+
618
+
err = db.UpsertProfile(tx, profile)
619
+
if err != nil {
620
+
log.Println("failed to update profile", err)
621
+
s.pages.Notice(w, "update-profile", "Failed to update profile, try again later.")
622
+
return
623
+
}
616
624
617
625
client, err := s.oauth.AuthorizedClient(r)
618
626
if err != nil {
···
661
669
return
662
670
}
663
671
664
-
err = db.UpsertProfile(tx, profile)
665
-
if err != nil {
666
-
log.Println("failed to update profile", err)
667
-
s.pages.Notice(w, "update-profile", "Failed to update profile, try again later.")
668
-
return
672
+
if err := tx.Commit(); err != nil {
673
+
s.logger.Error("failed to commit transaction", "err", err)
674
+
// db failed, but PDS operation succeed.
675
+
// log error and continue
669
676
}
670
677
671
678
s.notifier.UpdateProfile(r.Context(), profile)
+51
-26
appview/state/reaction.go
+51
-26
appview/state/reaction.go
···
45
45
46
46
switch r.Method {
47
47
case http.MethodPost:
48
-
createdAt := time.Now().Format(time.RFC3339)
49
-
rkey := tid.TID()
48
+
reaction := models.Reaction{
49
+
ReactedByDid: currentUser.Active.Did,
50
+
Rkey: tid.TID(),
51
+
Kind: reactionKind,
52
+
ThreadAt: subjectUri,
53
+
Created: time.Now(),
54
+
}
55
+
56
+
tx, err := s.db.BeginTx(r.Context(), nil)
57
+
if err != nil {
58
+
s.logger.Error("failed to start transaction", "err", err)
59
+
return
60
+
}
61
+
defer tx.Rollback()
62
+
63
+
if err := db.UpsertReaction(tx, reaction); err != nil {
64
+
log.Println("failed to react", err)
65
+
return
66
+
}
67
+
68
+
record := reaction.AsRecord()
50
69
resp, err := comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{
51
70
Collection: tangled.FeedReactionNSID,
52
71
Repo: currentUser.Active.Did,
53
-
Rkey: rkey,
72
+
Rkey: reaction.Rkey,
54
73
Record: &lexutil.LexiconTypeDecoder{
55
-
Val: &tangled.FeedReaction{
56
-
Subject: subjectUri.String(),
57
-
Reaction: reactionKind.String(),
58
-
CreatedAt: createdAt,
59
-
},
74
+
Val: &record,
60
75
},
61
76
})
62
77
if err != nil {
63
78
log.Println("failed to create atproto record", err)
64
79
return
65
80
}
81
+
log.Println("created atproto record: ", resp.Uri)
66
82
67
-
err = db.AddReaction(s.db, currentUser.Active.Did, subjectUri, reactionKind, rkey)
68
-
if err != nil {
69
-
log.Println("failed to react", err)
70
-
return
83
+
if err := tx.Commit(); err != nil {
84
+
s.logger.Error("failed to commit transaction", "err", err)
85
+
// DB op failed but record is created in PDS. Ingester will backfill the missed operation
71
86
}
72
87
73
88
reactionMap, err := db.GetReactionMap(s.db, 20, subjectUri)
···
75
90
log.Println("failed to get reactions for ", subjectUri)
76
91
}
77
92
78
-
log.Println("created atproto record: ", resp.Uri)
79
-
80
93
s.pages.ThreadReactionFragment(w, pages.ThreadReactionFragmentParams{
81
94
ThreadAt: subjectUri,
82
95
Kind: reactionKind,
···
87
100
88
101
return
89
102
case http.MethodDelete:
90
-
reaction, err := db.GetReaction(s.db, currentUser.Active.Did, subjectUri, reactionKind)
103
+
tx, err := s.db.BeginTx(r.Context(), nil)
104
+
if err != nil {
105
+
s.logger.Error("failed to start transaction", "err", err)
106
+
}
107
+
defer tx.Rollback()
108
+
109
+
reactions, err := db.DeleteReaction(tx, syntax.DID(currentUser.Active.Did), subjectUri, reactionKind)
91
110
if err != nil {
92
-
log.Println("failed to get reaction relationship for", currentUser.Active.Did, subjectUri)
111
+
s.logger.Error("failed to delete reactions from db", "err", err)
93
112
return
94
113
}
95
114
96
-
_, err = comatproto.RepoDeleteRecord(r.Context(), client, &comatproto.RepoDeleteRecord_Input{
97
-
Collection: tangled.FeedReactionNSID,
98
-
Repo: currentUser.Active.Did,
99
-
Rkey: reaction.Rkey,
115
+
var writes []*comatproto.RepoApplyWrites_Input_Writes_Elem
116
+
for _, reactionAt := range reactions {
117
+
writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
118
+
RepoApplyWrites_Delete: &comatproto.RepoApplyWrites_Delete{
119
+
Collection: tangled.FeedReactionNSID,
120
+
Rkey: reactionAt.RecordKey().String(),
121
+
},
122
+
})
123
+
}
124
+
_, err = comatproto.RepoApplyWrites(r.Context(), client, &comatproto.RepoApplyWrites_Input{
125
+
Repo: currentUser.Active.Did,
126
+
Writes: writes,
100
127
})
101
-
102
128
if err != nil {
103
-
log.Println("failed to remove reaction")
129
+
s.logger.Error("failed to delete reactions from PDS", "err", err)
104
130
return
105
131
}
106
132
107
-
err = db.DeleteReactionByRkey(s.db, currentUser.Active.Did, reaction.Rkey)
108
-
if err != nil {
109
-
log.Println("failed to delete reaction from DB")
110
-
// this is not an issue, the firehose event might have already done this
133
+
if err := tx.Commit(); err != nil {
134
+
s.logger.Error("failed to commit transaction", "err", err)
135
+
// DB op failed but record is created in PDS. Ingester will backfill the missed operation
111
136
}
112
137
113
138
reactionMap, err := db.GetReactionMap(s.db, 20, subjectUri)
-4
appview/state/router.go
-4
appview/state/router.go
···
274
274
s.db,
275
275
s.config,
276
276
s.notifier,
277
-
s.validator,
278
277
s.indexer.Issues,
279
278
log.SubLogger(s.logger, "issues"),
280
279
)
···
292
291
s.config,
293
292
s.notifier,
294
293
s.enforcer,
295
-
s.validator,
296
294
s.indexer.Pulls,
297
295
log.SubLogger(s.logger, "pulls"),
298
296
)
···
311
309
s.notifier,
312
310
s.enforcer,
313
311
log.SubLogger(s.logger, "repo"),
314
-
s.validator,
315
312
)
316
313
return repo.Router(mw)
317
314
}
···
336
333
s.oauth,
337
334
s.pages,
338
335
s.db,
339
-
s.validator,
340
336
s.enforcer,
341
337
log.SubLogger(s.logger, "labels"),
342
338
)
+89
appview/state/spindlestream.go
+89
appview/state/spindlestream.go
···
20
20
"tangled.org/core/orm"
21
21
"tangled.org/core/rbac"
22
22
spindle "tangled.org/core/spindle/models"
23
+
"tangled.org/core/workflow"
23
24
)
24
25
25
26
func Spindlestream(ctx context.Context, c *config.Config, d *db.DB, enforcer *rbac.Enforcer) (*ec.Consumer, error) {
···
62
63
func spindleIngester(ctx context.Context, logger *slog.Logger, d *db.DB) ec.ProcessFunc {
63
64
return func(ctx context.Context, source ec.Source, msg ec.Message) error {
64
65
switch msg.Nsid {
66
+
case tangled.PipelineNSID:
67
+
return ingestPipeline(logger, d, source, msg)
65
68
case tangled.PipelineStatusNSID:
66
69
return ingestPipelineStatus(ctx, logger, d, source, msg)
67
70
}
68
71
69
72
return nil
70
73
}
74
+
}
75
+
76
+
func ingestPipeline(l *slog.Logger, d *db.DB, source ec.Source, msg ec.Message) error {
77
+
var record tangled.Pipeline
78
+
err := json.Unmarshal(msg.EventJson, &record)
79
+
if err != nil {
80
+
return err
81
+
}
82
+
83
+
if record.TriggerMetadata == nil {
84
+
return fmt.Errorf("empty trigger metadata: nsid %s, rkey %s", msg.Nsid, msg.Rkey)
85
+
}
86
+
87
+
if record.TriggerMetadata.Repo == nil {
88
+
return fmt.Errorf("empty repo: nsid %s, rkey %s", msg.Nsid, msg.Rkey)
89
+
}
90
+
91
+
// does this repo have a spindle configured?
92
+
repos, err := db.GetRepos(
93
+
d,
94
+
0,
95
+
orm.FilterEq("did", record.TriggerMetadata.Repo.Did),
96
+
orm.FilterEq("name", record.TriggerMetadata.Repo.Repo),
97
+
)
98
+
if err != nil {
99
+
return fmt.Errorf("failed to look for repo in DB: nsid %s, rkey %s, %w", msg.Nsid, msg.Rkey, err)
100
+
}
101
+
if len(repos) != 1 {
102
+
return fmt.Errorf("incorrect number of repos returned: %d (expected 1)", len(repos))
103
+
}
104
+
if repos[0].Spindle == "" {
105
+
return fmt.Errorf("repo does not have a spindle configured yet: nsid %s, rkey %s", msg.Nsid, msg.Rkey)
106
+
}
107
+
108
+
// trigger info
109
+
var trigger models.Trigger
110
+
var sha string
111
+
trigger.Kind = workflow.TriggerKind(record.TriggerMetadata.Kind)
112
+
switch trigger.Kind {
113
+
case workflow.TriggerKindPush:
114
+
trigger.PushRef = &record.TriggerMetadata.Push.Ref
115
+
trigger.PushNewSha = &record.TriggerMetadata.Push.NewSha
116
+
trigger.PushOldSha = &record.TriggerMetadata.Push.OldSha
117
+
sha = *trigger.PushNewSha
118
+
case workflow.TriggerKindPullRequest:
119
+
trigger.PRSourceBranch = &record.TriggerMetadata.PullRequest.SourceBranch
120
+
trigger.PRTargetBranch = &record.TriggerMetadata.PullRequest.TargetBranch
121
+
trigger.PRSourceSha = &record.TriggerMetadata.PullRequest.SourceSha
122
+
trigger.PRAction = &record.TriggerMetadata.PullRequest.Action
123
+
sha = *trigger.PRSourceSha
124
+
}
125
+
126
+
tx, err := d.Begin()
127
+
if err != nil {
128
+
return fmt.Errorf("failed to start txn: %w", err)
129
+
}
130
+
131
+
triggerId, err := db.AddTrigger(tx, trigger)
132
+
if err != nil {
133
+
return fmt.Errorf("failed to add trigger entry: %w", err)
134
+
}
135
+
136
+
// TODO: we shouldn't even use knot to identify pipelines
137
+
knot := record.TriggerMetadata.Repo.Knot
138
+
pipeline := models.Pipeline{
139
+
Rkey: msg.Rkey,
140
+
Knot: knot,
141
+
RepoOwner: syntax.DID(record.TriggerMetadata.Repo.Did),
142
+
RepoName: record.TriggerMetadata.Repo.Repo,
143
+
TriggerId: int(triggerId),
144
+
Sha: sha,
145
+
}
146
+
147
+
err = db.AddPipeline(tx, pipeline)
148
+
if err != nil {
149
+
return fmt.Errorf("failed to add pipeline: %w", err)
150
+
}
151
+
152
+
err = tx.Commit()
153
+
if err != nil {
154
+
return fmt.Errorf("failed to commit txn: %w", err)
155
+
}
156
+
157
+
l.Info("added pipeline", "pipeline", pipeline)
158
+
159
+
return nil
71
160
}
72
161
73
162
func ingestPipelineStatus(ctx context.Context, logger *slog.Logger, d *db.DB, source ec.Source, msg ec.Message) error {
+58
-33
appview/state/star.go
+58
-33
appview/state/star.go
···
38
38
39
39
switch r.Method {
40
40
case http.MethodPost:
41
-
createdAt := time.Now().Format(time.RFC3339)
42
-
rkey := tid.TID()
41
+
star := models.Star{
42
+
Did: currentUser.Active.Did,
43
+
Rkey: tid.TID(),
44
+
RepoAt: subjectUri,
45
+
Created: time.Now(),
46
+
}
47
+
48
+
tx, err := s.db.BeginTx(r.Context(), nil)
49
+
if err != nil {
50
+
s.logger.Error("failed to start transaction", "err", err)
51
+
return
52
+
}
53
+
defer tx.Rollback()
54
+
55
+
if err := db.UpsertStar(tx, star); err != nil {
56
+
s.logger.Error("failed to star", "err", err)
57
+
return
58
+
}
59
+
60
+
record := star.AsRecord()
43
61
resp, err := comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{
44
62
Collection: tangled.FeedStarNSID,
45
63
Repo: currentUser.Active.Did,
46
-
Rkey: rkey,
64
+
Rkey: star.Rkey,
47
65
Record: &lexutil.LexiconTypeDecoder{
48
-
Val: &tangled.FeedStar{
49
-
Subject: subjectUri.String(),
50
-
CreatedAt: createdAt,
51
-
}},
66
+
Val: &record,
67
+
},
52
68
})
53
69
if err != nil {
54
70
log.Println("failed to create atproto record", err)
···
56
72
}
57
73
log.Println("created atproto record: ", resp.Uri)
58
74
59
-
star := &models.Star{
60
-
Did: currentUser.Active.Did,
61
-
RepoAt: subjectUri,
62
-
Rkey: rkey,
75
+
if err := tx.Commit(); err != nil {
76
+
s.logger.Error("failed to commit transaction", "err", err)
77
+
// DB op failed but record is created in PDS. Ingester will backfill the missed operation
63
78
}
64
79
65
-
err = db.AddStar(s.db, star)
66
-
if err != nil {
67
-
log.Println("failed to star", err)
68
-
return
69
-
}
80
+
s.notifier.NewStar(r.Context(), &star)
70
81
71
82
starCount, err := db.GetStarCount(s.db, subjectUri)
72
83
if err != nil {
73
84
log.Println("failed to get star count for ", subjectUri)
74
85
}
75
-
76
-
s.notifier.NewStar(r.Context(), star)
77
86
78
87
s.pages.StarBtnFragment(w, pages.StarBtnFragmentParams{
79
88
IsStarred: true,
···
83
92
84
93
return
85
94
case http.MethodDelete:
86
-
// find the record in the db
87
-
star, err := db.GetStar(s.db, currentUser.Active.Did, subjectUri)
95
+
tx, err := s.db.BeginTx(r.Context(), nil)
88
96
if err != nil {
89
-
log.Println("failed to get star relationship")
97
+
s.logger.Error("failed to start transaction", "err", err)
98
+
}
99
+
defer tx.Rollback()
100
+
101
+
stars, err := db.DeleteStar(tx, syntax.DID(currentUser.Active.Did), subjectUri)
102
+
if err != nil {
103
+
s.logger.Error("failed to delete stars from db", "err", err)
90
104
return
91
105
}
92
106
93
-
_, err = comatproto.RepoDeleteRecord(r.Context(), client, &comatproto.RepoDeleteRecord_Input{
94
-
Collection: tangled.FeedStarNSID,
95
-
Repo: currentUser.Active.Did,
96
-
Rkey: star.Rkey,
107
+
var writes []*comatproto.RepoApplyWrites_Input_Writes_Elem
108
+
for _, starAt := range stars {
109
+
writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{
110
+
RepoApplyWrites_Delete: &comatproto.RepoApplyWrites_Delete{
111
+
Collection: tangled.FeedStarNSID,
112
+
Rkey: starAt.RecordKey().String(),
113
+
},
114
+
})
115
+
}
116
+
_, err = comatproto.RepoApplyWrites(r.Context(), client, &comatproto.RepoApplyWrites_Input{
117
+
Repo: currentUser.Active.Did,
118
+
Writes: writes,
97
119
})
98
-
99
120
if err != nil {
100
-
log.Println("failed to unstar")
121
+
s.logger.Error("failed to delete stars from PDS", "err", err)
101
122
return
102
123
}
103
124
104
-
err = db.DeleteStarByRkey(s.db, currentUser.Active.Did, star.Rkey)
105
-
if err != nil {
106
-
log.Println("failed to delete star from DB")
107
-
// this is not an issue, the firehose event might have already done this
125
+
if err := tx.Commit(); err != nil {
126
+
s.logger.Error("failed to commit transaction", "err", err)
127
+
// DB op failed but record is created in PDS. Ingester will backfill the missed operation
108
128
}
109
129
130
+
s.notifier.DeleteStar(r.Context(), &models.Star{
131
+
Did: currentUser.Active.Did,
132
+
RepoAt: subjectUri,
133
+
// Rkey
134
+
// Created
135
+
})
136
+
110
137
starCount, err := db.GetStarCount(s.db, subjectUri)
111
138
if err != nil {
112
139
log.Println("failed to get star count for ", subjectUri)
113
140
return
114
141
}
115
-
116
-
s.notifier.DeleteStar(r.Context(), star)
117
142
118
143
s.pages.StarBtnFragment(w, pages.StarBtnFragmentParams{
119
144
IsStarred: false,
-5
appview/state/state.go
-5
appview/state/state.go
···
23
23
"tangled.org/core/appview/oauth"
24
24
"tangled.org/core/appview/pages"
25
25
"tangled.org/core/appview/reporesolver"
26
-
"tangled.org/core/appview/validator"
27
26
xrpcclient "tangled.org/core/appview/xrpcclient"
28
27
"tangled.org/core/eventconsumer"
29
28
"tangled.org/core/idresolver"
···
59
58
knotstream *eventconsumer.Consumer
60
59
spindlestream *eventconsumer.Consumer
61
60
logger *slog.Logger
62
-
validator *validator.Validator
63
61
}
64
62
65
63
func Make(ctx context.Context, config *config.Config) (*State, error) {
···
97
95
if err != nil {
98
96
return nil, fmt.Errorf("failed to start oauth handler: %w", err)
99
97
}
100
-
validator := validator.New(d, res, enforcer)
101
98
102
99
repoResolver := reporesolver.New(config, enforcer, d)
103
100
···
144
141
IdResolver: res,
145
142
Config: config,
146
143
Logger: log.SubLogger(logger, "ingester"),
147
-
Validator: validator,
148
144
}
149
145
err = jc.StartJetstream(ctx, ingester.Ingest())
150
146
if err != nil {
···
191
187
knotstream,
192
188
spindlestream,
193
189
logger,
194
-
validator,
195
190
}
196
191
197
192
return state, nil
-55
appview/validator/issue.go
-55
appview/validator/issue.go
···
1
-
package validator
2
-
3
-
import (
4
-
"fmt"
5
-
"strings"
6
-
7
-
"tangled.org/core/appview/db"
8
-
"tangled.org/core/appview/models"
9
-
"tangled.org/core/orm"
10
-
)
11
-
12
-
func (v *Validator) ValidateIssueComment(comment *models.IssueComment) error {
13
-
// if comments have parents, only ingest ones that are 1 level deep
14
-
if comment.ReplyTo != nil {
15
-
parents, err := db.GetIssueComments(v.db, orm.FilterEq("at_uri", *comment.ReplyTo))
16
-
if err != nil {
17
-
return fmt.Errorf("failed to fetch parent comment: %w", err)
18
-
}
19
-
if len(parents) != 1 {
20
-
return fmt.Errorf("incorrect number of parent comments returned: %d", len(parents))
21
-
}
22
-
23
-
// depth check
24
-
parent := parents[0]
25
-
if parent.ReplyTo != nil {
26
-
return fmt.Errorf("incorrect depth, this comment is replying at depth >1")
27
-
}
28
-
}
29
-
30
-
if sb := strings.TrimSpace(v.sanitizer.SanitizeDefault(comment.Body)); sb == "" {
31
-
return fmt.Errorf("body is empty after HTML sanitization")
32
-
}
33
-
34
-
return nil
35
-
}
36
-
37
-
func (v *Validator) ValidateIssue(issue *models.Issue) error {
38
-
if issue.Title == "" {
39
-
return fmt.Errorf("issue title is empty")
40
-
}
41
-
42
-
if issue.Body == "" {
43
-
return fmt.Errorf("issue body is empty")
44
-
}
45
-
46
-
if st := strings.TrimSpace(v.sanitizer.SanitizeDescription(issue.Title)); st == "" {
47
-
return fmt.Errorf("title is empty after HTML sanitization")
48
-
}
49
-
50
-
if sb := strings.TrimSpace(v.sanitizer.SanitizeDefault(issue.Body)); sb == "" {
51
-
return fmt.Errorf("body is empty after HTML sanitization")
52
-
}
53
-
54
-
return nil
55
-
}
-217
appview/validator/label.go
-217
appview/validator/label.go
···
1
-
package validator
2
-
3
-
import (
4
-
"context"
5
-
"fmt"
6
-
"regexp"
7
-
"strings"
8
-
9
-
"github.com/bluesky-social/indigo/atproto/syntax"
10
-
"golang.org/x/exp/slices"
11
-
"tangled.org/core/api/tangled"
12
-
"tangled.org/core/appview/models"
13
-
)
14
-
15
-
var (
16
-
// Label name should be alphanumeric with hyphens/underscores, but not start/end with them
17
-
labelNameRegex = regexp.MustCompile(`^[a-zA-Z0-9]([a-zA-Z0-9_-]*[a-zA-Z0-9])?$`)
18
-
// Color should be a valid hex color
19
-
colorRegex = regexp.MustCompile(`^#[a-fA-F0-9]{6}$`)
20
-
// You can only label issues and pulls presently
21
-
validScopes = []string{tangled.RepoIssueNSID, tangled.RepoPullNSID}
22
-
)
23
-
24
-
func (v *Validator) ValidateLabelDefinition(label *models.LabelDefinition) error {
25
-
if label.Name == "" {
26
-
return fmt.Errorf("label name is empty")
27
-
}
28
-
if len(label.Name) > 40 {
29
-
return fmt.Errorf("label name too long (max 40 graphemes)")
30
-
}
31
-
if len(label.Name) < 1 {
32
-
return fmt.Errorf("label name too short (min 1 grapheme)")
33
-
}
34
-
if !labelNameRegex.MatchString(label.Name) {
35
-
return fmt.Errorf("label name contains invalid characters (use only letters, numbers, hyphens, and underscores)")
36
-
}
37
-
38
-
if !label.ValueType.IsConcreteType() {
39
-
return fmt.Errorf("invalid value type: %q (must be one of: null, boolean, integer, string)", label.ValueType.Type)
40
-
}
41
-
42
-
// null type checks: cannot be enums, multiple or explicit format
43
-
if label.ValueType.IsNull() && label.ValueType.IsEnum() {
44
-
return fmt.Errorf("null type cannot be used in conjunction with enum type")
45
-
}
46
-
if label.ValueType.IsNull() && label.Multiple {
47
-
return fmt.Errorf("null type labels cannot be multiple")
48
-
}
49
-
if label.ValueType.IsNull() && !label.ValueType.IsAnyFormat() {
50
-
return fmt.Errorf("format cannot be used in conjunction with null type")
51
-
}
52
-
53
-
// format checks: cannot be used with enum, or integers
54
-
if !label.ValueType.IsAnyFormat() && label.ValueType.IsEnum() {
55
-
return fmt.Errorf("enum types cannot be used in conjunction with format specification")
56
-
}
57
-
58
-
if !label.ValueType.IsAnyFormat() && !label.ValueType.IsString() {
59
-
return fmt.Errorf("format specifications are only permitted on string types")
60
-
}
61
-
62
-
// validate scope (nsid format)
63
-
if label.Scope == nil {
64
-
return fmt.Errorf("scope is required")
65
-
}
66
-
for _, s := range label.Scope {
67
-
if _, err := syntax.ParseNSID(s); err != nil {
68
-
return fmt.Errorf("failed to parse scope: %w", err)
69
-
}
70
-
if !slices.Contains(validScopes, s) {
71
-
return fmt.Errorf("invalid scope: scope must be present in %q", validScopes)
72
-
}
73
-
}
74
-
75
-
// validate color if provided
76
-
if label.Color != nil {
77
-
color := strings.TrimSpace(*label.Color)
78
-
if color == "" {
79
-
// empty color is fine, set to nil
80
-
label.Color = nil
81
-
} else {
82
-
if !colorRegex.MatchString(color) {
83
-
return fmt.Errorf("color must be a valid hex color (e.g. #79FFE1 or #000)")
84
-
}
85
-
// expand 3-digit hex to 6-digit hex
86
-
if len(color) == 4 { // #ABC
87
-
color = fmt.Sprintf("#%c%c%c%c%c%c", color[1], color[1], color[2], color[2], color[3], color[3])
88
-
}
89
-
// convert to uppercase for consistency
90
-
color = strings.ToUpper(color)
91
-
label.Color = &color
92
-
}
93
-
}
94
-
95
-
return nil
96
-
}
97
-
98
-
func (v *Validator) ValidateLabelOp(labelDef *models.LabelDefinition, repo *models.Repo, labelOp *models.LabelOp) error {
99
-
if labelDef == nil {
100
-
return fmt.Errorf("label definition is required")
101
-
}
102
-
if repo == nil {
103
-
return fmt.Errorf("repo is required")
104
-
}
105
-
if labelOp == nil {
106
-
return fmt.Errorf("label operation is required")
107
-
}
108
-
109
-
// validate permissions: only collaborators can apply labels currently
110
-
//
111
-
// TODO: introduce a repo:triage permission
112
-
ok, err := v.enforcer.IsPushAllowed(labelOp.Did, repo.Knot, repo.DidSlashRepo())
113
-
if err != nil {
114
-
return fmt.Errorf("failed to enforce permissions: %w", err)
115
-
}
116
-
if !ok {
117
-
return fmt.Errorf("unauhtorized label operation")
118
-
}
119
-
120
-
expectedKey := labelDef.AtUri().String()
121
-
if labelOp.OperandKey != expectedKey {
122
-
return fmt.Errorf("operand key %q does not match label definition URI %q", labelOp.OperandKey, expectedKey)
123
-
}
124
-
125
-
if labelOp.Operation != models.LabelOperationAdd && labelOp.Operation != models.LabelOperationDel {
126
-
return fmt.Errorf("invalid operation: %q (must be 'add' or 'del')", labelOp.Operation)
127
-
}
128
-
129
-
if labelOp.Subject == "" {
130
-
return fmt.Errorf("subject URI is required")
131
-
}
132
-
if _, err := syntax.ParseATURI(string(labelOp.Subject)); err != nil {
133
-
return fmt.Errorf("invalid subject URI: %w", err)
134
-
}
135
-
136
-
if err := v.validateOperandValue(labelDef, labelOp); err != nil {
137
-
return fmt.Errorf("invalid operand value: %w", err)
138
-
}
139
-
140
-
// Validate performed time is not zero/invalid
141
-
if labelOp.PerformedAt.IsZero() {
142
-
return fmt.Errorf("performed_at timestamp is required")
143
-
}
144
-
145
-
return nil
146
-
}
147
-
148
-
func (v *Validator) validateOperandValue(labelDef *models.LabelDefinition, labelOp *models.LabelOp) error {
149
-
valueType := labelDef.ValueType
150
-
151
-
// this is permitted, it "unsets" a label
152
-
if labelOp.OperandValue == "" {
153
-
labelOp.Operation = models.LabelOperationDel
154
-
return nil
155
-
}
156
-
157
-
switch valueType.Type {
158
-
case models.ConcreteTypeNull:
159
-
// For null type, value should be empty
160
-
if labelOp.OperandValue != "null" {
161
-
return fmt.Errorf("null type requires empty value, got %q", labelOp.OperandValue)
162
-
}
163
-
164
-
case models.ConcreteTypeString:
165
-
// For string type, validate enum constraints if present
166
-
if valueType.IsEnum() {
167
-
if !slices.Contains(valueType.Enum, labelOp.OperandValue) {
168
-
return fmt.Errorf("value %q is not in allowed enum values %v", labelOp.OperandValue, valueType.Enum)
169
-
}
170
-
}
171
-
172
-
switch valueType.Format {
173
-
case models.ValueTypeFormatDid:
174
-
id, err := v.resolver.ResolveIdent(context.Background(), labelOp.OperandValue)
175
-
if err != nil {
176
-
return fmt.Errorf("failed to resolve did/handle: %w", err)
177
-
}
178
-
179
-
labelOp.OperandValue = id.DID.String()
180
-
181
-
case models.ValueTypeFormatAny, "":
182
-
default:
183
-
return fmt.Errorf("unsupported format constraint: %q", valueType.Format)
184
-
}
185
-
186
-
case models.ConcreteTypeInt:
187
-
if labelOp.OperandValue == "" {
188
-
return fmt.Errorf("integer type requires non-empty value")
189
-
}
190
-
if _, err := fmt.Sscanf(labelOp.OperandValue, "%d", new(int)); err != nil {
191
-
return fmt.Errorf("value %q is not a valid integer", labelOp.OperandValue)
192
-
}
193
-
194
-
if valueType.IsEnum() {
195
-
if !slices.Contains(valueType.Enum, labelOp.OperandValue) {
196
-
return fmt.Errorf("value %q is not in allowed enum values %v", labelOp.OperandValue, valueType.Enum)
197
-
}
198
-
}
199
-
200
-
case models.ConcreteTypeBool:
201
-
if labelOp.OperandValue != "true" && labelOp.OperandValue != "false" {
202
-
return fmt.Errorf("boolean type requires value to be 'true' or 'false', got %q", labelOp.OperandValue)
203
-
}
204
-
205
-
// validate enum constraints if present (though uncommon for booleans)
206
-
if valueType.IsEnum() {
207
-
if !slices.Contains(valueType.Enum, labelOp.OperandValue) {
208
-
return fmt.Errorf("value %q is not in allowed enum values %v", labelOp.OperandValue, valueType.Enum)
209
-
}
210
-
}
211
-
212
-
default:
213
-
return fmt.Errorf("unsupported value type: %q", valueType.Type)
214
-
}
215
-
216
-
return nil
217
-
}
-25
appview/validator/patch.go
-25
appview/validator/patch.go
···
1
-
package validator
2
-
3
-
import (
4
-
"fmt"
5
-
"strings"
6
-
7
-
"tangled.org/core/patchutil"
8
-
)
9
-
10
-
func (v *Validator) ValidatePatch(patch *string) error {
11
-
if patch == nil || *patch == "" {
12
-
return fmt.Errorf("patch is empty")
13
-
}
14
-
15
-
// add newline if not present to diff style patches
16
-
if !patchutil.IsFormatPatch(*patch) && !strings.HasSuffix(*patch, "\n") {
17
-
*patch = *patch + "\n"
18
-
}
19
-
20
-
if err := patchutil.IsPatchValid(*patch); err != nil {
21
-
return err
22
-
}
23
-
24
-
return nil
25
-
}
-53
appview/validator/repo_topics.go
-53
appview/validator/repo_topics.go
···
1
-
package validator
2
-
3
-
import (
4
-
"fmt"
5
-
"maps"
6
-
"regexp"
7
-
"slices"
8
-
"strings"
9
-
)
10
-
11
-
const (
12
-
maxTopicLen = 50
13
-
maxTopics = 20
14
-
)
15
-
16
-
var (
17
-
topicRE = regexp.MustCompile(`\A[a-z0-9-]+\z`)
18
-
)
19
-
20
-
// ValidateRepoTopicStr parses and validates whitespace-separated topic string.
21
-
//
22
-
// Rules:
23
-
// - topics are separated by whitespace
24
-
// - each topic may contain lowercase letters, digits, and hyphens only
25
-
// - each topic must be <= 50 characters long
26
-
// - no more than 20 topics allowed
27
-
// - duplicates are removed
28
-
func (v *Validator) ValidateRepoTopicStr(topicsStr string) ([]string, error) {
29
-
topicsStr = strings.TrimSpace(topicsStr)
30
-
if topicsStr == "" {
31
-
return nil, nil
32
-
}
33
-
parts := strings.Fields(topicsStr)
34
-
if len(parts) > maxTopics {
35
-
return nil, fmt.Errorf("too many topics: %d (maximum %d)", len(parts), maxTopics)
36
-
}
37
-
38
-
topicSet := make(map[string]struct{})
39
-
40
-
for _, t := range parts {
41
-
if _, exists := topicSet[t]; exists {
42
-
continue
43
-
}
44
-
if len(t) > maxTopicLen {
45
-
return nil, fmt.Errorf("topic '%s' is too long (maximum %d characters)", t, maxTopics)
46
-
}
47
-
if !topicRE.MatchString(t) {
48
-
return nil, fmt.Errorf("topic '%s' contains invalid characters (allowed: lowercase letters, digits, hyphens)", t)
49
-
}
50
-
topicSet[t] = struct{}{}
51
-
}
52
-
return slices.Collect(maps.Keys(topicSet)), nil
53
-
}
-27
appview/validator/string.go
-27
appview/validator/string.go
···
1
-
package validator
2
-
3
-
import (
4
-
"errors"
5
-
"fmt"
6
-
"unicode/utf8"
7
-
8
-
"tangled.org/core/appview/models"
9
-
)
10
-
11
-
func (v *Validator) ValidateString(s *models.String) error {
12
-
var err error
13
-
14
-
if utf8.RuneCountInString(s.Filename) > 140 {
15
-
err = errors.Join(err, fmt.Errorf("filename too long"))
16
-
}
17
-
18
-
if utf8.RuneCountInString(s.Description) > 280 {
19
-
err = errors.Join(err, fmt.Errorf("description too long"))
20
-
}
21
-
22
-
if len(s.Contents) == 0 {
23
-
err = errors.Join(err, fmt.Errorf("contents is empty"))
24
-
}
25
-
26
-
return err
27
-
}
-17
appview/validator/uri.go
-17
appview/validator/uri.go
···
1
-
package validator
2
-
3
-
import (
4
-
"fmt"
5
-
"net/url"
6
-
)
7
-
8
-
func (v *Validator) ValidateURI(uri string) error {
9
-
parsed, err := url.Parse(uri)
10
-
if err != nil {
11
-
return fmt.Errorf("invalid uri format")
12
-
}
13
-
if parsed.Scheme == "" {
14
-
return fmt.Errorf("uri scheme missing")
15
-
}
16
-
return nil
17
-
}
-24
appview/validator/validator.go
-24
appview/validator/validator.go
···
1
-
package validator
2
-
3
-
import (
4
-
"tangled.org/core/appview/db"
5
-
"tangled.org/core/appview/pages/markup"
6
-
"tangled.org/core/idresolver"
7
-
"tangled.org/core/rbac"
8
-
)
9
-
10
-
type Validator struct {
11
-
db *db.DB
12
-
sanitizer markup.Sanitizer
13
-
resolver *idresolver.Resolver
14
-
enforcer *rbac.Enforcer
15
-
}
16
-
17
-
func New(db *db.DB, res *idresolver.Resolver, enforcer *rbac.Enforcer) *Validator {
18
-
return &Validator{
19
-
db: db,
20
-
sanitizer: markup.NewSanitizer(),
21
-
resolver: res,
22
-
enforcer: enforcer,
23
-
}
24
-
}
+11
contrib/certs/root.crt
+11
contrib/certs/root.crt
···
1
+
-----BEGIN CERTIFICATE-----
2
+
MIIBozCCAUmgAwIBAgIQRnYoKs3BuihlLFeydgURVzAKBggqhkjOPQQDAjAwMS4w
3
+
LAYDVQQDEyVDYWRkeSBMb2NhbCBBdXRob3JpdHkgLSAyMDI2IEVDQyBSb290MB4X
4
+
DTI2MDEwODEzNTk1MloXDTM1MTExNzEzNTk1MlowMDEuMCwGA1UEAxMlQ2FkZHkg
5
+
TG9jYWwgQXV0aG9yaXR5IC0gMjAyNiBFQ0MgUm9vdDBZMBMGByqGSM49AgEGCCqG
6
+
SM49AwEHA0IABCQlYShhxLaX8/ZP7rcBtD5xL4u3wYMe77JS/lRFjjpAUGmJPxUE
7
+
ctsNvukG1hU4MeLMSqAEIqFWjs8dQBxLjGSjRTBDMA4GA1UdDwEB/wQEAwIBBjAS
8
+
BgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQWBBQ7Mt/6izTOOXCSWDS6HrwrqMDB
9
+
vzAKBggqhkjOPQQDAgNIADBFAiEA9QAYIuHR5qsGJ1JMZnuAAQpEwaqewhUICsKO
10
+
e2fWj4ACICPgj9Kh9++8FH5eVyDI1AD/BLwmMmiaqs1ojZT7QJqb
11
+
-----END CERTIFICATE-----
+31
contrib/example.env
+31
contrib/example.env
···
1
+
# NOTE: put actual DIDs here
2
+
alice_did=did:plc:alice-did
3
+
tangled_did=did:plc:tangled-did
4
+
5
+
#core
6
+
export TANGLED_DEV=true
7
+
export TANGLED_APPVIEW_HOST=127.0.0.1:3000
8
+
# plc
9
+
export TANGLED_PLC_URL=https://plc.tngl.boltless.dev
10
+
# jetstream
11
+
export TANGLED_JETSTREAM_ENDPOINT=wss://jetstream.tngl.boltless.dev/subscribe
12
+
# label
13
+
export TANGLED_LABEL_GFI=at://${tangled_did}/sh.tangled.label.definition/good-first-issue
14
+
export TANGLED_LABEL_DEFAULTS=$TANGLED_LABEL_GFI
15
+
export TANGLED_LABEL_DEFAULTS=$TANGLED_LABEL_DEFAULTS,at://${tangled_did}/sh.tangled.label.definition/assignee
16
+
export TANGLED_LABEL_DEFAULTS=$TANGLED_LABEL_DEFAULTS,at://${tangled_did}/sh.tangled.label.definition/documentation
17
+
export TANGLED_LABEL_DEFAULTS=$TANGLED_LABEL_DEFAULTS,at://${tangled_did}/sh.tangled.label.definition/duplicate
18
+
export TANGLED_LABEL_DEFAULTS=$TANGLED_LABEL_DEFAULTS,at://${tangled_did}/sh.tangled.label.definition/wontfix
19
+
20
+
# vm settings
21
+
export TANGLED_VM_PLC_URL=https://plc.tngl.boltless.dev
22
+
export TANGLED_VM_JETSTREAM_ENDPOINT=wss://jetstream.tngl.boltless.dev/subscribe
23
+
export TANGLED_VM_KNOT_HOST=knot.tngl.boltless.dev
24
+
export TANGLED_VM_KNOT_OWNER=$alice_did
25
+
export TANGLED_VM_SPINDLE_HOST=spindle.tngl.boltless.dev
26
+
export TANGLED_VM_SPINDLE_OWNER=$alice_did
27
+
28
+
if [ -n "${TANGLED_RESEND_API_KEY:-}" ] && [ -n "${TANGLED_RESEND_SENT_FROM:-}" ]; then
29
+
export TANGLED_VM_PDS_EMAIL_SMTP_URL=smtps://resend:$TANGLED_RESEND_API_KEY@smtp.resend.com:465/
30
+
export TANGLED_VM_PDS_EMAIL_FROM_ADDRESS=$TANGLED_RESEND_SENT_FROM
31
+
fi
+12
contrib/pds.env
+12
contrib/pds.env
···
1
+
LOG_ENABLED=true
2
+
3
+
PDS_JWT_SECRET=8cae8bffcc73d9932819650791e4e89a
4
+
PDS_ADMIN_PASSWORD=d6a902588cd93bee1af83f924f60cfd3
5
+
PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX=2e92e336a50a618458e1097d94a1db86ec3fd8829d7735020cbae80625c761d7
6
+
7
+
PDS_DATA_DIRECTORY=/pds
8
+
PDS_BLOBSTORE_DISK_LOCATION=/pds/blocks
9
+
10
+
PDS_DID_PLC_URL=http://localhost:8080
11
+
PDS_HOSTNAME=pds.tngl.boltless.dev
12
+
PDS_PORT=3000
+25
contrib/readme.md
+25
contrib/readme.md
···
1
+
# how to setup local appview dev environment
2
+
3
+
Appview requires several microservices from knot and spindle to entire atproto infra. This test environment is implemented under nixos vm.
4
+
5
+
1. copy `contrib/example.env` to `.env`, fill it and source it
6
+
2. run vm
7
+
```bash
8
+
nix run --impure .#vm
9
+
```
10
+
3. trust the generated cert from host machine
11
+
```bash
12
+
# for macos
13
+
sudo security add-trusted-cert -d -r trustRoot \
14
+
-k /Library/Keychains/System.keychain \
15
+
./nix/vm-data/caddy/.local/share/caddy/pki/authorities/local/root.crt
16
+
```
17
+
4. create test accounts with valid emails (use [`create-test-account.sh`](./scripts/create-test-account.sh))
18
+
5. create default labels (use [`setup-const-records`](./scripts/setup-const-records.sh))
19
+
6. restart vm with correct owner-did
20
+
21
+
for git-https, you should change your local git config:
22
+
```
23
+
[http "https://knot.tngl.boltless.dev"]
24
+
sslCAPath = /Users/boltless/repo/tangled/nix/vm-data/caddy/.local/share/caddy/pki/authorities/local/
25
+
```
+68
contrib/scripts/create-test-account.sh
+68
contrib/scripts/create-test-account.sh
···
1
+
#!/bin/bash
2
+
set -o errexit
3
+
set -o nounset
4
+
set -o pipefail
5
+
6
+
source "$(dirname "$0")/../pds.env"
7
+
8
+
# PDS_HOSTNAME=
9
+
# PDS_ADMIN_PASSWORD=
10
+
11
+
# curl a URL and fail if the request fails.
12
+
function curl_cmd_get {
13
+
curl --fail --silent --show-error "$@"
14
+
}
15
+
16
+
# curl a URL and fail if the request fails.
17
+
function curl_cmd_post {
18
+
curl --fail --silent --show-error --request POST --header "Content-Type: application/json" "$@"
19
+
}
20
+
21
+
# curl a URL but do not fail if the request fails.
22
+
function curl_cmd_post_nofail {
23
+
curl --silent --show-error --request POST --header "Content-Type: application/json" "$@"
24
+
}
25
+
26
+
USERNAME="${1:-}"
27
+
28
+
if [[ "${USERNAME}" == "" ]]; then
29
+
read -p "Enter a username: " USERNAME
30
+
fi
31
+
32
+
if [[ "${USERNAME}" == "" ]]; then
33
+
echo "ERROR: missing USERNAME parameter." >/dev/stderr
34
+
echo "Usage: $0 ${SUBCOMMAND} <USERNAME>" >/dev/stderr
35
+
exit 1
36
+
fi
37
+
38
+
EMAIL=${USERNAME}@${PDS_HOSTNAME}
39
+
40
+
PASSWORD="password"
41
+
INVITE_CODE="$(curl_cmd_post \
42
+
--user "admin:${PDS_ADMIN_PASSWORD}" \
43
+
--data '{"useCount": 1}' \
44
+
"https://${PDS_HOSTNAME}/xrpc/com.atproto.server.createInviteCode" | jq --raw-output '.code'
45
+
)"
46
+
RESULT="$(curl_cmd_post_nofail \
47
+
--data "{\"email\":\"${EMAIL}\", \"handle\":\"${USERNAME}.${PDS_HOSTNAME}\", \"password\":\"${PASSWORD}\", \"inviteCode\":\"${INVITE_CODE}\"}" \
48
+
"https://${PDS_HOSTNAME}/xrpc/com.atproto.server.createAccount"
49
+
)"
50
+
51
+
DID="$(echo $RESULT | jq --raw-output '.did')"
52
+
if [[ "${DID}" != did:* ]]; then
53
+
ERR="$(echo ${RESULT} | jq --raw-output '.message')"
54
+
echo "ERROR: ${ERR}" >/dev/stderr
55
+
echo "Usage: $0 <EMAIL> <HANDLE>" >/dev/stderr
56
+
exit 1
57
+
fi
58
+
59
+
echo
60
+
echo "Account created successfully!"
61
+
echo "-----------------------------"
62
+
echo "Handle : ${USERNAME}.${PDS_HOSTNAME}"
63
+
echo "DID : ${DID}"
64
+
echo "Password : ${PASSWORD}"
65
+
echo "-----------------------------"
66
+
echo "This is a test account with an insecure password."
67
+
echo "Make sure it's only used for development."
68
+
echo
+106
contrib/scripts/setup-const-records.sh
+106
contrib/scripts/setup-const-records.sh
···
1
+
#!/bin/bash
2
+
set -o errexit
3
+
set -o nounset
4
+
set -o pipefail
5
+
6
+
source "$(dirname "$0")/../pds.env"
7
+
8
+
# PDS_HOSTNAME=
9
+
10
+
# curl a URL and fail if the request fails.
11
+
function curl_cmd_get {
12
+
curl --fail --silent --show-error "$@"
13
+
}
14
+
15
+
# curl a URL and fail if the request fails.
16
+
function curl_cmd_post {
17
+
curl --fail --silent --show-error --request POST --header "Content-Type: application/json" "$@"
18
+
}
19
+
20
+
# curl a URL but do not fail if the request fails.
21
+
function curl_cmd_post_nofail {
22
+
curl --silent --show-error --request POST --header "Content-Type: application/json" "$@"
23
+
}
24
+
25
+
USERNAME="${1:-}"
26
+
27
+
if [[ "${USERNAME}" == "" ]]; then
28
+
read -p "Enter a username: " USERNAME
29
+
fi
30
+
31
+
if [[ "${USERNAME}" == "" ]]; then
32
+
echo "ERROR: missing USERNAME parameter." >/dev/stderr
33
+
echo "Usage: $0 ${SUBCOMMAND} <USERNAME>" >/dev/stderr
34
+
exit 1
35
+
fi
36
+
37
+
SESS_RESULT="$(curl_cmd_post \
38
+
--data "$(cat <<EOF
39
+
{
40
+
"identifier": "$USERNAME",
41
+
"password": "password"
42
+
}
43
+
EOF
44
+
)" \
45
+
https://pds.tngl.boltless.dev/xrpc/com.atproto.server.createSession
46
+
)"
47
+
48
+
echo $SESS_RESULT | jq
49
+
50
+
DID="$(echo $SESS_RESULT | jq --raw-output '.did')"
51
+
ACCESS_JWT="$(echo $SESS_RESULT | jq --raw-output '.accessJwt')"
52
+
53
+
function add_label_def {
54
+
local color=$1
55
+
local name=$2
56
+
echo $color
57
+
echo $name
58
+
local json_payload=$(cat <<EOF
59
+
{
60
+
"repo": "$DID",
61
+
"collection": "sh.tangled.label.definition",
62
+
"rkey": "$name",
63
+
"record": {
64
+
"name": "$name",
65
+
"color": "$color",
66
+
"scope": ["sh.tangled.repo.issue"],
67
+
"multiple": false,
68
+
"createdAt": "2025-09-22T11:14:35+01:00",
69
+
"valueType": {"type": "null", "format": "any"}
70
+
}
71
+
}
72
+
EOF
73
+
)
74
+
echo $json_payload
75
+
echo $json_payload | jq
76
+
RESULT="$(curl_cmd_post \
77
+
--data "$json_payload" \
78
+
-H "Authorization: Bearer ${ACCESS_JWT}" \
79
+
"https://${PDS_HOSTNAME}/xrpc/com.atproto.repo.createRecord")"
80
+
echo $RESULT | jq
81
+
}
82
+
83
+
add_label_def '#64748b' 'wontfix'
84
+
add_label_def '#8B5CF6' 'good-first-issue'
85
+
add_label_def '#ef4444' 'duplicate'
86
+
add_label_def '#06b6d4' 'documentation'
87
+
json_payload=$(cat <<EOF
88
+
{
89
+
"repo": "$DID",
90
+
"collection": "sh.tangled.label.definition",
91
+
"rkey": "assignee",
92
+
"record": {
93
+
"name": "assignee",
94
+
"color": "#10B981",
95
+
"scope": ["sh.tangled.repo.issue", "sh.tangled.repo.pull"],
96
+
"multiple": false,
97
+
"createdAt": "2025-09-22T11:14:35+01:00",
98
+
"valueType": {"type": "string", "format": "did"}
99
+
}
100
+
}
101
+
EOF
102
+
)
103
+
curl_cmd_post \
104
+
--data "$json_payload" \
105
+
-H "Authorization: Bearer ${ACCESS_JWT}" \
106
+
"https://${PDS_HOSTNAME}/xrpc/com.atproto.repo.createRecord"
+35
-2
flake.nix
+35
-2
flake.nix
···
95
95
knot-unwrapped = self.callPackage ./nix/pkgs/knot-unwrapped.nix {};
96
96
knot = self.callPackage ./nix/pkgs/knot.nix {};
97
97
dolly = self.callPackage ./nix/pkgs/dolly.nix {};
98
+
did-method-plc = self.callPackage ./nix/pkgs/did-method-plc.nix {};
99
+
bluesky-jetstream = self.callPackage ./nix/pkgs/bluesky-jetstream.nix {};
100
+
bluesky-relay = self.callPackage ./nix/pkgs/bluesky-relay.nix {};
101
+
tap = self.callPackage ./nix/pkgs/tap.nix {};
98
102
});
99
103
in {
100
104
overlays.default = final: prev: {
101
-
inherit (mkPackageSet final) lexgen goat sqlite-lib spindle knot-unwrapped knot appview docs dolly;
105
+
inherit (mkPackageSet final) lexgen goat sqlite-lib spindle knot-unwrapped knot appview docs dolly did-method-plc bluesky-jetstream bluesky-relay tap;
102
106
};
103
107
104
108
packages = forAllSystems (system: let
···
119
123
sqlite-lib
120
124
docs
121
125
dolly
126
+
did-method-plc
127
+
bluesky-jetstream
128
+
bluesky-relay
129
+
tap
122
130
;
123
131
124
132
pkgsStatic-appview = staticPackages.appview;
···
248
256
rootDir=$(jj --ignore-working-copy root || git rev-parse --show-toplevel) || (echo "error: can't find repo root?"; exit 1)
249
257
cd "$rootDir"
250
258
251
-
mkdir -p nix/vm-data/{knot,repos,spindle,spindle-logs}
259
+
mkdir -p nix/vm-data/{caddy,knot,repos,spindle,spindle-logs}
252
260
253
261
export TANGLED_VM_DATA_DIR="$rootDir/nix/vm-data"
254
262
exec ${pkgs.lib.getExe
···
320
328
imports = [./nix/modules/spindle.nix];
321
329
322
330
services.tangled.spindle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.spindle;
331
+
services.tangled.spindle.tap-package = lib.mkDefault self.packages.${pkgs.system}.tap;
332
+
};
333
+
nixosModules.did-method-plc = {
334
+
lib,
335
+
pkgs,
336
+
...
337
+
}: {
338
+
imports = [./nix/modules/did-method-plc.nix];
339
+
services.did-method-plc.package = lib.mkDefault self.packages.${pkgs.system}.did-method-plc;
340
+
};
341
+
nixosModules.bluesky-relay = {
342
+
lib,
343
+
pkgs,
344
+
...
345
+
}: {
346
+
imports = [./nix/modules/bluesky-relay.nix];
347
+
services.bluesky-relay.package = lib.mkDefault self.packages.${pkgs.system}.bluesky-relay;
348
+
};
349
+
nixosModules.bluesky-jetstream = {
350
+
lib,
351
+
pkgs,
352
+
...
353
+
}: {
354
+
imports = [./nix/modules/bluesky-jetstream.nix];
355
+
services.bluesky-jetstream.package = lib.mkDefault self.packages.${pkgs.system}.bluesky-jetstream;
323
356
};
324
357
};
325
358
}
+1
go.mod
+1
go.mod
···
29
29
github.com/gorilla/feeds v1.2.0
30
30
github.com/gorilla/sessions v1.4.0
31
31
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
32
+
github.com/hashicorp/go-version v1.8.0
32
33
github.com/hiddeco/sshsig v0.2.0
33
34
github.com/hpcloud/tail v1.0.0
34
35
github.com/ipfs/go-cid v0.5.0
+2
go.sum
+2
go.sum
···
264
264
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
265
265
github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
266
266
github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
267
+
github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4=
268
+
github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
267
269
github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
268
270
github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
269
271
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+1
-1
input.css
+1
-1
input.css
···
96
96
@apply border border-gray-400 block rounded bg-gray-50 focus:ring-black p-3 dark:bg-gray-800 dark:border-gray-600 dark:text-white dark:focus:ring-gray-400;
97
97
}
98
98
textarea {
99
-
@apply border border-gray-400 block rounded bg-gray-50 focus:ring-black p-3 dark:bg-gray-800 dark:border-gray-600 dark:text-white dark:focus:ring-gray-400;
99
+
@apply border border-gray-400 block rounded bg-gray-50 focus:ring-black p-3 dark:bg-gray-800 dark:border-gray-600 dark:text-white dark:focus:ring-gray-400 font-mono;
100
100
}
101
101
details summary::-webkit-details-marker {
102
102
display: none;
+2
-1
jetstream/jetstream.go
+2
-1
jetstream/jetstream.go
···
159
159
j.cancelMu.Unlock()
160
160
161
161
if err := j.client.ConnectAndRead(connCtx, cursor); err != nil {
162
-
l.Error("error reading jetstream", "error", err)
162
+
l.Error("error reading jetstream, retry in 3s", "error", err)
163
163
cancel()
164
+
time.Sleep(3 * time.Second)
164
165
continue
165
166
}
166
167
-136
knotserver/ingester.go
-136
knotserver/ingester.go
···
7
7
"io"
8
8
"net/http"
9
9
"net/url"
10
-
"path/filepath"
11
10
"strings"
12
11
13
12
comatproto "github.com/bluesky-social/indigo/api/atproto"
···
17
16
securejoin "github.com/cyphar/filepath-securejoin"
18
17
"tangled.org/core/api/tangled"
19
18
"tangled.org/core/knotserver/db"
20
-
"tangled.org/core/knotserver/git"
21
19
"tangled.org/core/log"
22
20
"tangled.org/core/rbac"
23
-
"tangled.org/core/workflow"
24
21
)
25
22
26
23
func (h *Knot) processPublicKey(ctx context.Context, event *models.Event) error {
···
85
82
return nil
86
83
}
87
84
88
-
func (h *Knot) processPull(ctx context.Context, event *models.Event) error {
89
-
raw := json.RawMessage(event.Commit.Record)
90
-
did := event.Did
91
-
92
-
var record tangled.RepoPull
93
-
if err := json.Unmarshal(raw, &record); err != nil {
94
-
return fmt.Errorf("failed to unmarshal record: %w", err)
95
-
}
96
-
97
-
l := log.FromContext(ctx)
98
-
l = l.With("handler", "processPull")
99
-
l = l.With("did", did)
100
-
101
-
if record.Target == nil {
102
-
return fmt.Errorf("ignoring pull record: target repo is nil")
103
-
}
104
-
105
-
l = l.With("target_repo", record.Target.Repo)
106
-
l = l.With("target_branch", record.Target.Branch)
107
-
108
-
if record.Source == nil {
109
-
return fmt.Errorf("ignoring pull record: not a branch-based pull request")
110
-
}
111
-
112
-
if record.Source.Repo != nil {
113
-
return fmt.Errorf("ignoring pull record: fork based pull")
114
-
}
115
-
116
-
repoAt, err := syntax.ParseATURI(record.Target.Repo)
117
-
if err != nil {
118
-
return fmt.Errorf("failed to parse ATURI: %w", err)
119
-
}
120
-
121
-
// resolve this aturi to extract the repo record
122
-
ident, err := h.resolver.ResolveIdent(ctx, repoAt.Authority().String())
123
-
if err != nil || ident.Handle.IsInvalidHandle() {
124
-
return fmt.Errorf("failed to resolve handle: %w", err)
125
-
}
126
-
127
-
xrpcc := xrpc.Client{
128
-
Host: ident.PDSEndpoint(),
129
-
}
130
-
131
-
resp, err := comatproto.RepoGetRecord(ctx, &xrpcc, "", tangled.RepoNSID, repoAt.Authority().String(), repoAt.RecordKey().String())
132
-
if err != nil {
133
-
return fmt.Errorf("failed to resolver repo: %w", err)
134
-
}
135
-
136
-
repo := resp.Value.Val.(*tangled.Repo)
137
-
138
-
if repo.Knot != h.c.Server.Hostname {
139
-
return fmt.Errorf("rejected pull record: not this knot, %s != %s", repo.Knot, h.c.Server.Hostname)
140
-
}
141
-
142
-
didSlashRepo, err := securejoin.SecureJoin(ident.DID.String(), repo.Name)
143
-
if err != nil {
144
-
return fmt.Errorf("failed to construct relative repo path: %w", err)
145
-
}
146
-
147
-
repoPath, err := securejoin.SecureJoin(h.c.Repo.ScanPath, didSlashRepo)
148
-
if err != nil {
149
-
return fmt.Errorf("failed to construct absolute repo path: %w", err)
150
-
}
151
-
152
-
gr, err := git.Open(repoPath, record.Source.Sha)
153
-
if err != nil {
154
-
return fmt.Errorf("failed to open git repository: %w", err)
155
-
}
156
-
157
-
workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir)
158
-
if err != nil {
159
-
return fmt.Errorf("failed to open workflow directory: %w", err)
160
-
}
161
-
162
-
var pipeline workflow.RawPipeline
163
-
for _, e := range workflowDir {
164
-
if !e.IsFile() {
165
-
continue
166
-
}
167
-
168
-
fpath := filepath.Join(workflow.WorkflowDir, e.Name)
169
-
contents, err := gr.RawContent(fpath)
170
-
if err != nil {
171
-
continue
172
-
}
173
-
174
-
pipeline = append(pipeline, workflow.RawWorkflow{
175
-
Name: e.Name,
176
-
Contents: contents,
177
-
})
178
-
}
179
-
180
-
trigger := tangled.Pipeline_PullRequestTriggerData{
181
-
Action: "create",
182
-
SourceBranch: record.Source.Branch,
183
-
SourceSha: record.Source.Sha,
184
-
TargetBranch: record.Target.Branch,
185
-
}
186
-
187
-
compiler := workflow.Compiler{
188
-
Trigger: tangled.Pipeline_TriggerMetadata{
189
-
Kind: string(workflow.TriggerKindPullRequest),
190
-
PullRequest: &trigger,
191
-
Repo: &tangled.Pipeline_TriggerRepo{
192
-
Did: ident.DID.String(),
193
-
Knot: repo.Knot,
194
-
Repo: repo.Name,
195
-
},
196
-
},
197
-
}
198
-
199
-
cp := compiler.Compile(compiler.Parse(pipeline))
200
-
eventJson, err := json.Marshal(cp)
201
-
if err != nil {
202
-
return fmt.Errorf("failed to marshal pipeline event: %w", err)
203
-
}
204
-
205
-
// do not run empty pipelines
206
-
if cp.Workflows == nil {
207
-
return nil
208
-
}
209
-
210
-
ev := db.Event{
211
-
Rkey: TID(),
212
-
Nsid: tangled.PipelineNSID,
213
-
EventJson: string(eventJson),
214
-
}
215
-
216
-
return h.db.InsertEvent(ev, h.n)
217
-
}
218
-
219
85
// duplicated from add collaborator
220
86
func (h *Knot) processCollaborator(ctx context.Context, event *models.Event) error {
221
87
raw := json.RawMessage(event.Commit.Record)
···
338
204
err = h.processPublicKey(ctx, event)
339
205
case tangled.KnotMemberNSID:
340
206
err = h.processKnotMember(ctx, event)
341
-
case tangled.RepoPullNSID:
342
-
err = h.processPull(ctx, event)
343
207
case tangled.RepoCollaboratorNSID:
344
208
err = h.processCollaborator(ctx, event)
345
209
}
+1
-109
knotserver/internal.go
+1
-109
knotserver/internal.go
···
23
23
"tangled.org/core/log"
24
24
"tangled.org/core/notifier"
25
25
"tangled.org/core/rbac"
26
-
"tangled.org/core/workflow"
27
26
)
28
27
29
28
type InternalHandle struct {
···
176
175
}
177
176
178
177
for _, line := range lines {
178
+
// TODO: pass pushOptions to refUpdate
179
179
err := h.insertRefUpdate(line, gitUserDid, repoDid, repoName)
180
180
if err != nil {
181
181
l.Error("failed to insert op", "err", err, "line", line, "did", gitUserDid, "repo", gitRelativeDir)
···
185
185
err = h.emitCompareLink(&resp.Messages, line, repoDid, repoName)
186
186
if err != nil {
187
187
l.Error("failed to reply with compare link", "err", err, "line", line, "did", gitUserDid, "repo", gitRelativeDir)
188
-
// non-fatal
189
-
}
190
-
191
-
err = h.triggerPipeline(&resp.Messages, line, gitUserDid, repoDid, repoName, pushOptions)
192
-
if err != nil {
193
-
l.Error("failed to trigger pipeline", "err", err, "line", line, "did", gitUserDid, "repo", gitRelativeDir)
194
188
// non-fatal
195
189
}
196
190
}
···
241
235
}
242
236
243
237
return errors.Join(errs, h.db.InsertEvent(event, h.n))
244
-
}
245
-
246
-
func (h *InternalHandle) triggerPipeline(
247
-
clientMsgs *[]string,
248
-
line git.PostReceiveLine,
249
-
gitUserDid string,
250
-
repoDid string,
251
-
repoName string,
252
-
pushOptions PushOptions,
253
-
) error {
254
-
if pushOptions.skipCi {
255
-
return nil
256
-
}
257
-
258
-
didSlashRepo, err := securejoin.SecureJoin(repoDid, repoName)
259
-
if err != nil {
260
-
return err
261
-
}
262
-
263
-
repoPath, err := securejoin.SecureJoin(h.c.Repo.ScanPath, didSlashRepo)
264
-
if err != nil {
265
-
return err
266
-
}
267
-
268
-
gr, err := git.Open(repoPath, line.Ref)
269
-
if err != nil {
270
-
return err
271
-
}
272
-
273
-
workflowDir, err := gr.FileTree(context.Background(), workflow.WorkflowDir)
274
-
if err != nil {
275
-
return err
276
-
}
277
-
278
-
var pipeline workflow.RawPipeline
279
-
for _, e := range workflowDir {
280
-
if !e.IsFile() {
281
-
continue
282
-
}
283
-
284
-
fpath := filepath.Join(workflow.WorkflowDir, e.Name)
285
-
contents, err := gr.RawContent(fpath)
286
-
if err != nil {
287
-
continue
288
-
}
289
-
290
-
pipeline = append(pipeline, workflow.RawWorkflow{
291
-
Name: e.Name,
292
-
Contents: contents,
293
-
})
294
-
}
295
-
296
-
trigger := tangled.Pipeline_PushTriggerData{
297
-
Ref: line.Ref,
298
-
OldSha: line.OldSha.String(),
299
-
NewSha: line.NewSha.String(),
300
-
}
301
-
302
-
compiler := workflow.Compiler{
303
-
Trigger: tangled.Pipeline_TriggerMetadata{
304
-
Kind: string(workflow.TriggerKindPush),
305
-
Push: &trigger,
306
-
Repo: &tangled.Pipeline_TriggerRepo{
307
-
Did: repoDid,
308
-
Knot: h.c.Server.Hostname,
309
-
Repo: repoName,
310
-
},
311
-
},
312
-
}
313
-
314
-
cp := compiler.Compile(compiler.Parse(pipeline))
315
-
eventJson, err := json.Marshal(cp)
316
-
if err != nil {
317
-
return err
318
-
}
319
-
320
-
for _, e := range compiler.Diagnostics.Errors {
321
-
*clientMsgs = append(*clientMsgs, e.String())
322
-
}
323
-
324
-
if pushOptions.verboseCi {
325
-
if compiler.Diagnostics.IsEmpty() {
326
-
*clientMsgs = append(*clientMsgs, "success: pipeline compiled with no diagnostics")
327
-
}
328
-
329
-
for _, w := range compiler.Diagnostics.Warnings {
330
-
*clientMsgs = append(*clientMsgs, w.String())
331
-
}
332
-
}
333
-
334
-
// do not run empty pipelines
335
-
if cp.Workflows == nil {
336
-
return nil
337
-
}
338
-
339
-
event := db.Event{
340
-
Rkey: TID(),
341
-
Nsid: tangled.PipelineNSID,
342
-
EventJson: string(eventJson),
343
-
}
344
-
345
-
return h.db.InsertEvent(event, h.n)
346
238
}
347
239
348
240
func (h *InternalHandle) emitCompareLink(
-1
knotserver/server.go
-1
knotserver/server.go
···
79
79
jc, err := jetstream.NewJetstreamClient(c.Server.JetstreamEndpoint, "knotserver", []string{
80
80
tangled.PublicKeyNSID,
81
81
tangled.KnotMemberNSID,
82
-
tangled.RepoPullNSID,
83
82
tangled.RepoCollaboratorNSID,
84
83
}, nil, log.SubLogger(logger, "jetstream"), db, true, c.Server.LogDids)
85
84
if err != nil {
+3
nix/gomod2nix.toml
+3
nix/gomod2nix.toml
···
304
304
[mod."github.com/hashicorp/go-sockaddr"]
305
305
version = "v1.0.7"
306
306
hash = "sha256-p6eDOrGzN1jMmT/F/f/VJMq0cKNFhUcEuVVwTE6vSrs="
307
+
[mod."github.com/hashicorp/go-version"]
308
+
version = "v1.8.0"
309
+
hash = "sha256-KXtqERmYrWdpqPCViWcHbe6jnuH7k16bvBIcuJuevj8="
307
310
[mod."github.com/hashicorp/golang-lru"]
308
311
version = "v1.0.2"
309
312
hash = "sha256-yy+5botc6T5wXgOe2mfNXJP3wr+MkVlUZ2JBkmmrA48="
+64
nix/modules/bluesky-jetstream.nix
+64
nix/modules/bluesky-jetstream.nix
···
1
+
{
2
+
config,
3
+
pkgs,
4
+
lib,
5
+
...
6
+
}: let
7
+
cfg = config.services.bluesky-jetstream;
8
+
in
9
+
with lib; {
10
+
options.services.bluesky-jetstream = {
11
+
enable = mkEnableOption "jetstream server";
12
+
package = mkPackageOption pkgs "bluesky-jetstream" {};
13
+
14
+
# dataDir = mkOption {
15
+
# type = types.str;
16
+
# default = "/var/lib/jetstream";
17
+
# description = "directory to store data (pebbleDB)";
18
+
# };
19
+
livenessTtl = mkOption {
20
+
type = types.int;
21
+
default = 15;
22
+
description = "time to restart when no event detected (seconds)";
23
+
};
24
+
websocketUrl = mkOption {
25
+
type = types.str;
26
+
default = "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos";
27
+
description = "full websocket path to the ATProto SubscribeRepos XRPC endpoint";
28
+
};
29
+
};
30
+
config = mkIf cfg.enable {
31
+
systemd.services.bluesky-jetstream = {
32
+
description = "bluesky jetstream";
33
+
after = ["network.target" "pds.service"];
34
+
wantedBy = ["multi-user.target"];
35
+
36
+
serviceConfig = {
37
+
User = "jetstream";
38
+
Group = "jetstream";
39
+
StateDirectory = "jetstream";
40
+
StateDirectoryMode = "0755";
41
+
# preStart = ''
42
+
# mkdir -p "${cfg.dataDir}"
43
+
# chown -R jetstream:jetstream "${cfg.dataDir}"
44
+
# '';
45
+
# WorkingDirectory = cfg.dataDir;
46
+
Environment = [
47
+
"JETSTREAM_DATA_DIR=/var/lib/jetstream/data"
48
+
"JETSTREAM_LIVENESS_TTL=${toString cfg.livenessTtl}s"
49
+
"JETSTREAM_WS_URL=${cfg.websocketUrl}"
50
+
];
51
+
ExecStart = getExe cfg.package;
52
+
Restart = "always";
53
+
RestartSec = 5;
54
+
};
55
+
};
56
+
users = {
57
+
users.jetstream = {
58
+
group = "jetstream";
59
+
isSystemUser = true;
60
+
};
61
+
groups.jetstream = {};
62
+
};
63
+
};
64
+
}
+48
nix/modules/bluesky-relay.nix
+48
nix/modules/bluesky-relay.nix
···
1
+
{
2
+
config,
3
+
pkgs,
4
+
lib,
5
+
...
6
+
}: let
7
+
cfg = config.services.bluesky-relay;
8
+
in
9
+
with lib; {
10
+
options.services.bluesky-relay = {
11
+
enable = mkEnableOption "relay server";
12
+
package = mkPackageOption pkgs "bluesky-relay" {};
13
+
};
14
+
config = mkIf cfg.enable {
15
+
systemd.services.bluesky-relay = {
16
+
description = "bluesky relay";
17
+
after = ["network.target" "pds.service"];
18
+
wantedBy = ["multi-user.target"];
19
+
20
+
serviceConfig = {
21
+
User = "relay";
22
+
Group = "relay";
23
+
StateDirectory = "relay";
24
+
StateDirectoryMode = "0755";
25
+
Environment = [
26
+
"RELAY_ADMIN_PASSWORD=password"
27
+
"RELAY_PLC_HOST=https://plc.tngl.boltless.dev"
28
+
"DATABASE_URL=sqlite:///var/lib/relay/relay.sqlite"
29
+
"RELAY_IP_BIND=:2470"
30
+
"RELAY_PERSIST_DIR=/var/lib/relay"
31
+
"RELAY_DISABLE_REQUEST_CRAWL=0"
32
+
"RELAY_INITIAL_SEQ_NUMBER=1"
33
+
"RELAY_ALLOW_INSECURE_HOSTS=1"
34
+
];
35
+
ExecStart = "${getExe cfg.package} serve";
36
+
Restart = "always";
37
+
RestartSec = 5;
38
+
};
39
+
};
40
+
users = {
41
+
users.relay = {
42
+
group = "relay";
43
+
isSystemUser = true;
44
+
};
45
+
groups.relay = {};
46
+
};
47
+
};
48
+
}
+76
nix/modules/did-method-plc.nix
+76
nix/modules/did-method-plc.nix
···
1
+
{
2
+
config,
3
+
pkgs,
4
+
lib,
5
+
...
6
+
}: let
7
+
cfg = config.services.did-method-plc;
8
+
in
9
+
with lib; {
10
+
options.services.did-method-plc = {
11
+
enable = mkEnableOption "did-method-plc server";
12
+
package = mkPackageOption pkgs "did-method-plc" {};
13
+
};
14
+
config = mkIf cfg.enable {
15
+
services.postgresql = {
16
+
enable = true;
17
+
package = pkgs.postgresql_14;
18
+
ensureDatabases = ["plc"];
19
+
ensureUsers = [
20
+
{
21
+
name = "pg";
22
+
# ensurePermissions."DATABASE plc" = "ALL PRIVILEGES";
23
+
}
24
+
];
25
+
authentication = ''
26
+
local all all trust
27
+
host all all 127.0.0.1/32 trust
28
+
'';
29
+
};
30
+
systemd.services.did-method-plc = {
31
+
description = "did-method-plc";
32
+
33
+
after = ["postgresql.service"];
34
+
wants = ["postgresql.service"];
35
+
wantedBy = ["multi-user.target"];
36
+
37
+
environment = let
38
+
db_creds_json = builtins.toJSON {
39
+
username = "pg";
40
+
password = "";
41
+
host = "127.0.0.1";
42
+
port = 5432;
43
+
};
44
+
in {
45
+
# TODO: inherit from config
46
+
DEBUG_MODE = "1";
47
+
LOG_ENABLED = "true";
48
+
LOG_LEVEL = "debug";
49
+
LOG_DESTINATION = "1";
50
+
ENABLE_MIGRATIONS = "true";
51
+
DB_CREDS_JSON = db_creds_json;
52
+
DB_MIGRATE_CREDS_JSON = db_creds_json;
53
+
PLC_VERSION = "0.0.1";
54
+
PORT = "8080";
55
+
};
56
+
57
+
serviceConfig = {
58
+
ExecStart = getExe cfg.package;
59
+
User = "plc";
60
+
Group = "plc";
61
+
StateDirectory = "plc";
62
+
StateDirectoryMode = "0755";
63
+
Restart = "always";
64
+
65
+
# Hardening
66
+
};
67
+
};
68
+
users = {
69
+
users.plc = {
70
+
group = "plc";
71
+
isSystemUser = true;
72
+
};
73
+
groups.plc = {};
74
+
};
75
+
};
76
+
}
+46
-12
nix/modules/spindle.nix
+46
-12
nix/modules/spindle.nix
···
1
1
{
2
2
config,
3
+
pkgs,
3
4
lib,
4
5
...
5
6
}: let
···
17
18
type = types.package;
18
19
description = "Package to use for the spindle";
19
20
};
21
+
tap-package = mkOption {
22
+
type = types.package;
23
+
description = "Package to use for the spindle";
24
+
};
25
+
26
+
atpRelayUrl = mkOption {
27
+
type = types.str;
28
+
default = "https://relay1.us-east.bsky.network";
29
+
description = "atproto relay";
30
+
};
20
31
21
32
server = {
22
33
listenAddr = mkOption {
···
25
36
description = "Address to listen on";
26
37
};
27
38
28
-
dbPath = mkOption {
39
+
stateDir = mkOption {
29
40
type = types.path;
30
-
default = "/var/lib/spindle/spindle.db";
31
-
description = "Path to the database file";
41
+
default = "/var/lib/spindle";
42
+
description = "Tangled spindle data directory";
32
43
};
33
44
34
45
hostname = mkOption {
···
41
52
type = types.str;
42
53
default = "https://plc.directory";
43
54
description = "atproto PLC directory";
44
-
};
45
-
46
-
jetstreamEndpoint = mkOption {
47
-
type = types.str;
48
-
default = "wss://jetstream1.us-west.bsky.network/subscribe";
49
-
description = "Jetstream endpoint to subscribe to";
50
55
};
51
56
52
57
dev = mkOption {
···
114
119
config = mkIf cfg.enable {
115
120
virtualisation.docker.enable = true;
116
121
122
+
systemd.services.spindle-tap = {
123
+
description = "spindle tap service";
124
+
after = ["network.target" "docker.service"];
125
+
wantedBy = ["multi-user.target"];
126
+
serviceConfig = {
127
+
LogsDirectory = "spindle-tap";
128
+
StateDirectory = "spindle-tap";
129
+
Environment = [
130
+
"TAP_BIND=:2480"
131
+
"TAP_PLC_URL=${cfg.server.plcUrl}"
132
+
"TAP_RELAY_URL=${cfg.atpRelayUrl}"
133
+
"TAP_DATABASE_URL=sqlite:///var/lib/spindle-tap/tap.db"
134
+
"TAP_RETRY_TIMEOUT=3s"
135
+
"TAP_COLLECTION_FILTERS=${concatStringsSep "," [
136
+
"sh.tangled.repo"
137
+
"sh.tangled.repo.collaborator"
138
+
"sh.tangled.spindle.member"
139
+
"sh.tangled.repo.pull"
140
+
]}"
141
+
# temporary hack to listen for repo.pull from non-tangled users
142
+
"TAP_SIGNAL_COLLECTION=sh.tangled.repo.pull"
143
+
];
144
+
ExecStart = "${getExe cfg.tap-package} run";
145
+
};
146
+
};
147
+
117
148
systemd.services.spindle = {
118
149
description = "spindle service";
119
-
after = ["network.target" "docker.service"];
150
+
after = ["network.target" "docker.service" "spindle-tap.service"];
120
151
wantedBy = ["multi-user.target"];
152
+
path = [
153
+
pkgs.git
154
+
];
121
155
serviceConfig = {
122
156
LogsDirectory = "spindle";
123
157
StateDirectory = "spindle";
124
158
Environment = [
125
159
"SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}"
126
-
"SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}"
160
+
"SPINDLE_SERVER_DATA_DIR=${cfg.server.stateDir}"
127
161
"SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}"
128
162
"SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}"
129
-
"SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}"
130
163
"SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}"
131
164
"SPINDLE_SERVER_OWNER=${cfg.server.owner}"
132
165
"SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}"
···
134
167
"SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}"
135
168
"SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}"
136
169
"SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}"
170
+
"SPINDLE_SERVER_TAP_URL=http://localhost:2480"
137
171
"SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}"
138
172
"SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}"
139
173
];
+20
nix/pkgs/bluesky-jetstream.nix
+20
nix/pkgs/bluesky-jetstream.nix
···
1
+
{
2
+
buildGoModule,
3
+
fetchFromGitHub,
4
+
}:
5
+
buildGoModule {
6
+
pname = "bluesky-jetstream";
7
+
version = "0.1.0";
8
+
src = fetchFromGitHub {
9
+
owner = "bluesky-social";
10
+
repo = "jetstream";
11
+
rev = "7d7efa58d7f14101a80ccc4f1085953948b7d5de";
12
+
sha256 = "sha256-1e9SL/8gaDPMA4YZed51ffzgpkptbMd0VTbTTDbPTFw=";
13
+
};
14
+
subPackages = ["cmd/jetstream"];
15
+
vendorHash = "sha256-/21XJQH6fo9uPzlABUAbdBwt1O90odmppH6gXu2wkiQ=";
16
+
doCheck = false;
17
+
meta = {
18
+
mainProgram = "jetstream";
19
+
};
20
+
}
+20
nix/pkgs/bluesky-relay.nix
+20
nix/pkgs/bluesky-relay.nix
···
1
+
{
2
+
buildGoModule,
3
+
fetchFromGitHub,
4
+
}:
5
+
buildGoModule {
6
+
pname = "bluesky-relay";
7
+
version = "0.1.0";
8
+
src = fetchFromGitHub {
9
+
owner = "boltlessengineer";
10
+
repo = "indigo";
11
+
rev = "7fe70a304d795b998f354d2b7b2050b909709c99";
12
+
sha256 = "sha256-+h34x67cqH5t30+8rua53/ucvbn3BanrmH0Og3moHok=";
13
+
};
14
+
subPackages = ["cmd/relay"];
15
+
vendorHash = "sha256-UOedwNYnM8Jx6B7Y9tFcZX8IeUBESAFAPTRYk7n0yo8=";
16
+
doCheck = false;
17
+
meta = {
18
+
mainProgram = "relay";
19
+
};
20
+
}
+65
nix/pkgs/did-method-plc.nix
+65
nix/pkgs/did-method-plc.nix
···
1
+
# inspired by https://github.com/NixOS/nixpkgs/blob/333bfb7c258fab089a834555ea1c435674c459b4/pkgs/by-name/ga/gatsby-cli/package.nix
2
+
{
3
+
lib,
4
+
stdenv,
5
+
fetchFromGitHub,
6
+
fetchYarnDeps,
7
+
yarnConfigHook,
8
+
yarnBuildHook,
9
+
nodejs,
10
+
makeBinaryWrapper,
11
+
}:
12
+
stdenv.mkDerivation (finalAttrs: {
13
+
pname = "did-method-plc";
14
+
version = "0.0.1";
15
+
16
+
src = fetchFromGitHub {
17
+
owner = "did-method-plc";
18
+
repo = "did-method-plc";
19
+
rev = "158ba5535ac3da4fd4309954bde41deab0b45972";
20
+
sha256 = "sha256-O5smubbrnTDMCvL6iRyMXkddr5G7YHxkQRVMRULHanQ=";
21
+
};
22
+
postPatch = ''
23
+
# remove dd-trace dependency
24
+
sed -i '3d' packages/server/service/index.js
25
+
'';
26
+
27
+
yarnOfflineCache = fetchYarnDeps {
28
+
yarnLock = finalAttrs.src + "/yarn.lock";
29
+
hash = "sha256-g8GzaAbWSnWwbQjJMV2DL5/ZlWCCX0sRkjjvX3tqU4Y=";
30
+
};
31
+
32
+
nativeBuildInputs = [
33
+
yarnConfigHook
34
+
yarnBuildHook
35
+
nodejs
36
+
makeBinaryWrapper
37
+
];
38
+
yarnBuildScript = "lerna";
39
+
yarnBuildFlags = [
40
+
"run"
41
+
"build"
42
+
"--scope"
43
+
"@did-plc/server"
44
+
"--include-dependencies"
45
+
];
46
+
47
+
installPhase = ''
48
+
runHook preInstall
49
+
50
+
mkdir -p $out/lib/node_modules/
51
+
mv packages/ $out/lib/packages/
52
+
mv node_modules/* $out/lib/node_modules/
53
+
54
+
makeWrapper ${lib.getExe nodejs} $out/bin/plc \
55
+
--add-flags $out/lib/packages/server/service/index.js \
56
+
--add-flags --enable-source-maps \
57
+
--set NODE_PATH $out/lib/node_modules
58
+
59
+
runHook postInstall
60
+
'';
61
+
62
+
meta = {
63
+
mainProgram = "plc";
64
+
};
65
+
})
+20
nix/pkgs/tap.nix
+20
nix/pkgs/tap.nix
···
1
+
{
2
+
buildGoModule,
3
+
fetchFromGitHub,
4
+
}:
5
+
buildGoModule {
6
+
pname = "tap";
7
+
version = "0.1.0";
8
+
src = fetchFromGitHub {
9
+
owner = "bluesky-social";
10
+
repo = "indigo";
11
+
rev = "498ecb9693e8ae050f73234c86f340f51ad896a9";
12
+
sha256 = "sha256-KASCdwkg/hlKBt7RTW3e3R5J3hqJkphoarFbaMgtN1k=";
13
+
};
14
+
subPackages = ["cmd/tap"];
15
+
vendorHash = "sha256-UOedwNYnM8Jx6B7Y9tFcZX8IeUBESAFAPTRYk7n0yo8=";
16
+
doCheck = false;
17
+
meta = {
18
+
mainProgram = "tap";
19
+
};
20
+
}
+130
-2
nix/vm.nix
+130
-2
nix/vm.nix
···
19
19
20
20
plcUrl = envVarOr "TANGLED_VM_PLC_URL" "https://plc.directory";
21
21
jetstream = envVarOr "TANGLED_VM_JETSTREAM_ENDPOINT" "wss://jetstream1.us-west.bsky.network/subscribe";
22
+
relayUrl = envVarOr "TANGLED_VM_RELAY_URL" "https://relay1.us-east.bsky.network";
22
23
in
23
24
nixpkgs.lib.nixosSystem {
24
25
inherit system;
25
26
modules = [
27
+
self.nixosModules.did-method-plc
28
+
self.nixosModules.bluesky-jetstream
29
+
self.nixosModules.bluesky-relay
26
30
self.nixosModules.knot
27
31
self.nixosModules.spindle
28
32
({
···
39
43
diskSize = 10 * 1024;
40
44
cores = 2;
41
45
forwardPorts = [
46
+
# caddy
47
+
{
48
+
from = "host";
49
+
host.port = 80;
50
+
guest.port = 80;
51
+
}
52
+
{
53
+
from = "host";
54
+
host.port = 443;
55
+
guest.port = 443;
56
+
}
57
+
{
58
+
from = "host";
59
+
proto = "udp";
60
+
host.port = 443;
61
+
guest.port = 443;
62
+
}
42
63
# ssh
43
64
{
44
65
from = "host";
···
57
78
host.port = 6555;
58
79
guest.port = 6555;
59
80
}
81
+
{
82
+
from = "host";
83
+
host.port = 6556;
84
+
guest.port = 2480;
85
+
}
60
86
];
61
87
sharedDirectories = {
62
88
# We can't use the 9p mounts directly for most of these
63
89
# as SQLite is incompatible with them. So instead we
64
90
# mount the shared directories to a different location
65
91
# and copy the contents around on service start/stop.
92
+
caddyData = {
93
+
source = "$TANGLED_VM_DATA_DIR/caddy";
94
+
target = config.services.caddy.dataDir;
95
+
};
66
96
knotData = {
67
97
source = "$TANGLED_VM_DATA_DIR/knot";
68
98
target = "/mnt/knot-data";
···
79
109
};
80
110
# This is fine because any and all ports that are forwarded to host are explicitly marked above, we don't need a separate guest firewall
81
111
networking.firewall.enable = false;
112
+
# resolve `*.tngl.boltless.dev` to host
113
+
services.dnsmasq.enable = true;
114
+
services.dnsmasq.settings.address = "/tngl.boltless.dev/10.0.2.2";
115
+
security.pki.certificates = [
116
+
(builtins.readFile ../contrib/certs/root.crt)
117
+
];
82
118
time.timeZone = "Europe/London";
119
+
services.timesyncd.enable = lib.mkVMOverride true;
83
120
services.getty.autologinUser = "root";
84
121
environment.systemPackages = with pkgs; [curl vim git sqlite litecli];
122
+
virtualisation.docker.extraOptions = ''
123
+
--dns 172.17.0.1
124
+
'';
85
125
services.tangled.knot = {
86
126
enable = true;
87
127
motd = "Welcome to the development knot!\n";
···
95
135
};
96
136
services.tangled.spindle = {
97
137
enable = true;
138
+
atpRelayUrl = relayUrl;
98
139
server = {
99
140
owner = envVar "TANGLED_VM_SPINDLE_OWNER";
100
141
hostname = envVarOr "TANGLED_VM_SPINDLE_HOST" "localhost:6555";
101
142
plcUrl = plcUrl;
102
-
jetstreamEndpoint = jetstream;
103
143
listenAddr = "0.0.0.0:6555";
104
144
dev = true;
105
145
queueSize = 100;
···
109
149
};
110
150
};
111
151
};
152
+
services.did-method-plc.enable = true;
153
+
services.bluesky-pds = {
154
+
enable = true;
155
+
# overriding package version to support emails
156
+
package = pkgs.bluesky-pds.overrideAttrs (old: rec {
157
+
version = "0.4.188";
158
+
src = pkgs.fetchFromGitHub {
159
+
owner = "bluesky-social";
160
+
repo = "pds";
161
+
tag = "v${version}";
162
+
hash = "sha256-t8KdyEygXdbj/5Rhj8W40e1o8mXprELpjsKddHExmo0=";
163
+
};
164
+
pnpmDeps = pkgs.fetchPnpmDeps {
165
+
inherit version src;
166
+
pname = old.pname;
167
+
sourceRoot = old.sourceRoot;
168
+
fetcherVersion = 2;
169
+
hash = "sha256-lQie7f8JbWKSpoavnMjHegBzH3GB9teXsn+S2SLJHHU=";
170
+
};
171
+
});
172
+
settings = {
173
+
LOG_ENABLED = "true";
174
+
175
+
PDS_JWT_SECRET = "8cae8bffcc73d9932819650791e4e89a";
176
+
PDS_ADMIN_PASSWORD = "d6a902588cd93bee1af83f924f60cfd3";
177
+
PDS_PLC_ROTATION_KEY_K256_PRIVATE_KEY_HEX = "2e92e336a50a618458e1097d94a1db86ec3fd8829d7735020cbae80625c761d7";
178
+
179
+
PDS_EMAIL_SMTP_URL = envVarOr "TANGLED_VM_PDS_EMAIL_SMTP_URL" null;
180
+
PDS_EMAIL_FROM_ADDRESS = envVarOr "TANGLED_VM_PDS_EMAIL_FROM_ADDRESS" null;
181
+
182
+
PDS_DID_PLC_URL = "http://localhost:8080";
183
+
PDS_CRAWLERS = "https://relay.tngl.boltless.dev";
184
+
PDS_HOSTNAME = "pds.tngl.boltless.dev";
185
+
PDS_PORT = 3000;
186
+
};
187
+
};
188
+
services.bluesky-relay = {
189
+
enable = true;
190
+
};
191
+
services.bluesky-jetstream = {
192
+
enable = true;
193
+
livenessTtl = 300;
194
+
websocketUrl = "ws://localhost:3000/xrpc/com.atproto.sync.subscribeRepos";
195
+
};
196
+
services.caddy = {
197
+
enable = true;
198
+
configFile = pkgs.writeText "Caddyfile" ''
199
+
{
200
+
debug
201
+
cert_lifetime 3601d
202
+
pki {
203
+
ca local {
204
+
intermediate_lifetime 3599d
205
+
}
206
+
}
207
+
}
208
+
209
+
plc.tngl.boltless.dev {
210
+
tls internal
211
+
reverse_proxy http://localhost:8080
212
+
}
213
+
214
+
*.pds.tngl.boltless.dev, pds.tngl.boltless.dev {
215
+
tls internal
216
+
reverse_proxy http://localhost:3000
217
+
}
218
+
219
+
jetstream.tngl.boltless.dev {
220
+
tls internal
221
+
reverse_proxy http://localhost:6008
222
+
}
223
+
224
+
relay.tngl.boltless.dev {
225
+
tls internal
226
+
reverse_proxy http://localhost:2470
227
+
}
228
+
229
+
knot.tngl.boltless.dev {
230
+
tls internal
231
+
reverse_proxy http://localhost:6444
232
+
}
233
+
234
+
spindle.tngl.boltless.dev {
235
+
tls internal
236
+
reverse_proxy http://localhost:6555
237
+
}
238
+
'';
239
+
};
112
240
users = {
113
241
# So we don't have to deal with permission clashing between
114
242
# blank disk VMs and existing state
···
134
262
};
135
263
in {
136
264
knot = mkDataSyncScripts "/mnt/knot-data" config.services.tangled.knot.stateDir;
137
-
spindle = mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled.spindle.server.dbPath);
265
+
spindle = mkDataSyncScripts "/mnt/spindle-data" config.services.tangled.spindle.server.stateDir;
138
266
};
139
267
})
140
268
];
+10
orm/orm.go
+10
orm/orm.go
···
20
20
}
21
21
defer tx.Rollback()
22
22
23
+
_, err = tx.Exec(`
24
+
create table if not exists migrations (
25
+
id integer primary key autoincrement,
26
+
name text unique
27
+
);
28
+
`)
29
+
if err != nil {
30
+
return fmt.Errorf("creating migrations table: %w", err)
31
+
}
32
+
23
33
var exists bool
24
34
err = tx.QueryRow("select exists (select 1 from migrations where name = ?)", name).Scan(&exists)
25
35
if err != nil {
+52
rbac2/bytesadapter/adapter.go
+52
rbac2/bytesadapter/adapter.go
···
1
+
package bytesadapter
2
+
3
+
import (
4
+
"bufio"
5
+
"bytes"
6
+
"errors"
7
+
"strings"
8
+
9
+
"github.com/casbin/casbin/v2/model"
10
+
"github.com/casbin/casbin/v2/persist"
11
+
)
12
+
13
+
var (
14
+
errNotImplemented = errors.New("not implemented")
15
+
)
16
+
17
+
type Adapter struct {
18
+
b []byte
19
+
}
20
+
21
+
var _ persist.Adapter = &Adapter{}
22
+
23
+
func NewAdapter(b []byte) *Adapter {
24
+
return &Adapter{b}
25
+
}
26
+
27
+
func (a *Adapter) LoadPolicy(model model.Model) error {
28
+
scanner := bufio.NewScanner(bytes.NewReader(a.b))
29
+
for scanner.Scan() {
30
+
line := strings.TrimSpace(scanner.Text())
31
+
if err := persist.LoadPolicyLine(line, model); err != nil {
32
+
return err
33
+
}
34
+
}
35
+
return scanner.Err()
36
+
}
37
+
38
+
func (a *Adapter) AddPolicy(sec string, ptype string, rule []string) error {
39
+
return errNotImplemented
40
+
}
41
+
42
+
func (a *Adapter) RemoveFilteredPolicy(sec string, ptype string, fieldIndex int, fieldValues ...string) error {
43
+
return errNotImplemented
44
+
}
45
+
46
+
func (a *Adapter) RemovePolicy(sec string, ptype string, rule []string) error {
47
+
return errNotImplemented
48
+
}
49
+
50
+
func (a *Adapter) SavePolicy(model model.Model) error {
51
+
return errNotImplemented
52
+
}
+139
rbac2/rbac2.go
+139
rbac2/rbac2.go
···
1
+
package rbac2
2
+
3
+
import (
4
+
"database/sql"
5
+
_ "embed"
6
+
"fmt"
7
+
8
+
adapter "github.com/Blank-Xu/sql-adapter"
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
"github.com/casbin/casbin/v2"
11
+
"github.com/casbin/casbin/v2/model"
12
+
"github.com/casbin/casbin/v2/util"
13
+
"tangled.org/core/rbac2/bytesadapter"
14
+
)
15
+
16
+
const (
17
+
Model = `
18
+
[request_definition]
19
+
r = sub, dom, obj, act
20
+
21
+
[policy_definition]
22
+
p = sub, dom, obj, act
23
+
24
+
[role_definition]
25
+
g = _, _, _
26
+
27
+
[policy_effect]
28
+
e = some(where (p.eft == allow))
29
+
30
+
[matchers]
31
+
m = g(r.sub, p.sub, r.dom) && keyMatch4(r.dom, p.dom) && r.obj == p.obj && r.act == p.act
32
+
`
33
+
)
34
+
35
+
type Enforcer struct {
36
+
e *casbin.Enforcer
37
+
}
38
+
39
+
//go:embed tangled_policy.csv
40
+
var tangledPolicy []byte
41
+
42
+
func NewEnforcer(path string) (*Enforcer, error) {
43
+
db, err := sql.Open("sqlite3", path+"?_foreign_keys=1")
44
+
if err != nil {
45
+
return nil, err
46
+
}
47
+
return NewEnforcerWithDB(db)
48
+
}
49
+
50
+
func NewEnforcerWithDB(db *sql.DB) (*Enforcer, error) {
51
+
m, err := model.NewModelFromString(Model)
52
+
if err != nil {
53
+
return nil, err
54
+
}
55
+
56
+
a, err := adapter.NewAdapter(db, "sqlite3", "acl")
57
+
if err != nil {
58
+
return nil, err
59
+
}
60
+
61
+
// // PATCH: create unique index to make `AddPoliciesEx` work
62
+
// _, err = db.Exec(fmt.Sprintf(
63
+
// `create unique index if not exists uq_%[1]s on %[1]s (p_type,v0,v1,v2,v3,v4,v5);`,
64
+
// tableName,
65
+
// ))
66
+
// if err != nil {
67
+
// return nil, err
68
+
// }
69
+
70
+
e, _ := casbin.NewEnforcer() // NewEnforcer() without param won't return error
71
+
// e.EnableLog(true)
72
+
73
+
// NOTE: casbin clears the model on init, so we should intialize with temporary adapter first
74
+
// and then override the adapter to sql-adapter.
75
+
// `e.SetModel(m)` after init doesn't work for some reason
76
+
if err := e.InitWithModelAndAdapter(m, bytesadapter.NewAdapter(tangledPolicy)); err != nil {
77
+
return nil, err
78
+
}
79
+
80
+
// load dynamic policy from db
81
+
e.EnableAutoSave(false)
82
+
if err := a.LoadPolicy(e.GetModel()); err != nil {
83
+
return nil, err
84
+
}
85
+
e.AddNamedDomainMatchingFunc("g", "keyMatch4", util.KeyMatch4)
86
+
e.BuildRoleLinks()
87
+
e.SetAdapter(a)
88
+
e.EnableAutoSave(true)
89
+
90
+
return &Enforcer{e}, nil
91
+
}
92
+
93
+
// CaptureModel returns copy of current model. Used for testing
94
+
func (e *Enforcer) CaptureModel() model.Model {
95
+
return e.e.GetModel().Copy()
96
+
}
97
+
98
+
func (e *Enforcer) hasImplicitRoleForUser(name string, role string, domain ...string) (bool, error) {
99
+
roles, err := e.e.GetImplicitRolesForUser(name, domain...)
100
+
if err != nil {
101
+
return false, err
102
+
}
103
+
for _, r := range roles {
104
+
if r == role {
105
+
return true, nil
106
+
}
107
+
}
108
+
return false, nil
109
+
}
110
+
111
+
// setRoleForUser sets single user role for specified domain.
112
+
// All existing users with that role will be removed.
113
+
func (e *Enforcer) setRoleForUser(name string, role string, domain ...string) error {
114
+
currentUsers, err := e.e.GetUsersForRole(role, domain...)
115
+
if err != nil {
116
+
return err
117
+
}
118
+
119
+
for _, oldUser := range currentUsers {
120
+
_, err = e.e.DeleteRoleForUser(oldUser, role, domain...)
121
+
if err != nil {
122
+
return err
123
+
}
124
+
}
125
+
126
+
_, err = e.e.AddRoleForUser(name, role, domain...)
127
+
return err
128
+
}
129
+
130
+
// validateAtUri enforeces AT-URI to have valid did as authority and match collection NSID.
131
+
func validateAtUri(uri syntax.ATURI, expected string) error {
132
+
if !uri.Authority().IsDID() {
133
+
return fmt.Errorf("expected at-uri with did")
134
+
}
135
+
if expected != "" && uri.Collection().String() != expected {
136
+
return fmt.Errorf("incorrect repo at-uri collection nsid '%s' (expected '%s')", uri.Collection(), expected)
137
+
}
138
+
return nil
139
+
}
+150
rbac2/rbac2_test.go
+150
rbac2/rbac2_test.go
···
1
+
package rbac2_test
2
+
3
+
import (
4
+
"database/sql"
5
+
"testing"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
_ "github.com/mattn/go-sqlite3"
9
+
"github.com/stretchr/testify/assert"
10
+
"tangled.org/core/rbac2"
11
+
)
12
+
13
+
func setup(t *testing.T) *rbac2.Enforcer {
14
+
enforcer, err := rbac2.NewEnforcer(":memory:")
15
+
assert.NoError(t, err)
16
+
17
+
return enforcer
18
+
}
19
+
20
+
func TestNewEnforcer(t *testing.T) {
21
+
db, err := sql.Open("sqlite3", ":memory:?_foreign_keys=1")
22
+
assert.NoError(t, err)
23
+
24
+
enforcer1, err := rbac2.NewEnforcerWithDB(db)
25
+
assert.NoError(t, err)
26
+
enforcer1.AddRepo(syntax.ATURI("at://did:plc:foo/sh.tangled.repo/reporkey"))
27
+
model1 := enforcer1.CaptureModel()
28
+
29
+
enforcer2, err := rbac2.NewEnforcerWithDB(db)
30
+
assert.NoError(t, err)
31
+
model2 := enforcer2.CaptureModel()
32
+
33
+
// model1.GetLogger().EnableLog(true)
34
+
// model1.PrintModel()
35
+
// model1.PrintPolicy()
36
+
// model1.GetLogger().EnableLog(false)
37
+
38
+
model2.GetLogger().EnableLog(true)
39
+
model2.PrintModel()
40
+
model2.PrintPolicy()
41
+
model2.GetLogger().EnableLog(false)
42
+
43
+
assert.Equal(t, model1, model2)
44
+
}
45
+
46
+
func TestRepoOwnerPermissions(t *testing.T) {
47
+
var (
48
+
e = setup(t)
49
+
ok bool
50
+
err error
51
+
fooRepo = syntax.ATURI("at://did:plc:foo/sh.tangled.repo/reporkey")
52
+
fooUser = syntax.DID("did:plc:foo")
53
+
)
54
+
55
+
assert.NoError(t, e.AddRepo(fooRepo))
56
+
57
+
ok, err = e.IsRepoOwner(fooUser, fooRepo)
58
+
assert.NoError(t, err)
59
+
assert.True(t, ok, "repo author should be repo owner")
60
+
61
+
ok, err = e.IsRepoWriteAllowed(fooUser, fooRepo)
62
+
assert.NoError(t, err)
63
+
assert.True(t, ok, "repo owner should be able to modify the repo itself")
64
+
65
+
ok, err = e.IsRepoCollaborator(fooUser, fooRepo)
66
+
assert.NoError(t, err)
67
+
assert.True(t, ok, "repo owner should inherit role role:collaborator")
68
+
69
+
ok, err = e.IsRepoSettingsWriteAllowed(fooUser, fooRepo)
70
+
assert.NoError(t, err)
71
+
assert.True(t, ok, "repo owner should inherit collaborator permissions")
72
+
}
73
+
74
+
func TestRepoCollaboratorPermissions(t *testing.T) {
75
+
var (
76
+
e = setup(t)
77
+
ok bool
78
+
err error
79
+
fooRepo = syntax.ATURI("at://did:plc:foo/sh.tangled.repo/reporkey")
80
+
barUser = syntax.DID("did:plc:bar")
81
+
)
82
+
83
+
assert.NoError(t, e.AddRepo(fooRepo))
84
+
assert.NoError(t, e.AddRepoCollaborator(barUser, fooRepo))
85
+
86
+
ok, err = e.IsRepoCollaborator(barUser, fooRepo)
87
+
assert.NoError(t, err)
88
+
assert.True(t, ok, "should set repo collaborator")
89
+
90
+
ok, err = e.IsRepoSettingsWriteAllowed(barUser, fooRepo)
91
+
assert.NoError(t, err)
92
+
assert.True(t, ok, "repo collaborator should be able to edit repo settings")
93
+
94
+
ok, err = e.IsRepoWriteAllowed(barUser, fooRepo)
95
+
assert.NoError(t, err)
96
+
assert.False(t, ok, "repo collaborator shouldn't be able to modify the repo itself")
97
+
}
98
+
99
+
func TestGetByRole(t *testing.T) {
100
+
var (
101
+
e = setup(t)
102
+
err error
103
+
fooRepo = syntax.ATURI("at://did:plc:foo/sh.tangled.repo/reporkey")
104
+
owner = syntax.DID("did:plc:foo")
105
+
collaborator1 = syntax.DID("did:plc:bar")
106
+
collaborator2 = syntax.DID("did:plc:baz")
107
+
)
108
+
109
+
assert.NoError(t, e.AddRepo(fooRepo))
110
+
assert.NoError(t, e.AddRepoCollaborator(collaborator1, fooRepo))
111
+
assert.NoError(t, e.AddRepoCollaborator(collaborator2, fooRepo))
112
+
113
+
collaborators, err := e.GetRepoCollaborators(fooRepo)
114
+
assert.NoError(t, err)
115
+
assert.ElementsMatch(t, []syntax.DID{
116
+
owner,
117
+
collaborator1,
118
+
collaborator2,
119
+
}, collaborators)
120
+
}
121
+
122
+
func TestSpindleOwnerPermissions(t *testing.T) {
123
+
var (
124
+
e = setup(t)
125
+
ok bool
126
+
err error
127
+
spindle = syntax.DID("did:web:spindle.example.com")
128
+
owner = syntax.DID("did:plc:foo")
129
+
member = syntax.DID("did:plc:bar")
130
+
)
131
+
132
+
assert.NoError(t, e.SetSpindleOwner(owner, spindle))
133
+
assert.NoError(t, e.AddSpindleMember(member, spindle))
134
+
135
+
ok, err = e.IsSpindleMember(owner, spindle)
136
+
assert.NoError(t, err)
137
+
assert.True(t, ok, "spindle owner is spindle member")
138
+
139
+
ok, err = e.IsSpindleMember(member, spindle)
140
+
assert.NoError(t, err)
141
+
assert.True(t, ok, "spindle member is spindle member")
142
+
143
+
ok, err = e.IsSpindleMemberInviteAllowed(owner, spindle)
144
+
assert.NoError(t, err)
145
+
assert.True(t, ok, "spindle owner can invite members")
146
+
147
+
ok, err = e.IsSpindleMemberInviteAllowed(member, spindle)
148
+
assert.NoError(t, err)
149
+
assert.False(t, ok, "spindle member cannot invite members")
150
+
}
+91
rbac2/repo.go
+91
rbac2/repo.go
···
1
+
package rbac2
2
+
3
+
import (
4
+
"slices"
5
+
"strings"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
"tangled.org/core/api/tangled"
9
+
)
10
+
11
+
// AddRepo adds new repo with its owner to rbac enforcer
12
+
func (e *Enforcer) AddRepo(repo syntax.ATURI) error {
13
+
if err := validateAtUri(repo, tangled.RepoNSID); err != nil {
14
+
return err
15
+
}
16
+
user := repo.Authority()
17
+
18
+
return e.setRoleForUser(user.String(), "repo:owner", repo.String())
19
+
}
20
+
21
+
// DeleteRepo deletes all policies related to the repo
22
+
func (e *Enforcer) DeleteRepo(repo syntax.ATURI) error {
23
+
if err := validateAtUri(repo, tangled.RepoNSID); err != nil {
24
+
return err
25
+
}
26
+
27
+
_, err := e.e.DeleteDomains(repo.String())
28
+
return err
29
+
}
30
+
31
+
// AddRepoCollaborator adds new collaborator to the repo
32
+
func (e *Enforcer) AddRepoCollaborator(user syntax.DID, repo syntax.ATURI) error {
33
+
if err := validateAtUri(repo, tangled.RepoNSID); err != nil {
34
+
return err
35
+
}
36
+
37
+
_, err := e.e.AddRoleForUser(user.String(), "repo:collaborator", repo.String())
38
+
return err
39
+
}
40
+
41
+
// RemoveRepoCollaborator removes the collaborator from the repo.
42
+
// This won't remove inherited roles like repository owner.
43
+
func (e *Enforcer) RemoveRepoCollaborator(user syntax.DID, repo syntax.ATURI) error {
44
+
if err := validateAtUri(repo, tangled.RepoNSID); err != nil {
45
+
return err
46
+
}
47
+
48
+
_, err := e.e.DeleteRoleForUser(user.String(), "repo:collaborator", repo.String())
49
+
return err
50
+
}
51
+
52
+
func (e *Enforcer) GetRepoCollaborators(repo syntax.ATURI) ([]syntax.DID, error) {
53
+
var collaborators []syntax.DID
54
+
members, err := e.e.GetImplicitUsersForRole("repo:collaborator", repo.String())
55
+
if err != nil {
56
+
return nil, err
57
+
}
58
+
for _, m := range members {
59
+
if !strings.HasPrefix(m, "did:") { // skip non-user subjects like 'repo:owner'
60
+
continue
61
+
}
62
+
collaborators = append(collaborators, syntax.DID(m))
63
+
}
64
+
65
+
slices.Sort(collaborators)
66
+
return slices.Compact(collaborators), nil
67
+
}
68
+
69
+
func (e *Enforcer) IsRepoOwner(user syntax.DID, repo syntax.ATURI) (bool, error) {
70
+
return e.e.HasRoleForUser(user.String(), "repo:owner", repo.String())
71
+
}
72
+
73
+
func (e *Enforcer) IsRepoCollaborator(user syntax.DID, repo syntax.ATURI) (bool, error) {
74
+
return e.hasImplicitRoleForUser(user.String(), "repo:collaborator", repo.String())
75
+
}
76
+
77
+
func (e *Enforcer) IsRepoWriteAllowed(user syntax.DID, repo syntax.ATURI) (bool, error) {
78
+
return e.e.Enforce(user.String(), repo.String(), "/", "write")
79
+
}
80
+
81
+
func (e *Enforcer) IsRepoSettingsWriteAllowed(user syntax.DID, repo syntax.ATURI) (bool, error) {
82
+
return e.e.Enforce(user.String(), repo.String(), "/settings", "write")
83
+
}
84
+
85
+
func (e *Enforcer) IsRepoCollaboratorInviteAllowed(user syntax.DID, repo syntax.ATURI) (bool, error) {
86
+
return e.e.Enforce(user.String(), repo.String(), "/collaborator", "write")
87
+
}
88
+
89
+
func (e *Enforcer) IsRepoGitPushAllowed(user syntax.DID, repo syntax.ATURI) (bool, error) {
90
+
return e.e.Enforce(user.String(), repo.String(), "/git", "write")
91
+
}
+29
rbac2/spindle.go
+29
rbac2/spindle.go
···
1
+
package rbac2
2
+
3
+
import "github.com/bluesky-social/indigo/atproto/syntax"
4
+
5
+
func (e *Enforcer) SetSpindleOwner(user syntax.DID, spindle syntax.DID) error {
6
+
return e.setRoleForUser(user.String(), "server:owner", intoSpindle(spindle))
7
+
}
8
+
9
+
func (e *Enforcer) IsSpindleMember(user syntax.DID, spindle syntax.DID) (bool, error) {
10
+
return e.hasImplicitRoleForUser(user.String(), "server:member", intoSpindle(spindle))
11
+
}
12
+
13
+
func (e *Enforcer) AddSpindleMember(user syntax.DID, spindle syntax.DID) error {
14
+
_, err := e.e.AddRoleForUser(user.String(), "server:member", intoSpindle(spindle))
15
+
return err
16
+
}
17
+
18
+
func (e *Enforcer) RemoveSpindleMember(user syntax.DID, spindle syntax.DID) error {
19
+
_, err := e.e.DeleteRoleForUser(user.String(), "server:member", intoSpindle(spindle))
20
+
return err
21
+
}
22
+
23
+
func (e *Enforcer) IsSpindleMemberInviteAllowed(user syntax.DID, spindle syntax.DID) (bool, error) {
24
+
return e.e.Enforce(user.String(), intoSpindle(spindle), "/member", "write")
25
+
}
26
+
27
+
func intoSpindle(did syntax.DID) string {
28
+
return "/spindle/" + did.String()
29
+
}
+19
rbac2/tangled_policy.csv
+19
rbac2/tangled_policy.csv
···
1
+
#, policies
2
+
#, sub, dom, obj, act
3
+
p, repo:owner, at://{did}/sh.tangled.repo/{rkey}, /, write
4
+
p, repo:owner, at://{did}/sh.tangled.repo/{rkey}, /collaborator, write
5
+
p, repo:collaborator, at://{did}/sh.tangled.repo/{rkey}, /settings, write
6
+
p, repo:collaborator, at://{did}/sh.tangled.repo/{rkey}, /git, write
7
+
8
+
p, server:owner, /knot/{did}, /member, write
9
+
p, server:member, /knot/{did}, /git, write
10
+
11
+
p, server:owner, /spindle/{did}, /member, write
12
+
13
+
14
+
#, group policies
15
+
#, sub, role, dom
16
+
g, repo:owner, repo:collaborator, at://{did}/sh.tangled.repo/{rkey}
17
+
18
+
g, server:owner, server:member, /knot/{did}
19
+
g, server:owner, server:member, /spindle/{did}
+20
-11
spindle/config/config.go
+20
-11
spindle/config/config.go
···
3
3
import (
4
4
"context"
5
5
"fmt"
6
+
"path/filepath"
6
7
7
8
"github.com/bluesky-social/indigo/atproto/syntax"
8
9
"github.com/sethvargo/go-envconfig"
9
10
)
10
11
11
12
type Server struct {
12
-
ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:6555"`
13
-
DBPath string `env:"DB_PATH, default=spindle.db"`
14
-
Hostname string `env:"HOSTNAME, required"`
15
-
JetstreamEndpoint string `env:"JETSTREAM_ENDPOINT, default=wss://jetstream1.us-west.bsky.network/subscribe"`
16
-
PlcUrl string `env:"PLC_URL, default=https://plc.directory"`
17
-
Dev bool `env:"DEV, default=false"`
18
-
Owner string `env:"OWNER, required"`
19
-
Secrets Secrets `env:",prefix=SECRETS_"`
20
-
LogDir string `env:"LOG_DIR, default=/var/log/spindle"`
21
-
QueueSize int `env:"QUEUE_SIZE, default=100"`
22
-
MaxJobCount int `env:"MAX_JOB_COUNT, default=2"` // max number of jobs that run at a time
13
+
ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:6555"`
14
+
Hostname string `env:"HOSTNAME, required"`
15
+
TapUrl string `env:"TAP_URL, required"`
16
+
PlcUrl string `env:"PLC_URL, default=https://plc.directory"`
17
+
Dev bool `env:"DEV, default=false"`
18
+
Owner syntax.DID `env:"OWNER, required"`
19
+
Secrets Secrets `env:",prefix=SECRETS_"`
20
+
LogDir string `env:"LOG_DIR, default=/var/log/spindle"`
21
+
DataDir string `env:"DATA_DIR, default=/var/lib/spindle"`
22
+
QueueSize int `env:"QUEUE_SIZE, default=100"`
23
+
MaxJobCount int `env:"MAX_JOB_COUNT, default=2"` // max number of jobs that run at a time
23
24
}
24
25
25
26
func (s Server) Did() syntax.DID {
26
27
return syntax.DID(fmt.Sprintf("did:web:%s", s.Hostname))
28
+
}
29
+
30
+
func (s Server) RepoDir() string {
31
+
return filepath.Join(s.DataDir, "repos")
32
+
}
33
+
34
+
func (s Server) DBPath() string {
35
+
return filepath.Join(s.DataDir, "spindle.db")
27
36
}
28
37
29
38
type Secrets struct {
+73
-18
spindle/db/db.go
+73
-18
spindle/db/db.go
···
1
1
package db
2
2
3
3
import (
4
+
"context"
4
5
"database/sql"
5
6
"strings"
6
7
8
+
"github.com/bluesky-social/indigo/atproto/syntax"
7
9
_ "github.com/mattn/go-sqlite3"
10
+
"tangled.org/core/log"
11
+
"tangled.org/core/orm"
8
12
)
9
13
10
14
type DB struct {
11
15
*sql.DB
12
16
}
13
17
14
-
func Make(dbPath string) (*DB, error) {
18
+
func Make(ctx context.Context, dbPath string) (*DB, error) {
15
19
// https://github.com/mattn/go-sqlite3#connection-string
16
20
opts := []string{
17
21
"_foreign_keys=1",
···
20
24
"_auto_vacuum=incremental",
21
25
}
22
26
27
+
logger := log.FromContext(ctx)
28
+
logger = log.SubLogger(logger, "db")
29
+
23
30
db, err := sql.Open("sqlite3", dbPath+"?"+strings.Join(opts, "&"))
24
31
if err != nil {
25
32
return nil, err
26
33
}
27
34
28
-
// NOTE: If any other migration is added here, you MUST
29
-
// copy the pattern in appview: use a single sql.Conn
30
-
// for every migration.
35
+
conn, err := db.Conn(ctx)
36
+
if err != nil {
37
+
return nil, err
38
+
}
39
+
defer conn.Close()
31
40
32
41
_, err = db.Exec(`
33
42
create table if not exists _jetstream (
···
49
58
unique(owner, name)
50
59
);
51
60
61
+
create table if not exists repo_collaborators (
62
+
-- identifiers
63
+
id integer primary key autoincrement,
64
+
did text not null,
65
+
rkey text not null,
66
+
at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.repo.collaborator' || '/' || rkey) stored,
67
+
68
+
repo text not null,
69
+
subject text not null,
70
+
71
+
addedAt text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
72
+
unique(did, rkey)
73
+
);
74
+
52
75
create table if not exists spindle_members (
53
76
-- identifiers for the record
54
77
id integer primary key autoincrement,
···
76
99
return nil, err
77
100
}
78
101
102
+
// run migrations
103
+
104
+
// NOTE: this won't migrate existing records
105
+
// they will be fetched again with tap instead
106
+
orm.RunMigration(conn, logger, "add-rkey-to-repos", func(tx *sql.Tx) error {
107
+
// archive legacy repos (just in case)
108
+
_, err = tx.Exec(`alter table repos rename to repos_old`)
109
+
if err != nil {
110
+
return err
111
+
}
112
+
113
+
_, err := tx.Exec(`
114
+
create table repos (
115
+
-- identifiers
116
+
id integer primary key autoincrement,
117
+
did text not null,
118
+
rkey text not null,
119
+
at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.repo' || '/' || rkey) stored,
120
+
121
+
name text not null,
122
+
knot text not null,
123
+
124
+
addedAt text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
125
+
unique(did, rkey)
126
+
);
127
+
`)
128
+
if err != nil {
129
+
return err
130
+
}
131
+
132
+
return nil
133
+
})
134
+
79
135
return &DB{db}, nil
80
136
}
81
137
82
-
func (d *DB) SaveLastTimeUs(lastTimeUs int64) error {
83
-
_, err := d.Exec(`
84
-
insert into _jetstream (id, last_time_us)
85
-
values (1, ?)
86
-
on conflict(id) do update set last_time_us = excluded.last_time_us
87
-
`, lastTimeUs)
88
-
return err
89
-
}
90
-
91
-
func (d *DB) GetLastTimeUs() (int64, error) {
92
-
var lastTimeUs int64
93
-
row := d.QueryRow(`select last_time_us from _jetstream where id = 1;`)
94
-
err := row.Scan(&lastTimeUs)
95
-
return lastTimeUs, err
138
+
func (d *DB) IsKnownDid(did syntax.DID) (bool, error) {
139
+
// is spindle member / repo collaborator
140
+
var exists bool
141
+
err := d.QueryRow(
142
+
`select exists (
143
+
select 1 from repo_collaborators where subject = ?
144
+
union all
145
+
select 1 from spindle_members where did = ?
146
+
)`,
147
+
did,
148
+
did,
149
+
).Scan(&exists)
150
+
return exists, err
96
151
}
+14
spindle/db/events.go
+14
spindle/db/events.go
···
70
70
return evts, nil
71
71
}
72
72
73
+
func (d *DB) CreatePipelineEvent(rkey string, pipeline tangled.Pipeline, n *notifier.Notifier) error {
74
+
eventJson, err := json.Marshal(pipeline)
75
+
if err != nil {
76
+
return err
77
+
}
78
+
event := Event{
79
+
Rkey: rkey,
80
+
Nsid: tangled.PipelineNSID,
81
+
Created: time.Now().UnixNano(),
82
+
EventJson: string(eventJson),
83
+
}
84
+
return d.insertEvent(event, n)
85
+
}
86
+
73
87
func (d *DB) createStatusEvent(
74
88
workflowId models.WorkflowId,
75
89
statusKind models.StatusKind,
-44
spindle/db/known_dids.go
-44
spindle/db/known_dids.go
···
1
-
package db
2
-
3
-
func (d *DB) AddDid(did string) error {
4
-
_, err := d.Exec(`insert or ignore into known_dids (did) values (?)`, did)
5
-
return err
6
-
}
7
-
8
-
func (d *DB) RemoveDid(did string) error {
9
-
_, err := d.Exec(`delete from known_dids where did = ?`, did)
10
-
return err
11
-
}
12
-
13
-
func (d *DB) GetAllDids() ([]string, error) {
14
-
var dids []string
15
-
16
-
rows, err := d.Query(`select did from known_dids`)
17
-
if err != nil {
18
-
return nil, err
19
-
}
20
-
defer rows.Close()
21
-
22
-
for rows.Next() {
23
-
var did string
24
-
if err := rows.Scan(&did); err != nil {
25
-
return nil, err
26
-
}
27
-
dids = append(dids, did)
28
-
}
29
-
30
-
if err := rows.Err(); err != nil {
31
-
return nil, err
32
-
}
33
-
34
-
return dids, nil
35
-
}
36
-
37
-
func (d *DB) HasKnownDids() bool {
38
-
var count int
39
-
err := d.QueryRow(`select count(*) from known_dids`).Scan(&count)
40
-
if err != nil {
41
-
return false
42
-
}
43
-
return count > 0
44
-
}
+119
-11
spindle/db/repos.go
+119
-11
spindle/db/repos.go
···
1
1
package db
2
2
3
+
import "github.com/bluesky-social/indigo/atproto/syntax"
4
+
3
5
type Repo struct {
4
-
Knot string
5
-
Owner string
6
-
Name string
6
+
Did syntax.DID
7
+
Rkey syntax.RecordKey
8
+
Name string
9
+
Knot string
10
+
}
11
+
12
+
type RepoCollaborator struct {
13
+
Did syntax.DID
14
+
Rkey syntax.RecordKey
15
+
Repo syntax.ATURI
16
+
Subject syntax.DID
17
+
}
18
+
19
+
func (d *DB) PutRepo(repo *Repo) error {
20
+
_, err := d.Exec(
21
+
`insert or ignore into repos (did, rkey, name, knot)
22
+
values (?, ?, ?, ?)
23
+
on conflict(did, rkey) do update set
24
+
name = excluded.name,
25
+
knot = excluded.knot`,
26
+
repo.Did,
27
+
repo.Rkey,
28
+
repo.Name,
29
+
repo.Knot,
30
+
)
31
+
return err
7
32
}
8
33
9
-
func (d *DB) AddRepo(knot, owner, name string) error {
10
-
_, err := d.Exec(`insert or ignore into repos (knot, owner, name) values (?, ?, ?)`, knot, owner, name)
34
+
func (d *DB) DeleteRepo(did syntax.DID, rkey syntax.RecordKey) error {
35
+
_, err := d.Exec(
36
+
`delete from repos where did = ? and rkey = ?`,
37
+
did,
38
+
rkey,
39
+
)
11
40
return err
12
41
}
13
42
···
34
63
return knots, nil
35
64
}
36
65
37
-
func (d *DB) GetRepo(knot, owner, name string) (*Repo, error) {
66
+
func (d *DB) GetRepo(repoAt syntax.ATURI) (*Repo, error) {
38
67
var repo Repo
39
-
40
-
query := "select knot, owner, name from repos where knot = ? and owner = ? and name = ?"
41
-
err := d.DB.QueryRow(query, knot, owner, name).
42
-
Scan(&repo.Knot, &repo.Owner, &repo.Name)
43
-
68
+
err := d.DB.QueryRow(
69
+
`select
70
+
did,
71
+
rkey,
72
+
name,
73
+
knot
74
+
from repos where at_uri = ?`,
75
+
repoAt,
76
+
).Scan(
77
+
&repo.Did,
78
+
&repo.Rkey,
79
+
&repo.Name,
80
+
&repo.Knot,
81
+
)
44
82
if err != nil {
45
83
return nil, err
46
84
}
85
+
return &repo, nil
86
+
}
47
87
88
+
func (d *DB) GetRepoWithName(did syntax.DID, name string) (*Repo, error) {
89
+
var repo Repo
90
+
err := d.DB.QueryRow(
91
+
`select
92
+
did,
93
+
rkey,
94
+
name,
95
+
knot
96
+
from repos where did = ? and name = ?`,
97
+
did,
98
+
name,
99
+
).Scan(
100
+
&repo.Did,
101
+
&repo.Rkey,
102
+
&repo.Name,
103
+
&repo.Knot,
104
+
)
105
+
if err != nil {
106
+
return nil, err
107
+
}
48
108
return &repo, nil
49
109
}
110
+
111
+
func (d *DB) PutRepoCollaborator(collaborator *RepoCollaborator) error {
112
+
_, err := d.Exec(
113
+
`insert into repo_collaborators (did, rkey, repo, subject)
114
+
values (?, ?, ?, ?)
115
+
on conflict(did, rkey) do update set
116
+
repo = excluded.repo,
117
+
subject = excluded.subject`,
118
+
collaborator.Did,
119
+
collaborator.Rkey,
120
+
collaborator.Repo,
121
+
collaborator.Subject,
122
+
)
123
+
return err
124
+
}
125
+
126
+
func (d *DB) RemoveRepoCollaborator(did syntax.DID, rkey syntax.RecordKey) error {
127
+
_, err := d.Exec(
128
+
`delete from repo_collaborators where did = ? and rkey = ?`,
129
+
did,
130
+
rkey,
131
+
)
132
+
return err
133
+
}
134
+
135
+
func (d *DB) GetRepoCollaborator(did syntax.DID, rkey syntax.RecordKey) (*RepoCollaborator, error) {
136
+
var collaborator RepoCollaborator
137
+
err := d.DB.QueryRow(
138
+
`select
139
+
did,
140
+
rkey,
141
+
repo,
142
+
subject
143
+
from repo_collaborators
144
+
where did = ? and rkey = ?`,
145
+
did,
146
+
rkey,
147
+
).Scan(
148
+
&collaborator.Did,
149
+
&collaborator.Rkey,
150
+
&collaborator.Repo,
151
+
&collaborator.Subject,
152
+
)
153
+
if err != nil {
154
+
return nil, err
155
+
}
156
+
return &collaborator, nil
157
+
}
+73
spindle/git/git.go
+73
spindle/git/git.go
···
1
+
package git
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"os"
8
+
"os/exec"
9
+
"strings"
10
+
11
+
"github.com/hashicorp/go-version"
12
+
)
13
+
14
+
func Version() (*version.Version, error) {
15
+
var buf bytes.Buffer
16
+
cmd := exec.Command("git", "version")
17
+
cmd.Stdout = &buf
18
+
cmd.Stderr = os.Stderr
19
+
err := cmd.Run()
20
+
if err != nil {
21
+
return nil, err
22
+
}
23
+
fields := strings.Fields(buf.String())
24
+
if len(fields) < 3 {
25
+
return nil, fmt.Errorf("invalid git version: %s", buf.String())
26
+
}
27
+
28
+
// version string is like: "git version 2.29.3" or "git version 2.29.3.windows.1"
29
+
versionString := fields[2]
30
+
if pos := strings.Index(versionString, "windows"); pos >= 1 {
31
+
versionString = versionString[:pos-1]
32
+
}
33
+
return version.NewVersion(versionString)
34
+
}
35
+
36
+
const WorkflowDir = `/.tangled/workflows`
37
+
38
+
func SparseSyncGitRepo(ctx context.Context, cloneUri, path, rev string) error {
39
+
exist, err := isDir(path)
40
+
if err != nil {
41
+
return err
42
+
}
43
+
if rev == "" {
44
+
rev = "HEAD"
45
+
}
46
+
if !exist {
47
+
if err := exec.Command("git", "clone", "--no-checkout", "--depth=1", "--filter=tree:0", "--revision="+rev, cloneUri, path).Run(); err != nil {
48
+
return fmt.Errorf("git clone: %w", err)
49
+
}
50
+
if err := exec.Command("git", "-C", path, "sparse-checkout", "set", "--no-cone", WorkflowDir).Run(); err != nil {
51
+
return fmt.Errorf("git sparse-checkout set: %w", err)
52
+
}
53
+
} else {
54
+
if err := exec.Command("git", "-C", path, "fetch", "--depth=1", "--filter=tree:0", "origin", rev).Run(); err != nil {
55
+
return fmt.Errorf("git pull: %w", err)
56
+
}
57
+
}
58
+
if err := exec.Command("git", "-C", path, "checkout", rev).Run(); err != nil {
59
+
return fmt.Errorf("git checkout: %w", err)
60
+
}
61
+
return nil
62
+
}
63
+
64
+
func isDir(path string) (bool, error) {
65
+
info, err := os.Stat(path)
66
+
if err == nil && info.IsDir() {
67
+
return true, nil
68
+
}
69
+
if os.IsNotExist(err) {
70
+
return false, nil
71
+
}
72
+
return false, err
73
+
}
-300
spindle/ingester.go
-300
spindle/ingester.go
···
1
-
package spindle
2
-
3
-
import (
4
-
"context"
5
-
"encoding/json"
6
-
"errors"
7
-
"fmt"
8
-
"time"
9
-
10
-
"tangled.org/core/api/tangled"
11
-
"tangled.org/core/eventconsumer"
12
-
"tangled.org/core/rbac"
13
-
"tangled.org/core/spindle/db"
14
-
15
-
comatproto "github.com/bluesky-social/indigo/api/atproto"
16
-
"github.com/bluesky-social/indigo/atproto/identity"
17
-
"github.com/bluesky-social/indigo/atproto/syntax"
18
-
"github.com/bluesky-social/indigo/xrpc"
19
-
"github.com/bluesky-social/jetstream/pkg/models"
20
-
securejoin "github.com/cyphar/filepath-securejoin"
21
-
)
22
-
23
-
type Ingester func(ctx context.Context, e *models.Event) error
24
-
25
-
func (s *Spindle) ingest() Ingester {
26
-
return func(ctx context.Context, e *models.Event) error {
27
-
var err error
28
-
defer func() {
29
-
eventTime := e.TimeUS
30
-
lastTimeUs := eventTime + 1
31
-
if err := s.db.SaveLastTimeUs(lastTimeUs); err != nil {
32
-
err = fmt.Errorf("(deferred) failed to save last time us: %w", err)
33
-
}
34
-
}()
35
-
36
-
if e.Kind != models.EventKindCommit {
37
-
return nil
38
-
}
39
-
40
-
switch e.Commit.Collection {
41
-
case tangled.SpindleMemberNSID:
42
-
err = s.ingestMember(ctx, e)
43
-
case tangled.RepoNSID:
44
-
err = s.ingestRepo(ctx, e)
45
-
case tangled.RepoCollaboratorNSID:
46
-
err = s.ingestCollaborator(ctx, e)
47
-
}
48
-
49
-
if err != nil {
50
-
s.l.Debug("failed to process message", "nsid", e.Commit.Collection, "err", err)
51
-
}
52
-
53
-
return nil
54
-
}
55
-
}
56
-
57
-
func (s *Spindle) ingestMember(_ context.Context, e *models.Event) error {
58
-
var err error
59
-
did := e.Did
60
-
rkey := e.Commit.RKey
61
-
62
-
l := s.l.With("component", "ingester", "record", tangled.SpindleMemberNSID)
63
-
64
-
switch e.Commit.Operation {
65
-
case models.CommitOperationCreate, models.CommitOperationUpdate:
66
-
raw := e.Commit.Record
67
-
record := tangled.SpindleMember{}
68
-
err = json.Unmarshal(raw, &record)
69
-
if err != nil {
70
-
l.Error("invalid record", "error", err)
71
-
return err
72
-
}
73
-
74
-
domain := s.cfg.Server.Hostname
75
-
recordInstance := record.Instance
76
-
77
-
if recordInstance != domain {
78
-
l.Error("domain mismatch", "domain", recordInstance, "expected", domain)
79
-
return fmt.Errorf("domain mismatch: %s != %s", record.Instance, domain)
80
-
}
81
-
82
-
ok, err := s.e.IsSpindleInviteAllowed(did, rbacDomain)
83
-
if err != nil || !ok {
84
-
l.Error("failed to add member", "did", did, "error", err)
85
-
return fmt.Errorf("failed to enforce permissions: %w", err)
86
-
}
87
-
88
-
if err := db.AddSpindleMember(s.db, db.SpindleMember{
89
-
Did: syntax.DID(did),
90
-
Rkey: rkey,
91
-
Instance: recordInstance,
92
-
Subject: syntax.DID(record.Subject),
93
-
Created: time.Now(),
94
-
}); err != nil {
95
-
l.Error("failed to add member", "error", err)
96
-
return fmt.Errorf("failed to add member: %w", err)
97
-
}
98
-
99
-
if err := s.e.AddSpindleMember(rbacDomain, record.Subject); err != nil {
100
-
l.Error("failed to add member", "error", err)
101
-
return fmt.Errorf("failed to add member: %w", err)
102
-
}
103
-
l.Info("added member from firehose", "member", record.Subject)
104
-
105
-
if err := s.db.AddDid(record.Subject); err != nil {
106
-
l.Error("failed to add did", "error", err)
107
-
return fmt.Errorf("failed to add did: %w", err)
108
-
}
109
-
s.jc.AddDid(record.Subject)
110
-
111
-
return nil
112
-
113
-
case models.CommitOperationDelete:
114
-
record, err := db.GetSpindleMember(s.db, did, rkey)
115
-
if err != nil {
116
-
l.Error("failed to find member", "error", err)
117
-
return fmt.Errorf("failed to find member: %w", err)
118
-
}
119
-
120
-
if err := db.RemoveSpindleMember(s.db, did, rkey); err != nil {
121
-
l.Error("failed to remove member", "error", err)
122
-
return fmt.Errorf("failed to remove member: %w", err)
123
-
}
124
-
125
-
if err := s.e.RemoveSpindleMember(rbacDomain, record.Subject.String()); err != nil {
126
-
l.Error("failed to add member", "error", err)
127
-
return fmt.Errorf("failed to add member: %w", err)
128
-
}
129
-
l.Info("added member from firehose", "member", record.Subject)
130
-
131
-
if err := s.db.RemoveDid(record.Subject.String()); err != nil {
132
-
l.Error("failed to add did", "error", err)
133
-
return fmt.Errorf("failed to add did: %w", err)
134
-
}
135
-
s.jc.RemoveDid(record.Subject.String())
136
-
137
-
}
138
-
return nil
139
-
}
140
-
141
-
func (s *Spindle) ingestRepo(ctx context.Context, e *models.Event) error {
142
-
var err error
143
-
did := e.Did
144
-
145
-
l := s.l.With("component", "ingester", "record", tangled.RepoNSID)
146
-
147
-
l.Info("ingesting repo record", "did", did)
148
-
149
-
switch e.Commit.Operation {
150
-
case models.CommitOperationCreate, models.CommitOperationUpdate:
151
-
raw := e.Commit.Record
152
-
record := tangled.Repo{}
153
-
err = json.Unmarshal(raw, &record)
154
-
if err != nil {
155
-
l.Error("invalid record", "error", err)
156
-
return err
157
-
}
158
-
159
-
domain := s.cfg.Server.Hostname
160
-
161
-
// no spindle configured for this repo
162
-
if record.Spindle == nil {
163
-
l.Info("no spindle configured", "name", record.Name)
164
-
return nil
165
-
}
166
-
167
-
// this repo did not want this spindle
168
-
if *record.Spindle != domain {
169
-
l.Info("different spindle configured", "name", record.Name, "spindle", *record.Spindle, "domain", domain)
170
-
return nil
171
-
}
172
-
173
-
// add this repo to the watch list
174
-
if err := s.db.AddRepo(record.Knot, did, record.Name); err != nil {
175
-
l.Error("failed to add repo", "error", err)
176
-
return fmt.Errorf("failed to add repo: %w", err)
177
-
}
178
-
179
-
didSlashRepo, err := securejoin.SecureJoin(did, record.Name)
180
-
if err != nil {
181
-
return err
182
-
}
183
-
184
-
// add repo to rbac
185
-
if err := s.e.AddRepo(did, rbac.ThisServer, didSlashRepo); err != nil {
186
-
l.Error("failed to add repo to enforcer", "error", err)
187
-
return fmt.Errorf("failed to add repo: %w", err)
188
-
}
189
-
190
-
// add collaborators to rbac
191
-
owner, err := s.res.ResolveIdent(ctx, did)
192
-
if err != nil || owner.Handle.IsInvalidHandle() {
193
-
return err
194
-
}
195
-
if err := s.fetchAndAddCollaborators(ctx, owner, didSlashRepo); err != nil {
196
-
return err
197
-
}
198
-
199
-
// add this knot to the event consumer
200
-
src := eventconsumer.NewKnotSource(record.Knot)
201
-
s.ks.AddSource(context.Background(), src)
202
-
203
-
return nil
204
-
205
-
}
206
-
return nil
207
-
}
208
-
209
-
func (s *Spindle) ingestCollaborator(ctx context.Context, e *models.Event) error {
210
-
var err error
211
-
212
-
l := s.l.With("component", "ingester", "record", tangled.RepoCollaboratorNSID, "did", e.Did)
213
-
214
-
l.Info("ingesting collaborator record")
215
-
216
-
switch e.Commit.Operation {
217
-
case models.CommitOperationCreate, models.CommitOperationUpdate:
218
-
raw := e.Commit.Record
219
-
record := tangled.RepoCollaborator{}
220
-
err = json.Unmarshal(raw, &record)
221
-
if err != nil {
222
-
l.Error("invalid record", "error", err)
223
-
return err
224
-
}
225
-
226
-
subjectId, err := s.res.ResolveIdent(ctx, record.Subject)
227
-
if err != nil || subjectId.Handle.IsInvalidHandle() {
228
-
return err
229
-
}
230
-
231
-
repoAt, err := syntax.ParseATURI(record.Repo)
232
-
if err != nil {
233
-
l.Info("rejecting record, invalid repoAt", "repoAt", record.Repo)
234
-
return nil
235
-
}
236
-
237
-
// TODO: get rid of this entirely
238
-
// resolve this aturi to extract the repo record
239
-
owner, err := s.res.ResolveIdent(ctx, repoAt.Authority().String())
240
-
if err != nil || owner.Handle.IsInvalidHandle() {
241
-
return fmt.Errorf("failed to resolve handle: %w", err)
242
-
}
243
-
244
-
xrpcc := xrpc.Client{
245
-
Host: owner.PDSEndpoint(),
246
-
}
247
-
248
-
resp, err := comatproto.RepoGetRecord(ctx, &xrpcc, "", tangled.RepoNSID, repoAt.Authority().String(), repoAt.RecordKey().String())
249
-
if err != nil {
250
-
return err
251
-
}
252
-
253
-
repo := resp.Value.Val.(*tangled.Repo)
254
-
didSlashRepo, _ := securejoin.SecureJoin(owner.DID.String(), repo.Name)
255
-
256
-
// check perms for this user
257
-
if ok, err := s.e.IsCollaboratorInviteAllowed(owner.DID.String(), rbac.ThisServer, didSlashRepo); !ok || err != nil {
258
-
return fmt.Errorf("insufficient permissions: %w", err)
259
-
}
260
-
261
-
// add collaborator to rbac
262
-
if err := s.e.AddCollaborator(record.Subject, rbac.ThisServer, didSlashRepo); err != nil {
263
-
l.Error("failed to add repo to enforcer", "error", err)
264
-
return fmt.Errorf("failed to add repo: %w", err)
265
-
}
266
-
267
-
return nil
268
-
}
269
-
return nil
270
-
}
271
-
272
-
func (s *Spindle) fetchAndAddCollaborators(ctx context.Context, owner *identity.Identity, didSlashRepo string) error {
273
-
l := s.l.With("component", "ingester", "handler", "fetchAndAddCollaborators")
274
-
275
-
l.Info("fetching and adding existing collaborators")
276
-
277
-
xrpcc := xrpc.Client{
278
-
Host: owner.PDSEndpoint(),
279
-
}
280
-
281
-
resp, err := comatproto.RepoListRecords(ctx, &xrpcc, tangled.RepoCollaboratorNSID, "", 50, owner.DID.String(), false)
282
-
if err != nil {
283
-
return err
284
-
}
285
-
286
-
var errs error
287
-
for _, r := range resp.Records {
288
-
if r == nil {
289
-
continue
290
-
}
291
-
record := r.Value.Val.(*tangled.RepoCollaborator)
292
-
293
-
if err := s.e.AddCollaborator(record.Subject, rbac.ThisServer, didSlashRepo); err != nil {
294
-
l.Error("failed to add repo to enforcer", "error", err)
295
-
errors.Join(errs, fmt.Errorf("failed to add repo: %w", err))
296
-
}
297
-
}
298
-
299
-
return errs
300
-
}
+222
-150
spindle/server.go
+222
-150
spindle/server.go
···
4
4
"context"
5
5
_ "embed"
6
6
"encoding/json"
7
+
"errors"
7
8
"fmt"
8
9
"log/slog"
9
10
"maps"
10
11
"net/http"
12
+
"path/filepath"
11
13
"sync"
12
14
15
+
"github.com/bluesky-social/indigo/atproto/syntax"
13
16
"github.com/go-chi/chi/v5"
17
+
"github.com/go-git/go-git/v5/plumbing/object"
18
+
"github.com/hashicorp/go-version"
14
19
"tangled.org/core/api/tangled"
15
20
"tangled.org/core/eventconsumer"
16
21
"tangled.org/core/eventconsumer/cursor"
17
22
"tangled.org/core/idresolver"
18
-
"tangled.org/core/jetstream"
23
+
kgit "tangled.org/core/knotserver/git"
19
24
"tangled.org/core/log"
20
25
"tangled.org/core/notifier"
21
-
"tangled.org/core/rbac"
26
+
"tangled.org/core/rbac2"
22
27
"tangled.org/core/spindle/config"
23
28
"tangled.org/core/spindle/db"
24
29
"tangled.org/core/spindle/engine"
25
30
"tangled.org/core/spindle/engines/nixery"
31
+
"tangled.org/core/spindle/git"
26
32
"tangled.org/core/spindle/models"
27
33
"tangled.org/core/spindle/queue"
28
34
"tangled.org/core/spindle/secrets"
29
35
"tangled.org/core/spindle/xrpc"
36
+
"tangled.org/core/tap"
37
+
"tangled.org/core/tid"
38
+
"tangled.org/core/workflow"
30
39
"tangled.org/core/xrpc/serviceauth"
31
40
)
32
41
33
42
//go:embed motd
34
43
var defaultMotd []byte
35
-
36
-
const (
37
-
rbacDomain = "thisserver"
38
-
)
39
44
40
45
type Spindle struct {
41
-
jc *jetstream.JetstreamClient
46
+
tap *tap.Client
42
47
db *db.DB
43
-
e *rbac.Enforcer
48
+
e *rbac2.Enforcer
44
49
l *slog.Logger
45
50
n *notifier.Notifier
46
51
engs map[string]models.Engine
···
57
62
func New(ctx context.Context, cfg *config.Config, engines map[string]models.Engine) (*Spindle, error) {
58
63
logger := log.FromContext(ctx)
59
64
60
-
d, err := db.Make(cfg.Server.DBPath)
65
+
if err := ensureGitVersion(); err != nil {
66
+
return nil, fmt.Errorf("ensuring git version: %w", err)
67
+
}
68
+
69
+
d, err := db.Make(ctx, cfg.Server.DBPath())
61
70
if err != nil {
62
71
return nil, fmt.Errorf("failed to setup db: %w", err)
63
72
}
64
73
65
-
e, err := rbac.NewEnforcer(cfg.Server.DBPath)
74
+
e, err := rbac2.NewEnforcer(cfg.Server.DBPath())
66
75
if err != nil {
67
76
return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err)
68
77
}
69
-
e.E.EnableAutoSave(true)
70
78
71
79
n := notifier.New()
72
80
···
86
94
}
87
95
logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount)
88
96
case "sqlite", "":
89
-
vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets"))
97
+
vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath(), secrets.WithTableName("secrets"))
90
98
if err != nil {
91
99
return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err)
92
100
}
93
-
logger.Info("using sqlite secrets provider", "path", cfg.Server.DBPath)
101
+
logger.Info("using sqlite secrets provider", "path", cfg.Server.DBPath())
94
102
default:
95
103
return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider)
96
104
}
···
98
106
jq := queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount)
99
107
logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount)
100
108
101
-
collections := []string{
102
-
tangled.SpindleMemberNSID,
103
-
tangled.RepoNSID,
104
-
tangled.RepoCollaboratorNSID,
105
-
}
106
-
jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, log.SubLogger(logger, "jetstream"), d, true, true)
107
-
if err != nil {
108
-
return nil, fmt.Errorf("failed to setup jetstream client: %w", err)
109
-
}
110
-
jc.AddDid(cfg.Server.Owner)
111
-
112
-
// Check if the spindle knows about any Dids;
113
-
dids, err := d.GetAllDids()
114
-
if err != nil {
115
-
return nil, fmt.Errorf("failed to get all dids: %w", err)
116
-
}
117
-
for _, d := range dids {
118
-
jc.AddDid(d)
119
-
}
109
+
tap := tap.NewClient(cfg.Server.TapUrl, "")
120
110
121
111
resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl)
122
112
123
113
spindle := &Spindle{
124
-
jc: jc,
114
+
tap: &tap,
125
115
e: e,
126
116
db: d,
127
117
l: logger,
···
134
124
motd: defaultMotd,
135
125
}
136
126
137
-
err = e.AddSpindle(rbacDomain)
138
-
if err != nil {
139
-
return nil, fmt.Errorf("failed to set rbac domain: %w", err)
140
-
}
141
-
err = spindle.configureOwner()
127
+
err = e.SetSpindleOwner(spindle.cfg.Server.Owner, spindle.cfg.Server.Did())
142
128
if err != nil {
143
129
return nil, err
144
130
}
145
131
logger.Info("owner set", "did", cfg.Server.Owner)
146
132
147
-
cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath)
133
+
cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath())
148
134
if err != nil {
149
135
return nil, fmt.Errorf("failed to setup sqlite3 cursor store: %w", err)
150
136
}
151
137
152
-
err = jc.StartJetstream(ctx, spindle.ingest())
153
-
if err != nil {
154
-
return nil, fmt.Errorf("failed to start jetstream consumer: %w", err)
155
-
}
156
-
157
-
// for each incoming sh.tangled.pipeline, we execute
158
-
// spindle.processPipeline, which in turn enqueues the pipeline
159
-
// job in the above registered queue.
138
+
// spindle listen to knot stream for sh.tangled.git.refUpdate
139
+
// which will sync the local workflow files in spindle and enqueues the
140
+
// pipeline job for on-push workflows
160
141
ccfg := eventconsumer.NewConsumerConfig()
161
142
ccfg.Logger = log.SubLogger(logger, "eventconsumer")
162
143
ccfg.Dev = cfg.Server.Dev
163
-
ccfg.ProcessFunc = spindle.processPipeline
144
+
ccfg.ProcessFunc = spindle.processKnotStream
164
145
ccfg.CursorStore = cursorStore
165
146
knownKnots, err := d.Knots()
166
147
if err != nil {
···
201
182
}
202
183
203
184
// Enforcer returns the RBAC enforcer instance.
204
-
func (s *Spindle) Enforcer() *rbac.Enforcer {
185
+
func (s *Spindle) Enforcer() *rbac2.Enforcer {
205
186
return s.e
206
187
}
207
188
···
235
216
s.ks.Start(ctx)
236
217
}()
237
218
219
+
// ensure server owner is tracked
220
+
if err := s.tap.AddRepos(ctx, []syntax.DID{s.cfg.Server.Owner}); err != nil {
221
+
return err
222
+
}
223
+
224
+
go func() {
225
+
s.l.Info("starting tap stream consumer")
226
+
s.tap.Connect(ctx, &tap.SimpleIndexer{
227
+
EventHandler: s.processEvent,
228
+
})
229
+
}()
230
+
238
231
s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr)
239
232
return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router())
240
233
}
···
293
286
return x.Router()
294
287
}
295
288
296
-
func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, msg eventconsumer.Message) error {
297
-
if msg.Nsid == tangled.PipelineNSID {
298
-
tpl := tangled.Pipeline{}
299
-
err := json.Unmarshal(msg.EventJson, &tpl)
300
-
if err != nil {
301
-
fmt.Println("error unmarshalling", err)
289
+
func (s *Spindle) processKnotStream(ctx context.Context, src eventconsumer.Source, msg eventconsumer.Message) error {
290
+
l := log.FromContext(ctx).With("handler", "processKnotStream")
291
+
l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey)
292
+
if msg.Nsid == tangled.GitRefUpdateNSID {
293
+
event := tangled.GitRefUpdate{}
294
+
if err := json.Unmarshal(msg.EventJson, &event); err != nil {
295
+
l.Error("error unmarshalling", "err", err)
302
296
return err
303
297
}
298
+
l = l.With("repoDid", event.RepoDid, "repoName", event.RepoName)
304
299
305
-
if tpl.TriggerMetadata == nil {
306
-
return fmt.Errorf("no trigger metadata found")
300
+
// resolve repo name to rkey
301
+
// TODO: git.refUpdate should respond with rkey instead of repo name
302
+
repo, err := s.db.GetRepoWithName(syntax.DID(event.RepoDid), event.RepoName)
303
+
if err != nil {
304
+
return fmt.Errorf("get repo with did and name (%s/%s): %w", event.RepoDid, event.RepoName, err)
307
305
}
308
306
309
-
if tpl.TriggerMetadata.Repo == nil {
310
-
return fmt.Errorf("no repo data found")
307
+
// NOTE: we are blindly trusting the knot that it will return only repos it own
308
+
repoCloneUri := s.newRepoCloneUrl(src.Key(), event.RepoDid, event.RepoName)
309
+
repoPath := s.newRepoPath(repo.Did, repo.Rkey)
310
+
if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil {
311
+
return fmt.Errorf("sync git repo: %w", err)
311
312
}
313
+
l.Info("synced git repo")
312
314
313
-
if src.Key() != tpl.TriggerMetadata.Repo.Knot {
314
-
return fmt.Errorf("repo knot does not match event source: %s != %s", src.Key(), tpl.TriggerMetadata.Repo.Knot)
315
+
compiler := workflow.Compiler{
316
+
Trigger: tangled.Pipeline_TriggerMetadata{
317
+
Kind: string(workflow.TriggerKindPush),
318
+
Push: &tangled.Pipeline_PushTriggerData{
319
+
Ref: event.Ref,
320
+
OldSha: event.OldSha,
321
+
NewSha: event.NewSha,
322
+
},
323
+
Repo: &tangled.Pipeline_TriggerRepo{
324
+
Did: repo.Did.String(),
325
+
Knot: repo.Knot,
326
+
Repo: repo.Name,
327
+
},
328
+
},
315
329
}
316
330
317
-
// filter by repos
318
-
_, err = s.db.GetRepo(
319
-
tpl.TriggerMetadata.Repo.Knot,
320
-
tpl.TriggerMetadata.Repo.Did,
321
-
tpl.TriggerMetadata.Repo.Repo,
322
-
)
331
+
// load workflow definitions from rev (without spindle context)
332
+
rawPipeline, err := s.loadPipeline(ctx, repoCloneUri, repoPath, event.NewSha)
323
333
if err != nil {
324
-
return fmt.Errorf("failed to get repo: %w", err)
334
+
return fmt.Errorf("loading pipeline: %w", err)
335
+
}
336
+
if len(rawPipeline) == 0 {
337
+
l.Info("no workflow definition find for the repo. skipping the event")
338
+
return nil
339
+
}
340
+
tpl := compiler.Compile(compiler.Parse(rawPipeline))
341
+
// TODO: pass compile error to workflow log
342
+
for _, w := range compiler.Diagnostics.Errors {
343
+
l.Error(w.String())
344
+
}
345
+
for _, w := range compiler.Diagnostics.Warnings {
346
+
l.Warn(w.String())
325
347
}
326
348
327
349
pipelineId := models.PipelineId{
328
-
Knot: src.Key(),
329
-
Rkey: msg.Rkey,
350
+
Knot: tpl.TriggerMetadata.Repo.Knot,
351
+
Rkey: tid.TID(),
330
352
}
331
-
332
-
workflows := make(map[models.Engine][]models.Workflow)
353
+
if err := s.db.CreatePipelineEvent(pipelineId.Rkey, tpl, s.n); err != nil {
354
+
l.Error("failed to create pipeline event", "err", err)
355
+
return nil
356
+
}
357
+
err = s.processPipeline(ctx, tpl, pipelineId)
358
+
if err != nil {
359
+
return err
360
+
}
361
+
}
333
362
334
-
// Build pipeline environment variables once for all workflows
335
-
pipelineEnv := models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId, s.cfg.Server.Dev)
363
+
return nil
364
+
}
336
365
337
-
for _, w := range tpl.Workflows {
338
-
if w != nil {
339
-
if _, ok := s.engs[w.Engine]; !ok {
340
-
err = s.db.StatusFailed(models.WorkflowId{
341
-
PipelineId: pipelineId,
342
-
Name: w.Name,
343
-
}, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n)
344
-
if err != nil {
345
-
return fmt.Errorf("db.StatusFailed: %w", err)
346
-
}
366
+
func (s *Spindle) loadPipeline(ctx context.Context, repoUri, repoPath, rev string) (workflow.RawPipeline, error) {
367
+
if err := git.SparseSyncGitRepo(ctx, repoUri, repoPath, rev); err != nil {
368
+
return nil, fmt.Errorf("syncing git repo: %w", err)
369
+
}
370
+
gr, err := kgit.Open(repoPath, rev)
371
+
if err != nil {
372
+
return nil, fmt.Errorf("opening git repo: %w", err)
373
+
}
347
374
348
-
continue
349
-
}
375
+
workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir)
376
+
if errors.Is(err, object.ErrDirectoryNotFound) {
377
+
// return empty RawPipeline when directory doesn't exist
378
+
return nil, nil
379
+
} else if err != nil {
380
+
return nil, fmt.Errorf("loading file tree: %w", err)
381
+
}
350
382
351
-
eng := s.engs[w.Engine]
383
+
var rawPipeline workflow.RawPipeline
384
+
for _, e := range workflowDir {
385
+
if !e.IsFile() {
386
+
continue
387
+
}
352
388
353
-
if _, ok := workflows[eng]; !ok {
354
-
workflows[eng] = []models.Workflow{}
355
-
}
389
+
fpath := filepath.Join(workflow.WorkflowDir, e.Name)
390
+
contents, err := gr.RawContent(fpath)
391
+
if err != nil {
392
+
return nil, fmt.Errorf("reading raw content of '%s': %w", fpath, err)
393
+
}
356
394
357
-
ewf, err := s.engs[w.Engine].InitWorkflow(*w, tpl)
358
-
if err != nil {
359
-
return fmt.Errorf("init workflow: %w", err)
360
-
}
395
+
rawPipeline = append(rawPipeline, workflow.RawWorkflow{
396
+
Name: e.Name,
397
+
Contents: contents,
398
+
})
399
+
}
361
400
362
-
// inject TANGLED_* env vars after InitWorkflow
363
-
// This prevents user-defined env vars from overriding them
364
-
if ewf.Environment == nil {
365
-
ewf.Environment = make(map[string]string)
366
-
}
367
-
maps.Copy(ewf.Environment, pipelineEnv)
401
+
return rawPipeline, nil
402
+
}
368
403
369
-
workflows[eng] = append(workflows[eng], *ewf)
404
+
func (s *Spindle) processPipeline(ctx context.Context, tpl tangled.Pipeline, pipelineId models.PipelineId) error {
405
+
// Build pipeline environment variables once for all workflows
406
+
pipelineEnv := models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId, s.cfg.Server.Dev)
370
407
371
-
err = s.db.StatusPending(models.WorkflowId{
372
-
PipelineId: pipelineId,
373
-
Name: w.Name,
374
-
}, s.n)
375
-
if err != nil {
376
-
return fmt.Errorf("db.StatusPending: %w", err)
377
-
}
408
+
// filter & init workflows
409
+
workflows := make(map[models.Engine][]models.Workflow)
410
+
for _, w := range tpl.Workflows {
411
+
if w == nil {
412
+
continue
413
+
}
414
+
if _, ok := s.engs[w.Engine]; !ok {
415
+
err := s.db.StatusFailed(models.WorkflowId{
416
+
PipelineId: pipelineId,
417
+
Name: w.Name,
418
+
}, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n)
419
+
if err != nil {
420
+
return fmt.Errorf("db.StatusFailed: %w", err)
378
421
}
422
+
423
+
continue
379
424
}
380
425
381
-
ok := s.jq.Enqueue(queue.Job{
382
-
Run: func() error {
383
-
engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, ctx, &models.Pipeline{
384
-
RepoOwner: tpl.TriggerMetadata.Repo.Did,
385
-
RepoName: tpl.TriggerMetadata.Repo.Repo,
386
-
Workflows: workflows,
387
-
}, pipelineId)
388
-
return nil
389
-
},
390
-
OnFail: func(jobError error) {
391
-
s.l.Error("pipeline run failed", "error", jobError)
392
-
},
393
-
})
394
-
if ok {
395
-
s.l.Info("pipeline enqueued successfully", "id", msg.Rkey)
396
-
} else {
397
-
s.l.Error("failed to enqueue pipeline: queue is full")
426
+
eng := s.engs[w.Engine]
427
+
428
+
if _, ok := workflows[eng]; !ok {
429
+
workflows[eng] = []models.Workflow{}
398
430
}
431
+
432
+
ewf, err := s.engs[w.Engine].InitWorkflow(*w, tpl)
433
+
if err != nil {
434
+
return fmt.Errorf("init workflow: %w", err)
435
+
}
436
+
437
+
// inject TANGLED_* env vars after InitWorkflow
438
+
// This prevents user-defined env vars from overriding them
439
+
if ewf.Environment == nil {
440
+
ewf.Environment = make(map[string]string)
441
+
}
442
+
maps.Copy(ewf.Environment, pipelineEnv)
443
+
444
+
workflows[eng] = append(workflows[eng], *ewf)
399
445
}
400
446
447
+
// enqueue pipeline
448
+
ok := s.jq.Enqueue(queue.Job{
449
+
Run: func() error {
450
+
engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, ctx, &models.Pipeline{
451
+
RepoOwner: tpl.TriggerMetadata.Repo.Did,
452
+
RepoName: tpl.TriggerMetadata.Repo.Repo,
453
+
Workflows: workflows,
454
+
}, pipelineId)
455
+
return nil
456
+
},
457
+
OnFail: func(jobError error) {
458
+
s.l.Error("pipeline run failed", "error", jobError)
459
+
},
460
+
})
461
+
if !ok {
462
+
return fmt.Errorf("failed to enqueue pipeline: queue is full")
463
+
}
464
+
s.l.Info("pipeline enqueued successfully", "id", pipelineId)
465
+
466
+
// emit StatusPending for all workflows here (after successful enqueue)
467
+
for _, ewfs := range workflows {
468
+
for _, ewf := range ewfs {
469
+
err := s.db.StatusPending(models.WorkflowId{
470
+
PipelineId: pipelineId,
471
+
Name: ewf.Name,
472
+
}, s.n)
473
+
if err != nil {
474
+
return fmt.Errorf("db.StatusPending: %w", err)
475
+
}
476
+
}
477
+
}
401
478
return nil
402
479
}
403
480
404
-
func (s *Spindle) configureOwner() error {
405
-
cfgOwner := s.cfg.Server.Owner
481
+
// newRepoPath creates a path to store repository by its did and rkey.
482
+
// The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey
483
+
func (s *Spindle) newRepoPath(did syntax.DID, rkey syntax.RecordKey) string {
484
+
return filepath.Join(s.cfg.Server.RepoDir(), did.String(), tangled.RepoNSID, rkey.String())
485
+
}
406
486
407
-
existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain)
408
-
if err != nil {
409
-
return err
487
+
func (s *Spindle) newRepoCloneUrl(knot, did, name string) string {
488
+
scheme := "https://"
489
+
if s.cfg.Server.Dev {
490
+
scheme = "http://"
410
491
}
411
-
412
-
switch len(existing) {
413
-
case 0:
414
-
// no owner configured, continue
415
-
case 1:
416
-
// find existing owner
417
-
existingOwner := existing[0]
492
+
return fmt.Sprintf("%s%s/%s/%s", scheme, knot, did, name)
493
+
}
418
494
419
-
// no ownership change, this is okay
420
-
if existingOwner == s.cfg.Server.Owner {
421
-
break
422
-
}
495
+
const RequiredVersion = "2.49.0"
423
496
424
-
// remove existing owner
425
-
err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner)
426
-
if err != nil {
427
-
return nil
428
-
}
429
-
default:
430
-
return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath)
497
+
func ensureGitVersion() error {
498
+
v, err := git.Version()
499
+
if err != nil {
500
+
return fmt.Errorf("fetching git version: %w", err)
501
+
}
502
+
if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) {
503
+
return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion)
431
504
}
432
-
433
-
return s.e.AddSpindleOwner(rbacDomain, cfgOwner)
505
+
return nil
434
506
}
+391
spindle/tap.go
+391
spindle/tap.go
···
1
+
package spindle
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"time"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
"tangled.org/core/api/tangled"
11
+
"tangled.org/core/eventconsumer"
12
+
"tangled.org/core/spindle/db"
13
+
"tangled.org/core/spindle/git"
14
+
"tangled.org/core/spindle/models"
15
+
"tangled.org/core/tap"
16
+
"tangled.org/core/tid"
17
+
"tangled.org/core/workflow"
18
+
)
19
+
20
+
func (s *Spindle) processEvent(ctx context.Context, evt tap.Event) error {
21
+
l := s.l.With("component", "tapIndexer")
22
+
23
+
var err error
24
+
switch evt.Type {
25
+
case tap.EvtRecord:
26
+
switch evt.Record.Collection.String() {
27
+
case tangled.SpindleMemberNSID:
28
+
err = s.processMember(ctx, evt)
29
+
case tangled.RepoNSID:
30
+
err = s.processRepo(ctx, evt)
31
+
case tangled.RepoCollaboratorNSID:
32
+
err = s.processCollaborator(ctx, evt)
33
+
case tangled.RepoPullNSID:
34
+
err = s.processPull(ctx, evt)
35
+
}
36
+
case tap.EvtIdentity:
37
+
// no-op
38
+
}
39
+
40
+
if err != nil {
41
+
l.Error("failed to process message. will retry later", "event.ID", evt.ID, "err", err)
42
+
return err
43
+
}
44
+
return nil
45
+
}
46
+
47
+
// NOTE: make sure to return nil if we don't need to retry (e.g. forbidden, unrelated)
48
+
49
+
func (s *Spindle) processMember(ctx context.Context, evt tap.Event) error {
50
+
l := s.l.With("component", "tapIndexer", "record", evt.Record.AtUri())
51
+
52
+
l.Info("processing spindle.member record")
53
+
54
+
// only listen to members
55
+
if ok, err := s.e.IsSpindleMemberInviteAllowed(evt.Record.Did, s.cfg.Server.Did()); !ok || err != nil {
56
+
l.Warn("forbidden request: member invite not allowed", "did", evt.Record.Did, "error", err)
57
+
return nil
58
+
}
59
+
60
+
switch evt.Record.Action {
61
+
case tap.RecordCreateAction, tap.RecordUpdateAction:
62
+
record := tangled.SpindleMember{}
63
+
if err := json.Unmarshal(evt.Record.Record, &record); err != nil {
64
+
return fmt.Errorf("parsing record: %w", err)
65
+
}
66
+
67
+
domain := s.cfg.Server.Hostname
68
+
if record.Instance != domain {
69
+
l.Info("domain mismatch", "domain", record.Instance, "expected", domain)
70
+
return nil
71
+
}
72
+
73
+
created, err := time.Parse(record.CreatedAt, time.RFC3339)
74
+
if err != nil {
75
+
created = time.Now()
76
+
}
77
+
if err := db.AddSpindleMember(s.db, db.SpindleMember{
78
+
Did: evt.Record.Did,
79
+
Rkey: evt.Record.Rkey.String(),
80
+
Instance: record.Instance,
81
+
Subject: syntax.DID(record.Subject),
82
+
Created: created,
83
+
}); err != nil {
84
+
l.Error("failed to add member", "error", err)
85
+
return fmt.Errorf("adding member to db: %w", err)
86
+
}
87
+
if err := s.e.AddSpindleMember(syntax.DID(record.Subject), s.cfg.Server.Did()); err != nil {
88
+
return fmt.Errorf("adding member to rbac: %w", err)
89
+
}
90
+
if err := s.tap.AddRepos(ctx, []syntax.DID{syntax.DID(record.Subject)}); err != nil {
91
+
return fmt.Errorf("adding did to tap: %w", err)
92
+
}
93
+
94
+
l.Info("added member", "member", record.Subject)
95
+
return nil
96
+
97
+
case tap.RecordDeleteAction:
98
+
var (
99
+
did = evt.Record.Did.String()
100
+
rkey = evt.Record.Rkey.String()
101
+
)
102
+
member, err := db.GetSpindleMember(s.db, did, rkey)
103
+
if err != nil {
104
+
return fmt.Errorf("finding member: %w", err)
105
+
}
106
+
107
+
if err := db.RemoveSpindleMember(s.db, did, rkey); err != nil {
108
+
return fmt.Errorf("removing member from db: %w", err)
109
+
}
110
+
if err := s.e.RemoveSpindleMember(member.Subject, s.cfg.Server.Did()); err != nil {
111
+
return fmt.Errorf("removing member from rbac: %w", err)
112
+
}
113
+
if err := s.tapSafeRemoveDid(ctx, member.Subject); err != nil {
114
+
return fmt.Errorf("removing did from tap: %w", err)
115
+
}
116
+
117
+
l.Info("removed member", "member", member.Subject)
118
+
return nil
119
+
}
120
+
return nil
121
+
}
122
+
123
+
func (s *Spindle) processCollaborator(ctx context.Context, evt tap.Event) error {
124
+
l := s.l.With("component", "tapIndexer", "record", evt.Record.AtUri())
125
+
126
+
l.Info("processing repo.collaborator record")
127
+
128
+
// only listen to members
129
+
if ok, err := s.e.IsSpindleMember(evt.Record.Did, s.cfg.Server.Did()); !ok || err != nil {
130
+
l.Warn("forbidden request: not spindle member", "did", evt.Record.Did, "err", err)
131
+
return nil
132
+
}
133
+
134
+
switch evt.Record.Action {
135
+
case tap.RecordCreateAction, tap.RecordUpdateAction:
136
+
record := tangled.RepoCollaborator{}
137
+
if err := json.Unmarshal(evt.Record.Record, &record); err != nil {
138
+
l.Error("invalid record", "err", err)
139
+
return fmt.Errorf("parsing record: %w", err)
140
+
}
141
+
142
+
// retry later if target repo is not ingested yet
143
+
if _, err := s.db.GetRepo(syntax.ATURI(record.Repo)); err != nil {
144
+
l.Warn("target repo is not ingested yet", "repo", record.Repo, "err", err)
145
+
return fmt.Errorf("target repo is unknown")
146
+
}
147
+
148
+
// check perms for this user
149
+
if ok, err := s.e.IsRepoCollaboratorInviteAllowed(evt.Record.Did, syntax.ATURI(record.Repo)); !ok || err != nil {
150
+
l.Warn("forbidden request collaborator invite not allowed", "did", evt.Record.Did, "err", err)
151
+
return nil
152
+
}
153
+
154
+
if err := s.db.PutRepoCollaborator(&db.RepoCollaborator{
155
+
Did: evt.Record.Did,
156
+
Rkey: evt.Record.Rkey,
157
+
Repo: syntax.ATURI(record.Repo),
158
+
Subject: syntax.DID(record.Subject),
159
+
}); err != nil {
160
+
return fmt.Errorf("adding collaborator to db: %w", err)
161
+
}
162
+
if err := s.e.AddRepoCollaborator(syntax.DID(record.Subject), syntax.ATURI(record.Repo)); err != nil {
163
+
return fmt.Errorf("adding collaborator to rbac: %w", err)
164
+
}
165
+
if err := s.tap.AddRepos(ctx, []syntax.DID{syntax.DID(record.Subject)}); err != nil {
166
+
return fmt.Errorf("adding did to tap: %w", err)
167
+
}
168
+
169
+
l.Info("add repo collaborator", "subejct", record.Subject, "repo", record.Repo)
170
+
return nil
171
+
172
+
case tap.RecordDeleteAction:
173
+
// get existing collaborator
174
+
collaborator, err := s.db.GetRepoCollaborator(evt.Record.Did, evt.Record.Rkey)
175
+
if err != nil {
176
+
return fmt.Errorf("failed to get existing collaborator info: %w", err)
177
+
}
178
+
179
+
// check perms for this user
180
+
if ok, err := s.e.IsRepoCollaboratorInviteAllowed(evt.Record.Did, collaborator.Repo); !ok || err != nil {
181
+
l.Warn("forbidden request collaborator invite not allowed", "did", evt.Record.Did, "err", err)
182
+
return nil
183
+
}
184
+
185
+
if err := s.db.RemoveRepoCollaborator(collaborator.Subject, collaborator.Rkey); err != nil {
186
+
return fmt.Errorf("removing collaborator from db: %w", err)
187
+
}
188
+
if err := s.e.RemoveRepoCollaborator(collaborator.Subject, collaborator.Repo); err != nil {
189
+
return fmt.Errorf("removing collaborator from rbac: %w", err)
190
+
}
191
+
if err := s.tapSafeRemoveDid(ctx, collaborator.Subject); err != nil {
192
+
return fmt.Errorf("removing did from tap: %w", err)
193
+
}
194
+
195
+
l.Info("removed repo collaborator", "subejct", collaborator.Subject, "repo", collaborator.Repo)
196
+
return nil
197
+
}
198
+
return nil
199
+
}
200
+
201
+
func (s *Spindle) processRepo(ctx context.Context, evt tap.Event) error {
202
+
l := s.l.With("component", "tapIndexer", "record", evt.Record.AtUri())
203
+
204
+
l.Info("processing repo record")
205
+
206
+
// only listen to members
207
+
if ok, err := s.e.IsSpindleMember(evt.Record.Did, s.cfg.Server.Did()); !ok || err != nil {
208
+
l.Warn("forbidden request: not spindle member", "did", evt.Record.Did, "err", err)
209
+
return nil
210
+
}
211
+
212
+
switch evt.Record.Action {
213
+
case tap.RecordCreateAction, tap.RecordUpdateAction:
214
+
record := tangled.Repo{}
215
+
if err := json.Unmarshal(evt.Record.Record, &record); err != nil {
216
+
return fmt.Errorf("parsing record: %w", err)
217
+
}
218
+
219
+
domain := s.cfg.Server.Hostname
220
+
if record.Spindle == nil || *record.Spindle != domain {
221
+
if record.Spindle == nil {
222
+
l.Info("spindle isn't configured", "name", record.Name)
223
+
} else {
224
+
l.Info("different spindle configured", "name", record.Name, "spindle", *record.Spindle, "domain", domain)
225
+
}
226
+
if err := s.db.DeleteRepo(evt.Record.Did, evt.Record.Rkey); err != nil {
227
+
return fmt.Errorf("deleting repo from db: %w", err)
228
+
}
229
+
return nil
230
+
}
231
+
232
+
repo := &db.Repo{
233
+
Did: evt.Record.Did,
234
+
Rkey: evt.Record.Rkey,
235
+
Name: record.Name,
236
+
Knot: record.Knot,
237
+
}
238
+
239
+
if err := s.db.PutRepo(repo); err != nil {
240
+
return fmt.Errorf("adding repo to db: %w", err)
241
+
}
242
+
243
+
if err := s.e.AddRepo(evt.Record.AtUri()); err != nil {
244
+
return fmt.Errorf("adding repo to rbac")
245
+
}
246
+
247
+
// add this knot to the event consumer
248
+
src := eventconsumer.NewKnotSource(record.Knot)
249
+
s.ks.AddSource(context.Background(), src)
250
+
251
+
// setup sparse sync
252
+
repoCloneUri := s.newRepoCloneUrl(repo.Knot, repo.Did.String(), repo.Name)
253
+
repoPath := s.newRepoPath(repo.Did, repo.Rkey)
254
+
if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, ""); err != nil {
255
+
return fmt.Errorf("setting up sparse-clone git repo: %w", err)
256
+
}
257
+
258
+
l.Info("added repo", "repo", evt.Record.AtUri())
259
+
return nil
260
+
261
+
case tap.RecordDeleteAction:
262
+
// check perms for this user
263
+
if ok, err := s.e.IsRepoOwner(evt.Record.Did, evt.Record.AtUri()); !ok || err != nil {
264
+
l.Warn("forbidden request: not repo owner", "did", evt.Record.Did, "err", err)
265
+
return nil
266
+
}
267
+
268
+
if err := s.db.DeleteRepo(evt.Record.Did, evt.Record.Rkey); err != nil {
269
+
return fmt.Errorf("deleting repo from db: %w", err)
270
+
}
271
+
272
+
if err := s.e.DeleteRepo(evt.Record.AtUri()); err != nil {
273
+
return fmt.Errorf("deleting repo from rbac: %w", err)
274
+
}
275
+
276
+
l.Info("deleted repo", "repo", evt.Record.AtUri())
277
+
return nil
278
+
}
279
+
return nil
280
+
}
281
+
282
+
func (s *Spindle) processPull(ctx context.Context, evt tap.Event) error {
283
+
l := s.l.With("component", "tapIndexer", "record", evt.Record.AtUri())
284
+
285
+
l.Info("processing pull record")
286
+
287
+
// only listen to live events
288
+
if !evt.Record.Live {
289
+
l.Info("skipping backfill event", "event", evt.Record.AtUri())
290
+
return nil
291
+
}
292
+
293
+
switch evt.Record.Action {
294
+
case tap.RecordCreateAction, tap.RecordUpdateAction:
295
+
record := tangled.RepoPull{}
296
+
if err := json.Unmarshal(evt.Record.Record, &record); err != nil {
297
+
l.Error("invalid record", "err", err)
298
+
return fmt.Errorf("parsing record: %w", err)
299
+
}
300
+
301
+
// ignore legacy records
302
+
if record.Target == nil {
303
+
l.Info("ignoring pull record: target repo is nil")
304
+
return nil
305
+
}
306
+
307
+
// ignore patch-based and fork-based PRs
308
+
if record.Source == nil || record.Source.Repo != nil {
309
+
l.Info("ignoring pull record: not a branch-based pull request")
310
+
return nil
311
+
}
312
+
313
+
// skip if target repo is unknown
314
+
repo, err := s.db.GetRepo(syntax.ATURI(record.Target.Repo))
315
+
if err != nil {
316
+
l.Warn("target repo is not ingested yet", "repo", record.Target.Repo, "err", err)
317
+
return fmt.Errorf("target repo is unknown")
318
+
}
319
+
320
+
compiler := workflow.Compiler{
321
+
Trigger: tangled.Pipeline_TriggerMetadata{
322
+
Kind: string(workflow.TriggerKindPullRequest),
323
+
PullRequest: &tangled.Pipeline_PullRequestTriggerData{
324
+
Action: "create",
325
+
SourceBranch: record.Source.Branch,
326
+
SourceSha: record.Source.Sha,
327
+
TargetBranch: record.Target.Branch,
328
+
},
329
+
Repo: &tangled.Pipeline_TriggerRepo{
330
+
Did: repo.Did.String(),
331
+
Knot: repo.Knot,
332
+
Repo: repo.Name,
333
+
},
334
+
},
335
+
}
336
+
337
+
repoUri := s.newRepoCloneUrl(repo.Knot, repo.Did.String(), repo.Name)
338
+
repoPath := s.newRepoPath(repo.Did, repo.Rkey)
339
+
340
+
// load workflow definitions from rev (without spindle context)
341
+
rawPipeline, err := s.loadPipeline(ctx, repoUri, repoPath, record.Source.Sha)
342
+
if err != nil {
343
+
// don't retry
344
+
l.Error("failed loading pipeline", "err", err)
345
+
return nil
346
+
}
347
+
if len(rawPipeline) == 0 {
348
+
l.Info("no workflow definition find for the repo. skipping the event")
349
+
return nil
350
+
}
351
+
tpl := compiler.Compile(compiler.Parse(rawPipeline))
352
+
// TODO: pass compile error to workflow log
353
+
for _, w := range compiler.Diagnostics.Errors {
354
+
l.Error(w.String())
355
+
}
356
+
for _, w := range compiler.Diagnostics.Warnings {
357
+
l.Warn(w.String())
358
+
}
359
+
360
+
pipelineId := models.PipelineId{
361
+
Knot: tpl.TriggerMetadata.Repo.Knot,
362
+
Rkey: tid.TID(),
363
+
}
364
+
if err := s.db.CreatePipelineEvent(pipelineId.Rkey, tpl, s.n); err != nil {
365
+
l.Error("failed to create pipeline event", "err", err)
366
+
return nil
367
+
}
368
+
err = s.processPipeline(ctx, tpl, pipelineId)
369
+
if err != nil {
370
+
// don't retry
371
+
l.Error("failed processing pipeline", "err", err)
372
+
return nil
373
+
}
374
+
case tap.RecordDeleteAction:
375
+
// no-op
376
+
}
377
+
return nil
378
+
}
379
+
380
+
func (s *Spindle) tapSafeRemoveDid(ctx context.Context, did syntax.DID) error {
381
+
known, err := s.db.IsKnownDid(syntax.DID(did))
382
+
if err != nil {
383
+
return fmt.Errorf("ensuring did known state: %w", err)
384
+
}
385
+
if !known {
386
+
if err := s.tap.RemoveRepos(ctx, []syntax.DID{did}); err != nil {
387
+
return fmt.Errorf("removing did from tap: %w", err)
388
+
}
389
+
}
390
+
return nil
391
+
}
+1
-2
spindle/xrpc/add_secret.go
+1
-2
spindle/xrpc/add_secret.go
···
11
11
"github.com/bluesky-social/indigo/xrpc"
12
12
securejoin "github.com/cyphar/filepath-securejoin"
13
13
"tangled.org/core/api/tangled"
14
-
"tangled.org/core/rbac"
15
14
"tangled.org/core/spindle/secrets"
16
15
xrpcerr "tangled.org/core/xrpc/errors"
17
16
)
···
68
67
return
69
68
}
70
69
71
-
if ok, err := x.Enforcer.IsSettingsAllowed(actorDid.String(), rbac.ThisServer, didPath); !ok || err != nil {
70
+
if ok, err := x.Enforcer.IsRepoSettingsWriteAllowed(actorDid, repoAt); !ok || err != nil {
72
71
l.Error("insufficent permissions", "did", actorDid.String())
73
72
writeError(w, xrpcerr.AccessControlError(actorDid.String()), http.StatusUnauthorized)
74
73
return
+1
-2
spindle/xrpc/list_secrets.go
+1
-2
spindle/xrpc/list_secrets.go
···
11
11
"github.com/bluesky-social/indigo/xrpc"
12
12
securejoin "github.com/cyphar/filepath-securejoin"
13
13
"tangled.org/core/api/tangled"
14
-
"tangled.org/core/rbac"
15
14
"tangled.org/core/spindle/secrets"
16
15
xrpcerr "tangled.org/core/xrpc/errors"
17
16
)
···
63
62
return
64
63
}
65
64
66
-
if ok, err := x.Enforcer.IsSettingsAllowed(actorDid.String(), rbac.ThisServer, didPath); !ok || err != nil {
65
+
if ok, err := x.Enforcer.IsRepoSettingsWriteAllowed(actorDid, repoAt); !ok || err != nil {
67
66
l.Error("insufficent permissions", "did", actorDid.String())
68
67
writeError(w, xrpcerr.AccessControlError(actorDid.String()), http.StatusUnauthorized)
69
68
return
+1
-1
spindle/xrpc/owner.go
+1
-1
spindle/xrpc/owner.go
+1
-26
spindle/xrpc/pipeline_cancelPipeline.go
+1
-26
spindle/xrpc/pipeline_cancelPipeline.go
···
6
6
"net/http"
7
7
"strings"
8
8
9
-
"github.com/bluesky-social/indigo/api/atproto"
10
9
"github.com/bluesky-social/indigo/atproto/syntax"
11
-
"github.com/bluesky-social/indigo/xrpc"
12
-
securejoin "github.com/cyphar/filepath-securejoin"
13
10
"tangled.org/core/api/tangled"
14
-
"tangled.org/core/rbac"
15
11
"tangled.org/core/spindle/models"
16
12
xrpcerr "tangled.org/core/xrpc/errors"
17
13
)
···
53
49
return
54
50
}
55
51
56
-
ident, err := x.Resolver.ResolveIdent(r.Context(), repoAt.Authority().String())
57
-
if err != nil || ident.Handle.IsInvalidHandle() {
58
-
fail(xrpcerr.GenericError(fmt.Errorf("failed to resolve handle: %w", err)))
59
-
return
60
-
}
61
-
62
-
xrpcc := xrpc.Client{Host: ident.PDSEndpoint()}
63
-
resp, err := atproto.RepoGetRecord(r.Context(), &xrpcc, "", tangled.RepoNSID, repoAt.Authority().String(), repoAt.RecordKey().String())
64
-
if err != nil {
65
-
fail(xrpcerr.GenericError(err))
66
-
return
67
-
}
68
-
69
-
repo := resp.Value.Val.(*tangled.Repo)
70
-
didSlashRepo, err := securejoin.SecureJoin(ident.DID.String(), repo.Name)
71
-
if err != nil {
72
-
fail(xrpcerr.GenericError(err))
73
-
return
74
-
}
75
-
76
-
// TODO: fine-grained role based control
77
-
isRepoOwner, err := x.Enforcer.IsRepoOwner(actorDid.String(), rbac.ThisServer, didSlashRepo)
52
+
isRepoOwner, err := x.Enforcer.IsRepoOwner(actorDid, repoAt)
78
53
if err != nil || !isRepoOwner {
79
54
fail(xrpcerr.AccessControlError(actorDid.String()))
80
55
return
+1
-2
spindle/xrpc/remove_secret.go
+1
-2
spindle/xrpc/remove_secret.go
···
10
10
"github.com/bluesky-social/indigo/xrpc"
11
11
securejoin "github.com/cyphar/filepath-securejoin"
12
12
"tangled.org/core/api/tangled"
13
-
"tangled.org/core/rbac"
14
13
"tangled.org/core/spindle/secrets"
15
14
xrpcerr "tangled.org/core/xrpc/errors"
16
15
)
···
62
61
return
63
62
}
64
63
65
-
if ok, err := x.Enforcer.IsSettingsAllowed(actorDid.String(), rbac.ThisServer, didPath); !ok || err != nil {
64
+
if ok, err := x.Enforcer.IsRepoSettingsWriteAllowed(actorDid, repoAt); !ok || err != nil {
66
65
l.Error("insufficent permissions", "did", actorDid.String())
67
66
writeError(w, xrpcerr.AccessControlError(actorDid.String()), http.StatusUnauthorized)
68
67
return
+2
-2
spindle/xrpc/xrpc.go
+2
-2
spindle/xrpc/xrpc.go
···
11
11
"tangled.org/core/api/tangled"
12
12
"tangled.org/core/idresolver"
13
13
"tangled.org/core/notifier"
14
-
"tangled.org/core/rbac"
14
+
"tangled.org/core/rbac2"
15
15
"tangled.org/core/spindle/config"
16
16
"tangled.org/core/spindle/db"
17
17
"tangled.org/core/spindle/models"
···
25
25
type Xrpc struct {
26
26
Logger *slog.Logger
27
27
Db *db.DB
28
-
Enforcer *rbac.Enforcer
28
+
Enforcer *rbac2.Enforcer
29
29
Engines map[string]models.Engine
30
30
Config *config.Config
31
31
Resolver *idresolver.Resolver
+24
tap/simpleIndexer.go
+24
tap/simpleIndexer.go
···
1
+
package tap
2
+
3
+
import "context"
4
+
5
+
type SimpleIndexer struct {
6
+
EventHandler func(ctx context.Context, evt Event) error
7
+
ErrorHandler func(ctx context.Context, err error)
8
+
}
9
+
10
+
var _ Handler = (*SimpleIndexer)(nil)
11
+
12
+
func (i *SimpleIndexer) OnEvent(ctx context.Context, evt Event) error {
13
+
if i.EventHandler == nil {
14
+
return nil
15
+
}
16
+
return i.EventHandler(ctx, evt)
17
+
}
18
+
19
+
func (i *SimpleIndexer) OnError(ctx context.Context, err error) {
20
+
if i.ErrorHandler == nil {
21
+
return
22
+
}
23
+
i.ErrorHandler(ctx, err)
24
+
}
+169
tap/tap.go
+169
tap/tap.go
···
1
+
/// heavily inspired by <https://github.com/bluesky-social/atproto/blob/c7f5a868837d3e9b3289f988fee2267789327b06/packages/tap/README.md>
2
+
3
+
package tap
4
+
5
+
import (
6
+
"bytes"
7
+
"context"
8
+
"encoding/json"
9
+
"fmt"
10
+
"net/http"
11
+
"net/url"
12
+
13
+
"github.com/bluesky-social/indigo/atproto/syntax"
14
+
"github.com/gorilla/websocket"
15
+
"tangled.org/core/log"
16
+
)
17
+
18
+
// type WebsocketOptions struct {
19
+
// maxReconnectSeconds int
20
+
// heartbeatIntervalMs int
21
+
// // onReconnectError
22
+
// }
23
+
24
+
type Handler interface {
25
+
OnEvent(ctx context.Context, evt Event) error
26
+
OnError(ctx context.Context, err error)
27
+
}
28
+
29
+
type Client struct {
30
+
Url string
31
+
AdminPassword string
32
+
HTTPClient *http.Client
33
+
}
34
+
35
+
func NewClient(url, adminPassword string) Client {
36
+
return Client{
37
+
Url: url,
38
+
AdminPassword: adminPassword,
39
+
HTTPClient: &http.Client{},
40
+
}
41
+
}
42
+
43
+
func (c *Client) AddRepos(ctx context.Context, dids []syntax.DID) error {
44
+
body, err := json.Marshal(map[string][]syntax.DID{"dids": dids})
45
+
if err != nil {
46
+
return err
47
+
}
48
+
req, err := http.NewRequestWithContext(ctx, "POST", c.Url+"/repos/add", bytes.NewReader(body))
49
+
if err != nil {
50
+
return err
51
+
}
52
+
req.SetBasicAuth("admin", c.AdminPassword)
53
+
req.Header.Set("Content-Type", "application/json")
54
+
55
+
resp, err := c.HTTPClient.Do(req)
56
+
if err != nil {
57
+
return err
58
+
}
59
+
defer resp.Body.Close()
60
+
if resp.StatusCode != http.StatusOK {
61
+
return fmt.Errorf("tap: /repos/add failed with status %d", resp.StatusCode)
62
+
}
63
+
return nil
64
+
}
65
+
66
+
func (c *Client) RemoveRepos(ctx context.Context, dids []syntax.DID) error {
67
+
body, err := json.Marshal(map[string][]syntax.DID{"dids": dids})
68
+
if err != nil {
69
+
return err
70
+
}
71
+
req, err := http.NewRequestWithContext(ctx, "POST", c.Url+"/repos/remove", bytes.NewReader(body))
72
+
if err != nil {
73
+
return err
74
+
}
75
+
req.SetBasicAuth("admin", c.AdminPassword)
76
+
req.Header.Set("Content-Type", "application/json")
77
+
78
+
resp, err := c.HTTPClient.Do(req)
79
+
if err != nil {
80
+
return err
81
+
}
82
+
defer resp.Body.Close()
83
+
if resp.StatusCode != http.StatusOK {
84
+
return fmt.Errorf("tap: /repos/remove failed with status %d", resp.StatusCode)
85
+
}
86
+
return nil
87
+
}
88
+
89
+
func (c *Client) Connect(ctx context.Context, handler Handler) error {
90
+
l := log.FromContext(ctx)
91
+
92
+
u, err := url.Parse(c.Url)
93
+
if err != nil {
94
+
return err
95
+
}
96
+
if u.Scheme == "https" {
97
+
u.Scheme = "wss"
98
+
} else {
99
+
u.Scheme = "ws"
100
+
}
101
+
u.Path = "/channel"
102
+
103
+
// TODO: set auth on dial
104
+
105
+
url := u.String()
106
+
107
+
// var backoff int
108
+
// for {
109
+
// select {
110
+
// case <-ctx.Done():
111
+
// return ctx.Err()
112
+
// default:
113
+
// }
114
+
//
115
+
// header := http.Header{
116
+
// "Authorization": []string{""},
117
+
// }
118
+
// conn, res, err := websocket.DefaultDialer.DialContext(ctx, url, header)
119
+
// if err != nil {
120
+
// l.Warn("dialing failed", "url", url, "err", err, "backoff", backoff)
121
+
// time.Sleep(time.Duration(5+backoff) * time.Second)
122
+
// backoff++
123
+
//
124
+
// continue
125
+
// } else {
126
+
// backoff = 0
127
+
// }
128
+
//
129
+
// l.Info("event subscription response", "code", res.StatusCode)
130
+
// }
131
+
132
+
// TODO: keep websocket connection alive
133
+
conn, _, err := websocket.DefaultDialer.DialContext(ctx, url, nil)
134
+
if err != nil {
135
+
return err
136
+
}
137
+
defer conn.Close()
138
+
139
+
for {
140
+
select {
141
+
case <-ctx.Done():
142
+
return ctx.Err()
143
+
default:
144
+
}
145
+
_, message, err := conn.ReadMessage()
146
+
if err != nil {
147
+
return err
148
+
}
149
+
150
+
var ev Event
151
+
if err := json.Unmarshal(message, &ev); err != nil {
152
+
handler.OnError(ctx, fmt.Errorf("failed to parse message: %w", err))
153
+
continue
154
+
}
155
+
if err := handler.OnEvent(ctx, ev); err != nil {
156
+
handler.OnError(ctx, fmt.Errorf("failed to process event %d: %w", ev.ID, err))
157
+
continue
158
+
}
159
+
160
+
ack := map[string]any{
161
+
"type": "ack",
162
+
"id": ev.ID,
163
+
}
164
+
if err := conn.WriteJSON(ack); err != nil {
165
+
l.Warn("failed to send ack", "err", err)
166
+
continue
167
+
}
168
+
}
169
+
}
+62
tap/types.go
+62
tap/types.go
···
1
+
package tap
2
+
3
+
import (
4
+
"encoding/json"
5
+
"fmt"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
)
9
+
10
+
type EventType string
11
+
12
+
const (
13
+
EvtRecord EventType = "record"
14
+
EvtIdentity EventType = "identity"
15
+
)
16
+
17
+
type Event struct {
18
+
ID int64 `json:"id"`
19
+
Type EventType `json:"type"`
20
+
Record *RecordEventData `json:"record,omitempty"`
21
+
Identity *IdentityEventData `json:"identity,omitempty"`
22
+
}
23
+
24
+
type RecordEventData struct {
25
+
Live bool `json:"live"`
26
+
Did syntax.DID `json:"did"`
27
+
Rev string `json:"rev"`
28
+
Collection syntax.NSID `json:"collection"`
29
+
Rkey syntax.RecordKey `json:"rkey"`
30
+
Action RecordAction `json:"action"`
31
+
Record json.RawMessage `json:"record,omitempty"`
32
+
CID *syntax.CID `json:"cid,omitempty"`
33
+
}
34
+
35
+
func (r *RecordEventData) AtUri() syntax.ATURI {
36
+
return syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", r.Did, r.Collection, r.Rkey))
37
+
}
38
+
39
+
type RecordAction string
40
+
41
+
const (
42
+
RecordCreateAction RecordAction = "create"
43
+
RecordUpdateAction RecordAction = "update"
44
+
RecordDeleteAction RecordAction = "delete"
45
+
)
46
+
47
+
type IdentityEventData struct {
48
+
DID syntax.DID `json:"did"`
49
+
Handle string `json:"handle"`
50
+
IsActive bool `json:"is_active"`
51
+
Status RepoStatus `json:"status"`
52
+
}
53
+
54
+
type RepoStatus string
55
+
56
+
const (
57
+
RepoStatusActive RepoStatus = "active"
58
+
RepoStatusTakendown RepoStatus = "takendown"
59
+
RepoStatusSuspended RepoStatus = "suspended"
60
+
RepoStatusDeactivated RepoStatus = "deactivated"
61
+
RepoStatusDeleted RepoStatus = "deleted"
62
+
)