evan.jarrett.net
A container registry that uses the AT Protocol for manifest storage and S3 for blob storage.
atcr.io
docker
container
atproto
go
lazy load crew membership in admin panel
+1
pkg/hold/admin/admin.go
+1
pkg/hold/admin/admin.go
+85
-53
pkg/hold/admin/handlers_crew.go
+85
-53
pkg/hold/admin/handlers_crew.go
···
9
"time"
10
11
"atcr.io/pkg/atproto"
12
-
"atcr.io/pkg/hold/pds"
13
"github.com/go-chi/chi/v5"
14
)
15
16
-
// CrewMemberView represents a crew member for display
17
type CrewMemberView struct {
18
RKey string
19
DID string
···
28
AddedAt time.Time
29
}
30
31
// resolveHandle attempts to resolve a DID to a handle
32
// Returns empty string if resolution fails
33
func resolveHandle(ctx context.Context, did string) string {
···
50
Limit string
51
}
52
53
-
// getCrewViews builds the crew member view list
54
-
func (ui *AdminUI) getCrewViews(ctx context.Context) ([]CrewMemberView, error) {
55
-
crew, err := ui.pds.ListCrewMembers(ctx)
56
if err != nil {
57
-
return nil, err
58
-
}
59
-
60
-
// Single bulk query for all user quotas
61
-
allQuotas, err := ui.pds.GetAllUserQuotas(ctx)
62
-
if err != nil {
63
-
slog.Warn("Failed to get all user quotas for crew views", "error", err)
64
-
allQuotas = make(map[string]*pds.QuotaStats)
65
}
66
67
defaultTier := "default"
···
69
defaultTier = ui.quotaMgr.GetDefaultTier()
70
}
71
72
-
var crewViews []CrewMemberView
73
for _, member := range crew {
74
tier := member.Record.Tier
75
if tier == "" {
76
tier = defaultTier
77
}
78
-
79
-
view := CrewMemberView{
80
RKey: member.Rkey,
81
DID: member.Record.Member,
82
-
Handle: resolveHandle(ctx, member.Record.Member),
83
Role: member.Record.Role,
84
Permissions: member.Record.Permissions,
85
Tier: tier,
86
AddedAt: parseTime(member.Record.AddedAt),
87
-
}
88
-
89
-
usage := int64(0)
90
-
if q, ok := allQuotas[member.Record.Member]; ok {
91
-
usage = q.TotalSize
92
-
}
93
-
94
-
if ui.quotaMgr != nil && ui.quotaMgr.IsEnabled() {
95
-
if limit := ui.quotaMgr.GetTierLimit(tier); limit != nil {
96
-
view.TierLimit = formatHumanBytes(*limit)
97
-
if *limit > 0 {
98
-
view.UsagePercent = int(float64(usage) / float64(*limit) * 100)
99
-
}
100
-
} else {
101
-
view.TierLimit = "Unlimited"
102
-
}
103
-
} else {
104
-
view.TierLimit = "Unlimited"
105
-
}
106
-
107
-
view.CurrentUsage = usage
108
-
view.UsageHuman = formatHumanBytes(view.CurrentUsage)
109
-
110
-
crewViews = append(crewViews, view)
111
}
112
113
-
sort.Slice(crewViews, func(i, j int) bool {
114
-
return crewViews[i].CurrentUsage > crewViews[j].CurrentUsage
115
})
116
117
-
return crewViews, nil
118
}
119
120
-
// handleCrewTab returns the crew tab content (HTMX partial)
121
-
func (ui *AdminUI) handleCrewTab(w http.ResponseWriter, r *http.Request) {
122
-
crewViews, err := ui.getCrewViews(r.Context())
123
if err != nil {
124
-
http.Error(w, "Failed to list crew: "+err.Error(), http.StatusInternalServerError)
125
return
126
}
127
128
-
data := struct {
129
-
Crew []CrewMemberView
130
-
}{
131
-
Crew: crewViews,
132
}
133
-
ui.renderTemplate(w, "partials/tab_crew.html", data)
134
}
135
136
// handleCrewAddForm displays the add crew form
···
9
"time"
10
11
"atcr.io/pkg/atproto"
12
"github.com/go-chi/chi/v5"
13
)
14
15
+
// CrewMemberView represents a crew member for display (populated row)
16
type CrewMemberView struct {
17
RKey string
18
DID string
···
27
AddedAt time.Time
28
}
29
30
+
// CrewSkeletonView is the minimal crew member data for skeleton rendering.
31
+
// Contains only data available from the MST walk (no network calls).
32
+
type CrewSkeletonView struct {
33
+
RKey string
34
+
DID string
35
+
Role string
36
+
Permissions []string
37
+
Tier string
38
+
AddedAt time.Time
39
+
}
40
+
41
// resolveHandle attempts to resolve a DID to a handle
42
// Returns empty string if resolution fails
43
func resolveHandle(ctx context.Context, did string) string {
···
60
Limit string
61
}
62
63
+
// handleCrewTab returns the crew tab content (HTMX partial).
64
+
// Only does the MST walk — no handle resolution or usage queries.
65
+
// Each row lazy-loads its details via handleCrewMemberInfo.
66
+
func (ui *AdminUI) handleCrewTab(w http.ResponseWriter, r *http.Request) {
67
+
crew, err := ui.pds.ListCrewMembers(r.Context())
68
if err != nil {
69
+
http.Error(w, "Failed to list crew: "+err.Error(), http.StatusInternalServerError)
70
+
return
71
}
72
73
defaultTier := "default"
···
75
defaultTier = ui.quotaMgr.GetDefaultTier()
76
}
77
78
+
var skeletons []CrewSkeletonView
79
for _, member := range crew {
80
tier := member.Record.Tier
81
if tier == "" {
82
tier = defaultTier
83
}
84
+
skeletons = append(skeletons, CrewSkeletonView{
85
RKey: member.Rkey,
86
DID: member.Record.Member,
87
Role: member.Record.Role,
88
Permissions: member.Record.Permissions,
89
Tier: tier,
90
AddedAt: parseTime(member.Record.AddedAt),
91
+
})
92
}
93
94
+
sort.Slice(skeletons, func(i, j int) bool {
95
+
return skeletons[i].AddedAt.After(skeletons[j].AddedAt)
96
})
97
98
+
data := struct {
99
+
Crew []CrewSkeletonView
100
+
}{
101
+
Crew: skeletons,
102
+
}
103
+
ui.renderTemplate(w, "partials/tab_crew.html", data)
104
}
105
106
+
// handleCrewMemberInfo returns a fully populated crew member row (HTMX partial).
107
+
// Called per-row via hx-trigger="load" — resolves handle and fetches usage.
108
+
func (ui *AdminUI) handleCrewMemberInfo(w http.ResponseWriter, r *http.Request) {
109
+
ctx := r.Context()
110
+
rkey := r.URL.Query().Get("rkey")
111
+
if rkey == "" {
112
+
http.Error(w, "Missing rkey parameter", http.StatusBadRequest)
113
+
return
114
+
}
115
+
116
+
_, member, err := ui.pds.GetCrewMember(ctx, rkey)
117
if err != nil {
118
+
slog.Warn("Failed to get crew member for lazy load", "rkey", rkey, "error", err)
119
+
http.Error(w, "Crew member not found", http.StatusNotFound)
120
return
121
}
122
123
+
handle := resolveHandle(ctx, member.Member)
124
+
125
+
usage := int64(0)
126
+
if q, err := ui.pds.GetQuotaForUser(ctx, member.Member); err == nil {
127
+
usage = q.TotalSize
128
+
}
129
+
130
+
defaultTier := "default"
131
+
if ui.quotaMgr != nil && ui.quotaMgr.IsEnabled() {
132
+
defaultTier = ui.quotaMgr.GetDefaultTier()
133
+
}
134
+
135
+
tier := member.Tier
136
+
if tier == "" {
137
+
tier = defaultTier
138
}
139
+
140
+
view := CrewMemberView{
141
+
RKey: rkey,
142
+
DID: member.Member,
143
+
Handle: handle,
144
+
Role: member.Role,
145
+
Permissions: member.Permissions,
146
+
Tier: tier,
147
+
CurrentUsage: usage,
148
+
UsageHuman: formatHumanBytes(usage),
149
+
AddedAt: parseTime(member.AddedAt),
150
+
}
151
+
152
+
if ui.quotaMgr != nil && ui.quotaMgr.IsEnabled() {
153
+
if limit := ui.quotaMgr.GetTierLimit(tier); limit != nil {
154
+
view.TierLimit = formatHumanBytes(*limit)
155
+
if *limit > 0 {
156
+
view.UsagePercent = int(float64(usage) / float64(*limit) * 100)
157
+
}
158
+
} else {
159
+
view.TierLimit = "Unlimited"
160
+
}
161
+
} else {
162
+
view.TierLimit = "Unlimited"
163
+
}
164
+
165
+
ui.renderTemplate(w, "partials/crew_member_row.html", view)
166
}
167
168
// handleCrewAddForm displays the add crew form
+38
pkg/hold/admin/templates/partials/crew_member_row.html
+38
pkg/hold/admin/templates/partials/crew_member_row.html
···
···
1
+
{{define "partials/crew_member_row.html"}}
2
+
<tr id="crew-{{.RKey}}">
3
+
<td>
4
+
<div>
5
+
{{if .Handle}}<strong class="text-base-content">{{.Handle}}</strong><br>{{end}}
6
+
<code class="text-xs text-base-content/50 break-all font-mono">{{.DID}}</code>
7
+
</div>
8
+
</td>
9
+
<td>{{.Role}}</td>
10
+
<td>
11
+
{{range .Permissions}}
12
+
<span class="badge badge-ghost badge-sm mr-1 mb-1">{{.}}</span>
13
+
{{end}}
14
+
</td>
15
+
<td>
16
+
<span class="badge badge-primary badge-sm">{{.Tier}}</span>
17
+
<br><small class="text-base-content/50">{{.TierLimit}}</small>
18
+
</td>
19
+
<td>
20
+
<div class="flex flex-col gap-1 min-w-24">
21
+
<span class="text-sm">{{.UsageHuman}}</span>
22
+
<progress class="progress {{if gt .UsagePercent 90}}progress-error{{else if gt .UsagePercent 75}}progress-warning{{else}}progress-primary{{end}} w-full" value="{{.UsagePercent}}" max="100"></progress>
23
+
<small class="text-base-content/50">{{.UsagePercent}}%</small>
24
+
</div>
25
+
</td>
26
+
<td class="text-sm text-base-content/70">{{formatTime .AddedAt}}</td>
27
+
<td>
28
+
<div class="flex gap-1 justify-end">
29
+
<a href="/admin/crew/{{.RKey}}" class="btn btn-ghost btn-sm btn-square" title="Edit" aria-label="Edit crew member {{if .Handle}}{{.Handle}}{{else}}{{.DID}}{{end}}">
30
+
{{ icon "pencil" "size-4" }}
31
+
</a>
32
+
<button class="btn btn-error btn-ghost btn-sm btn-square" title="Delete" aria-label="Remove crew member {{if .Handle}}{{.Handle}}{{else}}{{.DID}}{{end}}" hx-post="/admin/crew/{{.RKey}}/delete" hx-confirm="Remove this crew member?" hx-target="#crew-{{.RKey}}" hx-swap="outerHTML">
33
+
{{ icon "trash-2" "size-4" }}
34
+
</button>
35
+
</div>
36
+
</td>
37
+
</tr>
38
+
{{end}}
+10
-20
pkg/hold/admin/templates/partials/tab_crew.html
+10
-20
pkg/hold/admin/templates/partials/tab_crew.html
···
36
</thead>
37
<tbody id="crew-list">
38
{{range .Crew}}
39
-
<tr id="crew-{{.RKey}}">
40
<td>
41
<div>
42
-
{{if .Handle}}<strong class="text-base-content">{{.Handle}}</strong><br>{{end}}
43
-
<code class="text-xs text-base-content/50 break-all font-mono">{{.DID}}</code>
44
</div>
45
</td>
46
<td>{{.Role}}</td>
···
51
</td>
52
<td>
53
<span class="badge badge-primary badge-sm">{{.Tier}}</span>
54
-
<br><small class="text-base-content/50">{{.TierLimit}}</small>
55
</td>
56
-
<td>
57
-
<div class="flex flex-col gap-1 min-w-24">
58
-
<span class="text-sm">{{.UsageHuman}}</span>
59
-
<progress class="progress {{if gt .UsagePercent 90}}progress-error{{else if gt .UsagePercent 75}}progress-warning{{else}}progress-primary{{end}} w-full" value="{{.UsagePercent}}" max="100"></progress>
60
-
<small class="text-base-content/50">{{.UsagePercent}}%</small>
61
-
</div>
62
</td>
63
<td class="text-sm text-base-content/70">{{formatTime .AddedAt}}</td>
64
-
<td>
65
-
<div class="flex gap-1 justify-end">
66
-
<a href="/admin/crew/{{.RKey}}" class="btn btn-ghost btn-sm btn-square" title="Edit" aria-label="Edit crew member {{if .Handle}}{{.Handle}}{{else}}{{.DID}}{{end}}">
67
-
{{ icon "pencil" "size-4" }}
68
-
</a>
69
-
<button class="btn btn-error btn-ghost btn-sm btn-square" title="Delete" aria-label="Remove crew member {{if .Handle}}{{.Handle}}{{else}}{{.DID}}{{end}}" hx-post="/admin/crew/{{.RKey}}/delete" hx-confirm="Remove this crew member?" hx-target="#crew-{{.RKey}}" hx-swap="outerHTML">
70
-
{{ icon "trash-2" "size-4" }}
71
-
</button>
72
-
</div>
73
-
</td>
74
</tr>
75
{{end}}
76
</tbody>
···
36
</thead>
37
<tbody id="crew-list">
38
{{range .Crew}}
39
+
<tr id="crew-{{.RKey}}"
40
+
hx-get="/admin/api/crew/member?rkey={{.RKey}}"
41
+
hx-trigger="load"
42
+
hx-swap="outerHTML">
43
<td>
44
<div>
45
+
<span class="loading loading-spinner loading-xs"></span>
46
+
<br>
47
+
<code class="text-xs text-base-content/50 break-all font-mono">{{truncate .DID 32}}</code>
48
</div>
49
</td>
50
<td>{{.Role}}</td>
···
55
</td>
56
<td>
57
<span class="badge badge-primary badge-sm">{{.Tier}}</span>
58
</td>
59
+
<td class="text-base-content/30 text-sm">
60
+
<span class="loading loading-spinner loading-xs"></span>
61
</td>
62
<td class="text-sm text-base-content/70">{{formatTime .AddedAt}}</td>
63
+
<td></td>
64
</tr>
65
{{end}}
66
</tbody>
+36
-21
pkg/hold/gc/gc.go
+36
-21
pkg/hold/gc/gc.go
···
741
return orphaned, totalBlobs, nil
742
}
743
744
// reconcileMissingRecords creates layer records for manifest+layer pairs that are missing.
745
func (gc *GarbageCollector) reconcileMissingRecords(ctx context.Context, missing []MissingRecordDetail, result *GCResult) {
746
-
for i, m := range missing {
747
-
record := atproto.NewLayerRecord(
748
-
m.Digest,
749
-
m.Size,
750
-
m.MediaType,
751
-
m.UserDID,
752
-
m.ManifestURI,
753
-
)
754
-
if _, _, err := gc.pds.CreateLayerRecord(ctx, record); err != nil {
755
-
gc.logger.Error("Failed to create reconciled layer record",
756
-
"digest", m.Digest,
757
-
"manifest", m.ManifestURI,
758
"error", err)
759
continue
760
}
761
-
result.RecordsReconciled++
762
-
if result.RecordsReconciled%100 == 0 {
763
-
gc.logger.Info("Reconciliation progress",
764
-
"created", result.RecordsReconciled,
765
-
"total", len(missing))
766
-
}
767
768
-
// Throttle: ramp delay based on record index to avoid flooding relays
769
-
delay := max(10*time.Millisecond, time.Duration(i)*100*time.Microsecond)
770
-
time.Sleep(delay)
771
}
772
773
if result.RecordsReconciled > 0 {
···
741
return orphaned, totalBlobs, nil
742
}
743
744
+
// reconcileBatchSize is the number of layer records per repo commit.
745
+
// Batching reduces firehose events from N to N/batchSize.
746
+
const reconcileBatchSize = 200
747
+
748
// reconcileMissingRecords creates layer records for manifest+layer pairs that are missing.
749
+
// Records are batched into single commits to avoid flooding relays with firehose events.
750
func (gc *GarbageCollector) reconcileMissingRecords(ctx context.Context, missing []MissingRecordDetail, result *GCResult) {
751
+
for i := 0; i < len(missing); i += reconcileBatchSize {
752
+
end := i + reconcileBatchSize
753
+
if end > len(missing) {
754
+
end = len(missing)
755
+
}
756
+
chunk := missing[i:end]
757
+
758
+
records := make([]*atproto.LayerRecord, 0, len(chunk))
759
+
for _, m := range chunk {
760
+
records = append(records, atproto.NewLayerRecord(
761
+
m.Digest,
762
+
m.Size,
763
+
m.MediaType,
764
+
m.UserDID,
765
+
m.ManifestURI,
766
+
))
767
+
}
768
+
769
+
created, err := gc.pds.BatchCreateLayerRecords(ctx, records)
770
+
if err != nil {
771
+
gc.logger.Error("Failed to create reconciled layer batch",
772
+
"batchStart", i,
773
+
"batchSize", len(chunk),
774
"error", err)
775
continue
776
}
777
778
+
result.RecordsReconciled += int64(created)
779
+
gc.logger.Info("Reconciliation progress",
780
+
"created", result.RecordsReconciled,
781
+
"total", len(missing),
782
+
"batch", len(chunk))
783
+
784
+
// Small delay between batches as a courtesy to relays
785
+
time.Sleep(100 * time.Millisecond)
786
}
787
788
if result.RecordsReconciled > 0 {
+41
pkg/hold/pds/layer.go
+41
pkg/hold/pds/layer.go
···
3
import (
4
"context"
5
"fmt"
6
7
"atcr.io/pkg/atproto"
8
"atcr.io/pkg/hold/quota"
9
lexutil "github.com/bluesky-social/indigo/lex/util"
10
"github.com/bluesky-social/indigo/repo"
11
)
···
39
}
40
41
return rkey, recordCID.String(), nil
42
}
43
44
// GetLayerRecord retrieves a specific layer record by rkey
···
3
import (
4
"context"
5
"fmt"
6
+
"log/slog"
7
8
"atcr.io/pkg/atproto"
9
"atcr.io/pkg/hold/quota"
10
+
indigoatproto "github.com/bluesky-social/indigo/api/atproto"
11
lexutil "github.com/bluesky-social/indigo/lex/util"
12
"github.com/bluesky-social/indigo/repo"
13
)
···
41
}
42
43
return rkey, recordCID.String(), nil
44
+
}
45
+
46
+
// BatchCreateLayerRecords creates multiple layer records in a single repo commit.
47
+
// This produces one firehose event instead of one per record.
48
+
// Invalid records are skipped with a warning. Returns the number of records created.
49
+
func (p *HoldPDS) BatchCreateLayerRecords(ctx context.Context, records []*atproto.LayerRecord) (int, error) {
50
+
var writes []*indigoatproto.RepoApplyWrites_Input_Writes_Elem
51
+
52
+
for _, record := range records {
53
+
if record.Type != atproto.LayerCollection {
54
+
slog.Warn("Skipping invalid record type in batch", "type", record.Type)
55
+
continue
56
+
}
57
+
if record.Digest == "" {
58
+
slog.Warn("Skipping record with empty digest in batch")
59
+
continue
60
+
}
61
+
if record.Size <= 0 {
62
+
slog.Warn("Skipping record with non-positive size in batch", "size", record.Size)
63
+
continue
64
+
}
65
+
66
+
writes = append(writes, &indigoatproto.RepoApplyWrites_Input_Writes_Elem{
67
+
RepoApplyWrites_Create: &indigoatproto.RepoApplyWrites_Create{
68
+
Collection: atproto.LayerCollection,
69
+
Value: &lexutil.LexiconTypeDecoder{Val: record},
70
+
},
71
+
})
72
+
}
73
+
74
+
if len(writes) == 0 {
75
+
return 0, nil
76
+
}
77
+
78
+
if err := p.repomgr.BatchWrite(ctx, p.uid, writes); err != nil {
79
+
return 0, fmt.Errorf("batch write failed: %w", err)
80
+
}
81
+
82
+
return len(writes), nil
83
}
84
85
// GetLayerRecord retrieves a specific layer record by rkey