tjh.dev
forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2
appview: add label-value search support to models and indexers
Add LabelNameValues method to LabelState, returning composite
"name:value" strings for all labels with non-empty values.
Add LabelValues and NegatedLabelValues fields to IssueSearchOptions and
PullSearchOptions. Change NegatedAuthorDid from a single string to
NegatedAuthorDids []string to support excluding multiple authors.
Update both issue and pull indexers: bump mapping version to 3, add
label_values keyword field, populate it via LabelNameValues, and add
search clauses for the new fields.
Signed-off-by: Thomas Karpiniec <tom.karpiniec@outlook.com>
authored by
+31
-20
appview/indexer/issues/indexer.go
+31
-20
appview/indexer/issues/indexer.go
···
18
18
"github.com/blevesearch/bleve/v2/search/query"
19
19
"tangled.org/core/appview/db"
20
20
"tangled.org/core/appview/indexer/base36"
21
-
"tangled.org/core/appview/indexer/bleve"
21
+
bleveutil "tangled.org/core/appview/indexer/bleve"
22
22
"tangled.org/core/appview/models"
23
23
"tangled.org/core/appview/pagination"
24
24
tlog "tangled.org/core/log"
···
31
31
unicodeNormalizeName = "uicodeNormalize"
32
32
33
33
// Bump this when the index mapping changes to trigger a rebuild.
34
-
issueIndexerVersion = 2
34
+
issueIndexerVersion = 3
35
35
)
36
36
37
37
type Indexer struct {
···
89
89
docMapping.AddFieldMappingsAt("is_open", boolFieldMapping)
90
90
docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping)
91
91
docMapping.AddFieldMappingsAt("labels", keywordFieldMapping)
92
+
docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping)
92
93
93
94
err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{
94
95
"type": unicodenorm.Name,
···
184
183
}
185
184
186
185
type issueData struct {
187
-
ID int64 `json:"id"`
188
-
RepoAt string `json:"repo_at"`
189
-
IssueID int `json:"issue_id"`
190
-
Title string `json:"title"`
191
-
Body string `json:"body"`
192
-
IsOpen bool `json:"is_open"`
193
-
AuthorDid string `json:"author_did"`
194
-
Labels []string `json:"labels"`
186
+
ID int64 `json:"id"`
187
+
RepoAt string `json:"repo_at"`
188
+
IssueID int `json:"issue_id"`
189
+
Title string `json:"title"`
190
+
Body string `json:"body"`
191
+
IsOpen bool `json:"is_open"`
192
+
AuthorDid string `json:"author_did"`
193
+
Labels []string `json:"labels"`
194
+
LabelValues []string `json:"label_values"`
195
195
196
196
Comments []IssueCommentData `json:"comments"`
197
197
}
198
198
199
199
func makeIssueData(issue *models.Issue) *issueData {
200
200
return &issueData{
201
-
ID: issue.Id,
202
-
RepoAt: issue.RepoAt.String(),
203
-
IssueID: issue.IssueId,
204
-
Title: issue.Title,
205
-
Body: issue.Body,
206
-
IsOpen: issue.Open,
207
-
AuthorDid: issue.Did,
208
-
Labels: issue.Labels.LabelNames(),
201
+
ID: issue.Id,
202
+
RepoAt: issue.RepoAt.String(),
203
+
IssueID: issue.IssueId,
204
+
Title: issue.Title,
205
+
Body: issue.Body,
206
+
IsOpen: issue.Open,
207
+
AuthorDid: issue.Did,
208
+
Labels: issue.Labels.LabelNames(),
209
+
LabelValues: issue.Labels.LabelNameValues(),
209
210
}
210
211
}
211
212
···
287
284
musts = append(musts, bleveutil.KeywordFieldQuery("labels", label))
288
285
}
289
286
290
-
if opts.NegatedAuthorDid != "" {
291
-
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", opts.NegatedAuthorDid))
287
+
for _, did := range opts.NegatedAuthorDids {
288
+
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did))
292
289
}
293
290
294
291
for _, label := range opts.NegatedLabels {
295
292
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label))
293
+
}
294
+
295
+
for _, lv := range opts.LabelValues {
296
+
musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv))
297
+
}
298
+
299
+
for _, lv := range opts.NegatedLabelValues {
300
+
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv))
296
301
}
297
302
298
303
indexerQuery := bleve.NewBooleanQuery()
+84
-4
appview/indexer/issues/indexer_test.go
+84
-4
appview/indexer/issues/indexer_test.go
···
3
3
import (
4
4
"context"
5
5
"os"
6
+
"strings"
6
7
"testing"
7
8
8
9
"github.com/blevesearch/bleve/v2"
···
39
38
40
39
func boolPtr(b bool) *bool { return &b }
41
40
42
-
func makeLabelState(labels ...string) models.LabelState {
41
+
// makeLabelState creates a LabelState for testing. Each entry is either
42
+
// "name" (null-type label) or "name=val" (valued label).
43
+
func makeLabelState(entries ...string) models.LabelState {
43
44
state := models.NewLabelState()
44
-
for _, label := range labels {
45
-
state.Inner()[label] = make(map[string]struct{})
46
-
state.SetName(label, label)
45
+
for _, entry := range entries {
46
+
if eqIdx := strings.Index(entry, "="); eqIdx > 0 {
47
+
name := entry[:eqIdx]
48
+
val := entry[eqIdx+1:]
49
+
if state.Inner()[name] == nil {
50
+
state.Inner()[name] = make(map[string]struct{})
51
+
}
52
+
state.Inner()[name][val] = struct{}{}
53
+
state.SetName(name, name)
54
+
} else {
55
+
state.Inner()[entry] = make(map[string]struct{})
56
+
state.SetName(entry, entry)
57
+
}
47
58
}
48
59
return state
49
60
}
···
274
261
require.NoError(t, err)
275
262
assert.Equal(t, uint64(0), result.Total)
276
263
assert.Empty(t, result.Hits)
264
+
}
265
+
266
+
func TestSearchLabelValues(t *testing.T) {
267
+
ix, cleanup := setupTestIndexer(t)
268
+
defer cleanup()
269
+
270
+
ctx := context.Background()
271
+
272
+
err := ix.Index(ctx,
273
+
models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority bug", Body: "Urgent", Open: true, Did: "did:plc:alice",
274
+
Labels: makeLabelState("bug", "priority=high")},
275
+
models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Low priority feature", Body: "Nice to have", Open: true, Did: "did:plc:bob",
276
+
Labels: makeLabelState("feature", "priority=low")},
277
+
models.Issue{Id: 3, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority feature", Body: "Important", Open: true, Did: "did:plc:alice",
278
+
Labels: makeLabelState("feature", "priority=high")},
279
+
)
280
+
require.NoError(t, err)
281
+
282
+
opts := func() models.IssueSearchOptions {
283
+
return models.IssueSearchOptions{
284
+
RepoAt: "at://did:plc:test/sh.tangled.repo/abc",
285
+
IsOpen: boolPtr(true),
286
+
Page: pagination.Page{Limit: 10},
287
+
}
288
+
}
289
+
290
+
o := opts()
291
+
o.LabelValues = []string{"priority:high"}
292
+
result, err := ix.Search(ctx, o)
293
+
require.NoError(t, err)
294
+
assert.Equal(t, uint64(2), result.Total)
295
+
assert.Contains(t, result.Hits, int64(1))
296
+
assert.Contains(t, result.Hits, int64(3))
297
+
298
+
o = opts()
299
+
o.LabelValues = []string{"priority:low"}
300
+
result, err = ix.Search(ctx, o)
301
+
require.NoError(t, err)
302
+
assert.Equal(t, uint64(1), result.Total)
303
+
assert.Contains(t, result.Hits, int64(2))
304
+
305
+
// Combined: plain label + label value
306
+
o = opts()
307
+
o.Labels = []string{"feature"}
308
+
o.LabelValues = []string{"priority:high"}
309
+
result, err = ix.Search(ctx, o)
310
+
require.NoError(t, err)
311
+
assert.Equal(t, uint64(1), result.Total)
312
+
assert.Contains(t, result.Hits, int64(3))
313
+
314
+
// Negated label value
315
+
o = opts()
316
+
o.NegatedLabelValues = []string{"priority:low"}
317
+
result, err = ix.Search(ctx, o)
318
+
require.NoError(t, err)
319
+
assert.Equal(t, uint64(2), result.Total)
320
+
assert.Contains(t, result.Hits, int64(1))
321
+
assert.Contains(t, result.Hits, int64(3))
322
+
323
+
// Label value + negated plain label
324
+
o = opts()
325
+
o.LabelValues = []string{"priority:high"}
326
+
o.NegatedLabels = []string{"feature"}
327
+
result, err = ix.Search(ctx, o)
328
+
require.NoError(t, err)
329
+
assert.Equal(t, uint64(1), result.Total)
330
+
assert.Contains(t, result.Hits, int64(1))
277
331
}
+31
-20
appview/indexer/pulls/indexer.go
+31
-20
appview/indexer/pulls/indexer.go
···
18
18
"github.com/blevesearch/bleve/v2/search/query"
19
19
"tangled.org/core/appview/db"
20
20
"tangled.org/core/appview/indexer/base36"
21
-
"tangled.org/core/appview/indexer/bleve"
21
+
bleveutil "tangled.org/core/appview/indexer/bleve"
22
22
"tangled.org/core/appview/models"
23
23
tlog "tangled.org/core/log"
24
24
)
···
30
30
unicodeNormalizeName = "uicodeNormalize"
31
31
32
32
// Bump this when the index mapping changes to trigger a rebuild.
33
-
pullIndexerVersion = 2
33
+
pullIndexerVersion = 3
34
34
)
35
35
36
36
type Indexer struct {
···
84
84
docMapping.AddFieldMappingsAt("state", keywordFieldMapping)
85
85
docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping)
86
86
docMapping.AddFieldMappingsAt("labels", keywordFieldMapping)
87
+
docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping)
87
88
88
89
err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{
89
90
"type": unicodenorm.Name,
···
179
178
}
180
179
181
180
type pullData struct {
182
-
ID int64 `json:"id"`
183
-
RepoAt string `json:"repo_at"`
184
-
PullID int `json:"pull_id"`
185
-
Title string `json:"title"`
186
-
Body string `json:"body"`
187
-
State string `json:"state"`
188
-
AuthorDid string `json:"author_did"`
189
-
Labels []string `json:"labels"`
181
+
ID int64 `json:"id"`
182
+
RepoAt string `json:"repo_at"`
183
+
PullID int `json:"pull_id"`
184
+
Title string `json:"title"`
185
+
Body string `json:"body"`
186
+
State string `json:"state"`
187
+
AuthorDid string `json:"author_did"`
188
+
Labels []string `json:"labels"`
189
+
LabelValues []string `json:"label_values"`
190
190
191
191
Comments []pullCommentData `json:"comments"`
192
192
}
193
193
194
194
func makePullData(pull *models.Pull) *pullData {
195
195
return &pullData{
196
-
ID: int64(pull.ID),
197
-
RepoAt: pull.RepoAt.String(),
198
-
PullID: pull.PullId,
199
-
Title: pull.Title,
200
-
Body: pull.Body,
201
-
State: pull.State.String(),
202
-
AuthorDid: pull.OwnerDid,
203
-
Labels: pull.Labels.LabelNames(),
196
+
ID: int64(pull.ID),
197
+
RepoAt: pull.RepoAt.String(),
198
+
PullID: pull.PullId,
199
+
Title: pull.Title,
200
+
Body: pull.Body,
201
+
State: pull.State.String(),
202
+
AuthorDid: pull.OwnerDid,
203
+
Labels: pull.Labels.LabelNames(),
204
+
LabelValues: pull.Labels.LabelNameValues(),
204
205
}
205
206
}
206
207
···
288
285
musts = append(musts, bleveutil.KeywordFieldQuery("labels", label))
289
286
}
290
287
291
-
if opts.NegatedAuthorDid != "" {
292
-
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", opts.NegatedAuthorDid))
288
+
for _, did := range opts.NegatedAuthorDids {
289
+
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did))
293
290
}
294
291
295
292
for _, label := range opts.NegatedLabels {
296
293
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label))
294
+
}
295
+
296
+
for _, lv := range opts.LabelValues {
297
+
musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv))
298
+
}
299
+
300
+
for _, lv := range opts.NegatedLabelValues {
301
+
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv))
297
302
}
298
303
299
304
indexerQuery := bleve.NewBooleanQuery()
+21
appview/models/label.go
+21
appview/models/label.go
···
314
314
return result
315
315
}
316
316
317
+
// LabelNameValues returns composite "name:value" strings for all labels
318
+
// that have non-empty values.
319
+
func (s LabelState) LabelNameValues() []string {
320
+
var result []string
321
+
for key, valset := range s.inner {
322
+
if valset == nil {
323
+
continue
324
+
}
325
+
name, ok := s.names[key]
326
+
if !ok {
327
+
continue
328
+
}
329
+
for val := range valset {
330
+
if val != "" {
331
+
result = append(result, name+":"+val)
332
+
}
333
+
}
334
+
}
335
+
return result
336
+
}
337
+
317
338
func (s LabelState) Inner() map[string]set {
318
339
return s.inner
319
340
}
+28
-22
appview/models/search.go
+28
-22
appview/models/search.go
···
3
3
import "tangled.org/core/appview/pagination"
4
4
5
5
type IssueSearchOptions struct {
6
-
Keywords []string
7
-
Phrases []string
8
-
RepoAt string
9
-
IsOpen *bool
10
-
AuthorDid string
11
-
Labels []string
6
+
Keywords []string
7
+
Phrases []string
8
+
RepoAt string
9
+
IsOpen *bool
10
+
AuthorDid string
11
+
Labels []string
12
+
LabelValues []string
12
13
13
-
NegatedKeywords []string
14
-
NegatedPhrases []string
15
-
NegatedLabels []string
16
-
NegatedAuthorDid string
14
+
NegatedKeywords []string
15
+
NegatedPhrases []string
16
+
NegatedLabels []string
17
+
NegatedLabelValues []string
18
+
NegatedAuthorDids []string
17
19
18
20
Page pagination.Page
19
21
}
20
22
21
23
func (o *IssueSearchOptions) HasSearchFilters() bool {
22
24
return len(o.Keywords) > 0 || len(o.Phrases) > 0 ||
23
-
o.AuthorDid != "" || o.NegatedAuthorDid != "" ||
25
+
o.AuthorDid != "" || len(o.NegatedAuthorDids) > 0 ||
24
26
len(o.Labels) > 0 || len(o.NegatedLabels) > 0 ||
27
+
len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 ||
25
28
len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0
26
29
}
27
30
28
31
type PullSearchOptions struct {
29
-
Keywords []string
30
-
Phrases []string
31
-
RepoAt string
32
-
State *PullState
33
-
AuthorDid string
34
-
Labels []string
32
+
Keywords []string
33
+
Phrases []string
34
+
RepoAt string
35
+
State *PullState
36
+
AuthorDid string
37
+
Labels []string
38
+
LabelValues []string
35
39
36
-
NegatedKeywords []string
37
-
NegatedPhrases []string
38
-
NegatedLabels []string
39
-
NegatedAuthorDid string
40
+
NegatedKeywords []string
41
+
NegatedPhrases []string
42
+
NegatedLabels []string
43
+
NegatedLabelValues []string
44
+
NegatedAuthorDids []string
40
45
41
46
Page pagination.Page
42
47
}
43
48
44
49
func (o *PullSearchOptions) HasSearchFilters() bool {
45
50
return len(o.Keywords) > 0 || len(o.Phrases) > 0 ||
46
-
o.AuthorDid != "" || o.NegatedAuthorDid != "" ||
51
+
o.AuthorDid != "" || len(o.NegatedAuthorDids) > 0 ||
47
52
len(o.Labels) > 0 || len(o.NegatedLabels) > 0 ||
53
+
len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 ||
48
54
len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0
49
55
}