lewis.moe
Monorepo for Tangled
appview: add label-value search support to models and indexers
Add LabelNameValues method to LabelState, returning composite
"name:value" strings for all labels with non-empty values.
Add LabelValues and NegatedLabelValues fields to IssueSearchOptions and
PullSearchOptions. Change NegatedAuthorDid from a single string to
NegatedAuthorDids []string to support excluding multiple authors.
Update both issue and pull indexers: bump mapping version to 3, add
label_values keyword field, populate it via LabelNameValues, and add
search clauses for the new fields.
Signed-off-by: Thomas Karpiniec <tom.karpiniec@outlook.com>
authored by
Thomas Karpiniec
and committed by
+31
-20
appview/indexer/issues/indexer.go
+31
-20
appview/indexer/issues/indexer.go
···
18
"github.com/blevesearch/bleve/v2/search/query"
19
"tangled.org/core/appview/db"
20
"tangled.org/core/appview/indexer/base36"
21
-
"tangled.org/core/appview/indexer/bleve"
22
"tangled.org/core/appview/models"
23
"tangled.org/core/appview/pagination"
24
tlog "tangled.org/core/log"
···
31
unicodeNormalizeName = "uicodeNormalize"
32
33
// Bump this when the index mapping changes to trigger a rebuild.
34
-
issueIndexerVersion = 2
35
)
36
37
type Indexer struct {
···
89
docMapping.AddFieldMappingsAt("is_open", boolFieldMapping)
90
docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping)
91
docMapping.AddFieldMappingsAt("labels", keywordFieldMapping)
92
93
err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{
94
"type": unicodenorm.Name,
···
183
}
184
185
type issueData struct {
186
-
ID int64 `json:"id"`
187
-
RepoAt string `json:"repo_at"`
188
-
IssueID int `json:"issue_id"`
189
-
Title string `json:"title"`
190
-
Body string `json:"body"`
191
-
IsOpen bool `json:"is_open"`
192
-
AuthorDid string `json:"author_did"`
193
-
Labels []string `json:"labels"`
194
195
Comments []IssueCommentData `json:"comments"`
196
}
197
198
func makeIssueData(issue *models.Issue) *issueData {
199
return &issueData{
200
-
ID: issue.Id,
201
-
RepoAt: issue.RepoAt.String(),
202
-
IssueID: issue.IssueId,
203
-
Title: issue.Title,
204
-
Body: issue.Body,
205
-
IsOpen: issue.Open,
206
-
AuthorDid: issue.Did,
207
-
Labels: issue.Labels.LabelNames(),
208
}
209
}
210
···
284
musts = append(musts, bleveutil.KeywordFieldQuery("labels", label))
285
}
286
287
-
if opts.NegatedAuthorDid != "" {
288
-
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", opts.NegatedAuthorDid))
289
}
290
291
for _, label := range opts.NegatedLabels {
292
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label))
293
}
294
295
indexerQuery := bleve.NewBooleanQuery()
···
18
"github.com/blevesearch/bleve/v2/search/query"
19
"tangled.org/core/appview/db"
20
"tangled.org/core/appview/indexer/base36"
21
+
bleveutil "tangled.org/core/appview/indexer/bleve"
22
"tangled.org/core/appview/models"
23
"tangled.org/core/appview/pagination"
24
tlog "tangled.org/core/log"
···
31
unicodeNormalizeName = "uicodeNormalize"
32
33
// Bump this when the index mapping changes to trigger a rebuild.
34
+
issueIndexerVersion = 3
35
)
36
37
type Indexer struct {
···
89
docMapping.AddFieldMappingsAt("is_open", boolFieldMapping)
90
docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping)
91
docMapping.AddFieldMappingsAt("labels", keywordFieldMapping)
92
+
docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping)
93
94
err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{
95
"type": unicodenorm.Name,
···
184
}
185
186
type issueData struct {
187
+
ID int64 `json:"id"`
188
+
RepoAt string `json:"repo_at"`
189
+
IssueID int `json:"issue_id"`
190
+
Title string `json:"title"`
191
+
Body string `json:"body"`
192
+
IsOpen bool `json:"is_open"`
193
+
AuthorDid string `json:"author_did"`
194
+
Labels []string `json:"labels"`
195
+
LabelValues []string `json:"label_values"`
196
197
Comments []IssueCommentData `json:"comments"`
198
}
199
200
func makeIssueData(issue *models.Issue) *issueData {
201
return &issueData{
202
+
ID: issue.Id,
203
+
RepoAt: issue.RepoAt.String(),
204
+
IssueID: issue.IssueId,
205
+
Title: issue.Title,
206
+
Body: issue.Body,
207
+
IsOpen: issue.Open,
208
+
AuthorDid: issue.Did,
209
+
Labels: issue.Labels.LabelNames(),
210
+
LabelValues: issue.Labels.LabelNameValues(),
211
}
212
}
213
···
287
musts = append(musts, bleveutil.KeywordFieldQuery("labels", label))
288
}
289
290
+
for _, did := range opts.NegatedAuthorDids {
291
+
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did))
292
}
293
294
for _, label := range opts.NegatedLabels {
295
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label))
296
+
}
297
+
298
+
for _, lv := range opts.LabelValues {
299
+
musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv))
300
+
}
301
+
302
+
for _, lv := range opts.NegatedLabelValues {
303
+
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv))
304
}
305
306
indexerQuery := bleve.NewBooleanQuery()
+84
-4
appview/indexer/issues/indexer_test.go
+84
-4
appview/indexer/issues/indexer_test.go
···
3
import (
4
"context"
5
"os"
6
"testing"
7
8
"github.com/blevesearch/bleve/v2"
···
38
39
func boolPtr(b bool) *bool { return &b }
40
41
-
func makeLabelState(labels ...string) models.LabelState {
42
state := models.NewLabelState()
43
-
for _, label := range labels {
44
-
state.Inner()[label] = make(map[string]struct{})
45
-
state.SetName(label, label)
46
}
47
return state
48
}
···
262
assert.Equal(t, uint64(0), result.Total)
263
assert.Empty(t, result.Hits)
264
}
···
3
import (
4
"context"
5
"os"
6
+
"strings"
7
"testing"
8
9
"github.com/blevesearch/bleve/v2"
···
39
40
func boolPtr(b bool) *bool { return &b }
41
42
+
// makeLabelState creates a LabelState for testing. Each entry is either
43
+
// "name" (null-type label) or "name=val" (valued label).
44
+
func makeLabelState(entries ...string) models.LabelState {
45
state := models.NewLabelState()
46
+
for _, entry := range entries {
47
+
if eqIdx := strings.Index(entry, "="); eqIdx > 0 {
48
+
name := entry[:eqIdx]
49
+
val := entry[eqIdx+1:]
50
+
if state.Inner()[name] == nil {
51
+
state.Inner()[name] = make(map[string]struct{})
52
+
}
53
+
state.Inner()[name][val] = struct{}{}
54
+
state.SetName(name, name)
55
+
} else {
56
+
state.Inner()[entry] = make(map[string]struct{})
57
+
state.SetName(entry, entry)
58
+
}
59
}
60
return state
61
}
···
275
assert.Equal(t, uint64(0), result.Total)
276
assert.Empty(t, result.Hits)
277
}
278
+
279
+
func TestSearchLabelValues(t *testing.T) {
280
+
ix, cleanup := setupTestIndexer(t)
281
+
defer cleanup()
282
+
283
+
ctx := context.Background()
284
+
285
+
err := ix.Index(ctx,
286
+
models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority bug", Body: "Urgent", Open: true, Did: "did:plc:alice",
287
+
Labels: makeLabelState("bug", "priority=high")},
288
+
models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Low priority feature", Body: "Nice to have", Open: true, Did: "did:plc:bob",
289
+
Labels: makeLabelState("feature", "priority=low")},
290
+
models.Issue{Id: 3, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority feature", Body: "Important", Open: true, Did: "did:plc:alice",
291
+
Labels: makeLabelState("feature", "priority=high")},
292
+
)
293
+
require.NoError(t, err)
294
+
295
+
opts := func() models.IssueSearchOptions {
296
+
return models.IssueSearchOptions{
297
+
RepoAt: "at://did:plc:test/sh.tangled.repo/abc",
298
+
IsOpen: boolPtr(true),
299
+
Page: pagination.Page{Limit: 10},
300
+
}
301
+
}
302
+
303
+
o := opts()
304
+
o.LabelValues = []string{"priority:high"}
305
+
result, err := ix.Search(ctx, o)
306
+
require.NoError(t, err)
307
+
assert.Equal(t, uint64(2), result.Total)
308
+
assert.Contains(t, result.Hits, int64(1))
309
+
assert.Contains(t, result.Hits, int64(3))
310
+
311
+
o = opts()
312
+
o.LabelValues = []string{"priority:low"}
313
+
result, err = ix.Search(ctx, o)
314
+
require.NoError(t, err)
315
+
assert.Equal(t, uint64(1), result.Total)
316
+
assert.Contains(t, result.Hits, int64(2))
317
+
318
+
// Combined: plain label + label value
319
+
o = opts()
320
+
o.Labels = []string{"feature"}
321
+
o.LabelValues = []string{"priority:high"}
322
+
result, err = ix.Search(ctx, o)
323
+
require.NoError(t, err)
324
+
assert.Equal(t, uint64(1), result.Total)
325
+
assert.Contains(t, result.Hits, int64(3))
326
+
327
+
// Negated label value
328
+
o = opts()
329
+
o.NegatedLabelValues = []string{"priority:low"}
330
+
result, err = ix.Search(ctx, o)
331
+
require.NoError(t, err)
332
+
assert.Equal(t, uint64(2), result.Total)
333
+
assert.Contains(t, result.Hits, int64(1))
334
+
assert.Contains(t, result.Hits, int64(3))
335
+
336
+
// Label value + negated plain label
337
+
o = opts()
338
+
o.LabelValues = []string{"priority:high"}
339
+
o.NegatedLabels = []string{"feature"}
340
+
result, err = ix.Search(ctx, o)
341
+
require.NoError(t, err)
342
+
assert.Equal(t, uint64(1), result.Total)
343
+
assert.Contains(t, result.Hits, int64(1))
344
+
}
+31
-20
appview/indexer/pulls/indexer.go
+31
-20
appview/indexer/pulls/indexer.go
···
18
"github.com/blevesearch/bleve/v2/search/query"
19
"tangled.org/core/appview/db"
20
"tangled.org/core/appview/indexer/base36"
21
-
"tangled.org/core/appview/indexer/bleve"
22
"tangled.org/core/appview/models"
23
tlog "tangled.org/core/log"
24
)
···
30
unicodeNormalizeName = "uicodeNormalize"
31
32
// Bump this when the index mapping changes to trigger a rebuild.
33
-
pullIndexerVersion = 2
34
)
35
36
type Indexer struct {
···
84
docMapping.AddFieldMappingsAt("state", keywordFieldMapping)
85
docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping)
86
docMapping.AddFieldMappingsAt("labels", keywordFieldMapping)
87
88
err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{
89
"type": unicodenorm.Name,
···
178
}
179
180
type pullData struct {
181
-
ID int64 `json:"id"`
182
-
RepoAt string `json:"repo_at"`
183
-
PullID int `json:"pull_id"`
184
-
Title string `json:"title"`
185
-
Body string `json:"body"`
186
-
State string `json:"state"`
187
-
AuthorDid string `json:"author_did"`
188
-
Labels []string `json:"labels"`
189
190
Comments []pullCommentData `json:"comments"`
191
}
192
193
func makePullData(pull *models.Pull) *pullData {
194
return &pullData{
195
-
ID: int64(pull.ID),
196
-
RepoAt: pull.RepoAt.String(),
197
-
PullID: pull.PullId,
198
-
Title: pull.Title,
199
-
Body: pull.Body,
200
-
State: pull.State.String(),
201
-
AuthorDid: pull.OwnerDid,
202
-
Labels: pull.Labels.LabelNames(),
203
}
204
}
205
···
285
musts = append(musts, bleveutil.KeywordFieldQuery("labels", label))
286
}
287
288
-
if opts.NegatedAuthorDid != "" {
289
-
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", opts.NegatedAuthorDid))
290
}
291
292
for _, label := range opts.NegatedLabels {
293
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label))
294
}
295
296
indexerQuery := bleve.NewBooleanQuery()
···
18
"github.com/blevesearch/bleve/v2/search/query"
19
"tangled.org/core/appview/db"
20
"tangled.org/core/appview/indexer/base36"
21
+
bleveutil "tangled.org/core/appview/indexer/bleve"
22
"tangled.org/core/appview/models"
23
tlog "tangled.org/core/log"
24
)
···
30
unicodeNormalizeName = "uicodeNormalize"
31
32
// Bump this when the index mapping changes to trigger a rebuild.
33
+
pullIndexerVersion = 3
34
)
35
36
type Indexer struct {
···
84
docMapping.AddFieldMappingsAt("state", keywordFieldMapping)
85
docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping)
86
docMapping.AddFieldMappingsAt("labels", keywordFieldMapping)
87
+
docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping)
88
89
err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{
90
"type": unicodenorm.Name,
···
179
}
180
181
type pullData struct {
182
+
ID int64 `json:"id"`
183
+
RepoAt string `json:"repo_at"`
184
+
PullID int `json:"pull_id"`
185
+
Title string `json:"title"`
186
+
Body string `json:"body"`
187
+
State string `json:"state"`
188
+
AuthorDid string `json:"author_did"`
189
+
Labels []string `json:"labels"`
190
+
LabelValues []string `json:"label_values"`
191
192
Comments []pullCommentData `json:"comments"`
193
}
194
195
func makePullData(pull *models.Pull) *pullData {
196
return &pullData{
197
+
ID: int64(pull.ID),
198
+
RepoAt: pull.RepoAt.String(),
199
+
PullID: pull.PullId,
200
+
Title: pull.Title,
201
+
Body: pull.Body,
202
+
State: pull.State.String(),
203
+
AuthorDid: pull.OwnerDid,
204
+
Labels: pull.Labels.LabelNames(),
205
+
LabelValues: pull.Labels.LabelNameValues(),
206
}
207
}
208
···
288
musts = append(musts, bleveutil.KeywordFieldQuery("labels", label))
289
}
290
291
+
for _, did := range opts.NegatedAuthorDids {
292
+
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did))
293
}
294
295
for _, label := range opts.NegatedLabels {
296
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label))
297
+
}
298
+
299
+
for _, lv := range opts.LabelValues {
300
+
musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv))
301
+
}
302
+
303
+
for _, lv := range opts.NegatedLabelValues {
304
+
mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv))
305
}
306
307
indexerQuery := bleve.NewBooleanQuery()
+21
appview/models/label.go
+21
appview/models/label.go
···
314
return result
315
}
316
317
+
// LabelNameValues returns composite "name:value" strings for all labels
318
+
// that have non-empty values.
319
+
func (s LabelState) LabelNameValues() []string {
320
+
var result []string
321
+
for key, valset := range s.inner {
322
+
if valset == nil {
323
+
continue
324
+
}
325
+
name, ok := s.names[key]
326
+
if !ok {
327
+
continue
328
+
}
329
+
for val := range valset {
330
+
if val != "" {
331
+
result = append(result, name+":"+val)
332
+
}
333
+
}
334
+
}
335
+
return result
336
+
}
337
+
338
func (s LabelState) Inner() map[string]set {
339
return s.inner
340
}
+28
-22
appview/models/search.go
+28
-22
appview/models/search.go
···
3
import "tangled.org/core/appview/pagination"
4
5
type IssueSearchOptions struct {
6
-
Keywords []string
7
-
Phrases []string
8
-
RepoAt string
9
-
IsOpen *bool
10
-
AuthorDid string
11
-
Labels []string
12
13
-
NegatedKeywords []string
14
-
NegatedPhrases []string
15
-
NegatedLabels []string
16
-
NegatedAuthorDid string
17
18
Page pagination.Page
19
}
20
21
func (o *IssueSearchOptions) HasSearchFilters() bool {
22
return len(o.Keywords) > 0 || len(o.Phrases) > 0 ||
23
-
o.AuthorDid != "" || o.NegatedAuthorDid != "" ||
24
len(o.Labels) > 0 || len(o.NegatedLabels) > 0 ||
25
len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0
26
}
27
28
type PullSearchOptions struct {
29
-
Keywords []string
30
-
Phrases []string
31
-
RepoAt string
32
-
State *PullState
33
-
AuthorDid string
34
-
Labels []string
35
36
-
NegatedKeywords []string
37
-
NegatedPhrases []string
38
-
NegatedLabels []string
39
-
NegatedAuthorDid string
40
41
Page pagination.Page
42
}
43
44
func (o *PullSearchOptions) HasSearchFilters() bool {
45
return len(o.Keywords) > 0 || len(o.Phrases) > 0 ||
46
-
o.AuthorDid != "" || o.NegatedAuthorDid != "" ||
47
len(o.Labels) > 0 || len(o.NegatedLabels) > 0 ||
48
len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0
49
}
···
3
import "tangled.org/core/appview/pagination"
4
5
type IssueSearchOptions struct {
6
+
Keywords []string
7
+
Phrases []string
8
+
RepoAt string
9
+
IsOpen *bool
10
+
AuthorDid string
11
+
Labels []string
12
+
LabelValues []string
13
14
+
NegatedKeywords []string
15
+
NegatedPhrases []string
16
+
NegatedLabels []string
17
+
NegatedLabelValues []string
18
+
NegatedAuthorDids []string
19
20
Page pagination.Page
21
}
22
23
func (o *IssueSearchOptions) HasSearchFilters() bool {
24
return len(o.Keywords) > 0 || len(o.Phrases) > 0 ||
25
+
o.AuthorDid != "" || len(o.NegatedAuthorDids) > 0 ||
26
len(o.Labels) > 0 || len(o.NegatedLabels) > 0 ||
27
+
len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 ||
28
len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0
29
}
30
31
type PullSearchOptions struct {
32
+
Keywords []string
33
+
Phrases []string
34
+
RepoAt string
35
+
State *PullState
36
+
AuthorDid string
37
+
Labels []string
38
+
LabelValues []string
39
40
+
NegatedKeywords []string
41
+
NegatedPhrases []string
42
+
NegatedLabels []string
43
+
NegatedLabelValues []string
44
+
NegatedAuthorDids []string
45
46
Page pagination.Page
47
}
48
49
func (o *PullSearchOptions) HasSearchFilters() bool {
50
return len(o.Keywords) > 0 || len(o.Phrases) > 0 ||
51
+
o.AuthorDid != "" || len(o.NegatedAuthorDids) > 0 ||
52
len(o.Labels) > 0 || len(o.NegatedLabels) > 0 ||
53
+
len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 ||
54
len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0
55
}