lewis.moe / tangled-core
Monorepo for Tangled
fork atom

appview: add label-value search support to models and indexers

Add LabelNameValues method to LabelState, returning composite
"name:value" strings for all labels with non-empty values.

Add LabelValues and NegatedLabelValues fields to IssueSearchOptions and
PullSearchOptions. Change NegatedAuthorDid from a single string to
NegatedAuthorDids []string to support excluding multiple authors.

Update both issue and pull indexers: bump mapping version to 3, add
label_values keyword field, populate it via LabelNameValues, and add
search clauses for the new fields.

Signed-off-by: Thomas Karpiniec <tom.karpiniec@outlook.com>

authored by

Thomas Karpiniec and committed by tangled.org 44086392 91350966

+195 -66
unified split
+31 -20
appview/indexer/issues/indexer.go
··· 18 18 "github.com/blevesearch/bleve/v2/search/query" 19 19 "tangled.org/core/appview/db" 20 20 "tangled.org/core/appview/indexer/base36" 21 - "tangled.org/core/appview/indexer/bleve" 21 + bleveutil "tangled.org/core/appview/indexer/bleve" 22 22 "tangled.org/core/appview/models" 23 23 "tangled.org/core/appview/pagination" 24 24 tlog "tangled.org/core/log" ··· 31 31 unicodeNormalizeName = "uicodeNormalize" 32 32 33 33 // Bump this when the index mapping changes to trigger a rebuild. 34 - issueIndexerVersion = 2 34 + issueIndexerVersion = 3 35 35 ) 36 36 37 37 type Indexer struct { ··· 89 89 docMapping.AddFieldMappingsAt("is_open", boolFieldMapping) 90 90 docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping) 91 91 docMapping.AddFieldMappingsAt("labels", keywordFieldMapping) 92 + docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping) 92 93 93 94 err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{ 94 95 "type": unicodenorm.Name, ··· 183 184 } 184 185 185 186 type issueData struct { 186 - ID int64 `json:"id"` 187 - RepoAt string `json:"repo_at"` 188 - IssueID int `json:"issue_id"` 189 - Title string `json:"title"` 190 - Body string `json:"body"` 191 - IsOpen bool `json:"is_open"` 192 - AuthorDid string `json:"author_did"` 193 - Labels []string `json:"labels"` 187 + ID int64 `json:"id"` 188 + RepoAt string `json:"repo_at"` 189 + IssueID int `json:"issue_id"` 190 + Title string `json:"title"` 191 + Body string `json:"body"` 192 + IsOpen bool `json:"is_open"` 193 + AuthorDid string `json:"author_did"` 194 + Labels []string `json:"labels"` 195 + LabelValues []string `json:"label_values"` 194 196 195 197 Comments []IssueCommentData `json:"comments"` 196 198 } 197 199 198 200 func makeIssueData(issue *models.Issue) *issueData { 199 201 return &issueData{ 200 - ID: issue.Id, 201 - RepoAt: issue.RepoAt.String(), 202 - IssueID: issue.IssueId, 203 - Title: issue.Title, 204 - Body: issue.Body, 205 - IsOpen: issue.Open, 206 - AuthorDid: issue.Did, 207 - Labels: issue.Labels.LabelNames(), 202 + ID: issue.Id, 203 + RepoAt: issue.RepoAt.String(), 204 + IssueID: issue.IssueId, 205 + Title: issue.Title, 206 + Body: issue.Body, 207 + IsOpen: issue.Open, 208 + AuthorDid: issue.Did, 209 + Labels: issue.Labels.LabelNames(), 210 + LabelValues: issue.Labels.LabelNameValues(), 208 211 } 209 212 } 210 213 ··· 284 287 musts = append(musts, bleveutil.KeywordFieldQuery("labels", label)) 285 288 } 286 289 287 - if opts.NegatedAuthorDid != "" { 288 - mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", opts.NegatedAuthorDid)) 290 + for _, did := range opts.NegatedAuthorDids { 291 + mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did)) 289 292 } 290 293 291 294 for _, label := range opts.NegatedLabels { 292 295 mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label)) 296 + } 297 + 298 + for _, lv := range opts.LabelValues { 299 + musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv)) 300 + } 301 + 302 + for _, lv := range opts.NegatedLabelValues { 303 + mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv)) 293 304 } 294 305 295 306 indexerQuery := bleve.NewBooleanQuery()
+84 -4
appview/indexer/issues/indexer_test.go
··· 3 3 import ( 4 4 "context" 5 5 "os" 6 + "strings" 6 7 "testing" 7 8 8 9 "github.com/blevesearch/bleve/v2" ··· 38 39 39 40 func boolPtr(b bool) *bool { return &b } 40 41 41 - func makeLabelState(labels ...string) models.LabelState { 42 + // makeLabelState creates a LabelState for testing. Each entry is either 43 + // "name" (null-type label) or "name=val" (valued label). 44 + func makeLabelState(entries ...string) models.LabelState { 42 45 state := models.NewLabelState() 43 - for _, label := range labels { 44 - state.Inner()[label] = make(map[string]struct{}) 45 - state.SetName(label, label) 46 + for _, entry := range entries { 47 + if eqIdx := strings.Index(entry, "="); eqIdx > 0 { 48 + name := entry[:eqIdx] 49 + val := entry[eqIdx+1:] 50 + if state.Inner()[name] == nil { 51 + state.Inner()[name] = make(map[string]struct{}) 52 + } 53 + state.Inner()[name][val] = struct{}{} 54 + state.SetName(name, name) 55 + } else { 56 + state.Inner()[entry] = make(map[string]struct{}) 57 + state.SetName(entry, entry) 58 + } 46 59 } 47 60 return state 48 61 } ··· 262 275 assert.Equal(t, uint64(0), result.Total) 263 276 assert.Empty(t, result.Hits) 264 277 } 278 + 279 + func TestSearchLabelValues(t *testing.T) { 280 + ix, cleanup := setupTestIndexer(t) 281 + defer cleanup() 282 + 283 + ctx := context.Background() 284 + 285 + err := ix.Index(ctx, 286 + models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority bug", Body: "Urgent", Open: true, Did: "did:plc:alice", 287 + Labels: makeLabelState("bug", "priority=high")}, 288 + models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Low priority feature", Body: "Nice to have", Open: true, Did: "did:plc:bob", 289 + Labels: makeLabelState("feature", "priority=low")}, 290 + models.Issue{Id: 3, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority feature", Body: "Important", Open: true, Did: "did:plc:alice", 291 + Labels: makeLabelState("feature", "priority=high")}, 292 + ) 293 + require.NoError(t, err) 294 + 295 + opts := func() models.IssueSearchOptions { 296 + return models.IssueSearchOptions{ 297 + RepoAt: "at://did:plc:test/sh.tangled.repo/abc", 298 + IsOpen: boolPtr(true), 299 + Page: pagination.Page{Limit: 10}, 300 + } 301 + } 302 + 303 + o := opts() 304 + o.LabelValues = []string{"priority:high"} 305 + result, err := ix.Search(ctx, o) 306 + require.NoError(t, err) 307 + assert.Equal(t, uint64(2), result.Total) 308 + assert.Contains(t, result.Hits, int64(1)) 309 + assert.Contains(t, result.Hits, int64(3)) 310 + 311 + o = opts() 312 + o.LabelValues = []string{"priority:low"} 313 + result, err = ix.Search(ctx, o) 314 + require.NoError(t, err) 315 + assert.Equal(t, uint64(1), result.Total) 316 + assert.Contains(t, result.Hits, int64(2)) 317 + 318 + // Combined: plain label + label value 319 + o = opts() 320 + o.Labels = []string{"feature"} 321 + o.LabelValues = []string{"priority:high"} 322 + result, err = ix.Search(ctx, o) 323 + require.NoError(t, err) 324 + assert.Equal(t, uint64(1), result.Total) 325 + assert.Contains(t, result.Hits, int64(3)) 326 + 327 + // Negated label value 328 + o = opts() 329 + o.NegatedLabelValues = []string{"priority:low"} 330 + result, err = ix.Search(ctx, o) 331 + require.NoError(t, err) 332 + assert.Equal(t, uint64(2), result.Total) 333 + assert.Contains(t, result.Hits, int64(1)) 334 + assert.Contains(t, result.Hits, int64(3)) 335 + 336 + // Label value + negated plain label 337 + o = opts() 338 + o.LabelValues = []string{"priority:high"} 339 + o.NegatedLabels = []string{"feature"} 340 + result, err = ix.Search(ctx, o) 341 + require.NoError(t, err) 342 + assert.Equal(t, uint64(1), result.Total) 343 + assert.Contains(t, result.Hits, int64(1)) 344 + }
+31 -20
appview/indexer/pulls/indexer.go
··· 18 18 "github.com/blevesearch/bleve/v2/search/query" 19 19 "tangled.org/core/appview/db" 20 20 "tangled.org/core/appview/indexer/base36" 21 - "tangled.org/core/appview/indexer/bleve" 21 + bleveutil "tangled.org/core/appview/indexer/bleve" 22 22 "tangled.org/core/appview/models" 23 23 tlog "tangled.org/core/log" 24 24 ) ··· 30 30 unicodeNormalizeName = "uicodeNormalize" 31 31 32 32 // Bump this when the index mapping changes to trigger a rebuild. 33 - pullIndexerVersion = 2 33 + pullIndexerVersion = 3 34 34 ) 35 35 36 36 type Indexer struct { ··· 84 84 docMapping.AddFieldMappingsAt("state", keywordFieldMapping) 85 85 docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping) 86 86 docMapping.AddFieldMappingsAt("labels", keywordFieldMapping) 87 + docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping) 87 88 88 89 err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{ 89 90 "type": unicodenorm.Name, ··· 178 179 } 179 180 180 181 type pullData struct { 181 - ID int64 `json:"id"` 182 - RepoAt string `json:"repo_at"` 183 - PullID int `json:"pull_id"` 184 - Title string `json:"title"` 185 - Body string `json:"body"` 186 - State string `json:"state"` 187 - AuthorDid string `json:"author_did"` 188 - Labels []string `json:"labels"` 182 + ID int64 `json:"id"` 183 + RepoAt string `json:"repo_at"` 184 + PullID int `json:"pull_id"` 185 + Title string `json:"title"` 186 + Body string `json:"body"` 187 + State string `json:"state"` 188 + AuthorDid string `json:"author_did"` 189 + Labels []string `json:"labels"` 190 + LabelValues []string `json:"label_values"` 189 191 190 192 Comments []pullCommentData `json:"comments"` 191 193 } 192 194 193 195 func makePullData(pull *models.Pull) *pullData { 194 196 return &pullData{ 195 - ID: int64(pull.ID), 196 - RepoAt: pull.RepoAt.String(), 197 - PullID: pull.PullId, 198 - Title: pull.Title, 199 - Body: pull.Body, 200 - State: pull.State.String(), 201 - AuthorDid: pull.OwnerDid, 202 - Labels: pull.Labels.LabelNames(), 197 + ID: int64(pull.ID), 198 + RepoAt: pull.RepoAt.String(), 199 + PullID: pull.PullId, 200 + Title: pull.Title, 201 + Body: pull.Body, 202 + State: pull.State.String(), 203 + AuthorDid: pull.OwnerDid, 204 + Labels: pull.Labels.LabelNames(), 205 + LabelValues: pull.Labels.LabelNameValues(), 203 206 } 204 207 } 205 208 ··· 285 288 musts = append(musts, bleveutil.KeywordFieldQuery("labels", label)) 286 289 } 287 290 288 - if opts.NegatedAuthorDid != "" { 289 - mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", opts.NegatedAuthorDid)) 291 + for _, did := range opts.NegatedAuthorDids { 292 + mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did)) 290 293 } 291 294 292 295 for _, label := range opts.NegatedLabels { 293 296 mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label)) 297 + } 298 + 299 + for _, lv := range opts.LabelValues { 300 + musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv)) 301 + } 302 + 303 + for _, lv := range opts.NegatedLabelValues { 304 + mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv)) 294 305 } 295 306 296 307 indexerQuery := bleve.NewBooleanQuery()
+21
appview/models/label.go
··· 314 314 return result 315 315 } 316 316 317 + // LabelNameValues returns composite "name:value" strings for all labels 318 + // that have non-empty values. 319 + func (s LabelState) LabelNameValues() []string { 320 + var result []string 321 + for key, valset := range s.inner { 322 + if valset == nil { 323 + continue 324 + } 325 + name, ok := s.names[key] 326 + if !ok { 327 + continue 328 + } 329 + for val := range valset { 330 + if val != "" { 331 + result = append(result, name+":"+val) 332 + } 333 + } 334 + } 335 + return result 336 + } 337 + 317 338 func (s LabelState) Inner() map[string]set { 318 339 return s.inner 319 340 }
+28 -22
appview/models/search.go
··· 3 3 import "tangled.org/core/appview/pagination" 4 4 5 5 type IssueSearchOptions struct { 6 - Keywords []string 7 - Phrases []string 8 - RepoAt string 9 - IsOpen *bool 10 - AuthorDid string 11 - Labels []string 6 + Keywords []string 7 + Phrases []string 8 + RepoAt string 9 + IsOpen *bool 10 + AuthorDid string 11 + Labels []string 12 + LabelValues []string 12 13 13 - NegatedKeywords []string 14 - NegatedPhrases []string 15 - NegatedLabels []string 16 - NegatedAuthorDid string 14 + NegatedKeywords []string 15 + NegatedPhrases []string 16 + NegatedLabels []string 17 + NegatedLabelValues []string 18 + NegatedAuthorDids []string 17 19 18 20 Page pagination.Page 19 21 } 20 22 21 23 func (o *IssueSearchOptions) HasSearchFilters() bool { 22 24 return len(o.Keywords) > 0 || len(o.Phrases) > 0 || 23 - o.AuthorDid != "" || o.NegatedAuthorDid != "" || 25 + o.AuthorDid != "" || len(o.NegatedAuthorDids) > 0 || 24 26 len(o.Labels) > 0 || len(o.NegatedLabels) > 0 || 27 + len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 || 25 28 len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0 26 29 } 27 30 28 31 type PullSearchOptions struct { 29 - Keywords []string 30 - Phrases []string 31 - RepoAt string 32 - State *PullState 33 - AuthorDid string 34 - Labels []string 32 + Keywords []string 33 + Phrases []string 34 + RepoAt string 35 + State *PullState 36 + AuthorDid string 37 + Labels []string 38 + LabelValues []string 35 39 36 - NegatedKeywords []string 37 - NegatedPhrases []string 38 - NegatedLabels []string 39 - NegatedAuthorDid string 40 + NegatedKeywords []string 41 + NegatedPhrases []string 42 + NegatedLabels []string 43 + NegatedLabelValues []string 44 + NegatedAuthorDids []string 40 45 41 46 Page pagination.Page 42 47 } 43 48 44 49 func (o *PullSearchOptions) HasSearchFilters() bool { 45 50 return len(o.Keywords) > 0 || len(o.Phrases) > 0 || 46 - o.AuthorDid != "" || o.NegatedAuthorDid != "" || 51 + o.AuthorDid != "" || len(o.NegatedAuthorDids) > 0 || 47 52 len(o.Labels) > 0 || len(o.NegatedLabels) > 0 || 53 + len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 || 48 54 len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0 49 55 }