tangled.org / core
Monorepo for Tangled tangled.org
fork atom

appview: add basic issue indexer (wip) #494

merged opened by boltless.me targeting master from boltless.me/core: feat/search
  • Heavily inspired by gitea
  • add GetAllIssues which only receives a paginator and gathers all issues ignoring repoAt field

Signed-off-by: Seongmin Lee boltlessengineer@proton.me

STACK 13
#711 appview/notify: merge NewPullClosed/Merged/Reopen to NewPullState
1
#6
#710 appview: add pull_reopen event
0
#6
#709 appview/pulls: add search endpoint
0
#6
#708 appview/indexer: add pulls indexer
0
#6
#707 appview: add issue_reopen event
0
#6
#706 appview/indexer: put bleve index dir under indexes/ dir
0
#6
#693 appview/notify: pass logger with mergedLogger
2
#9
#695 appview/indexer: handle NewIssueClosed
1
#9
#694 appview/notify: add DeleteIssue event
1
#9
#496 appview: add issue search endpoint
5
#13
#495 wip: add indexer mappings
1
#14
#494 appview: add basic issue indexer (wip)
2
#14
#685 go.mod: add bleve as a dependency
1
#10
Labels

None yet.

assignee

None yet.

Participants 2
AT URI
at://did:plc:xasnlahkri4ewmbuzly2rlc5/sh.tangled.repo.pull/3lwecnvoz4z22
+35 -32
Interdiff #4 → #5
unified split
.gitignore

This file has not been changed.

appview/indexer/base36/base36.go

This file has not been changed.

appview/indexer/bleve/batch.go

This file has not been changed.

appview/indexer/indexer.go

This file has not been changed.

+27 -19
appview/indexer/issues/indexer.go
··· 1 package issues_indexer 2 3 import ( ··· 90 return db.GetIssuesPaginated(e, page) 91 }, 92 func(issues []models.Issue) error { 93 - var dataList []*IssueData 94 - for _, issue := range issues { 95 - dataList = append(dataList, &IssueData{ 96 - ID: issue.Id, 97 - IssueID: issue.IssueId, 98 - Title: issue.Title, 99 - Body: issue.Body, 100 - IsOpen: issue.Open, 101 - }) 102 - err := ix.Index(ctx, dataList...) 103 - if err != nil { 104 - return err 105 - } 106 - } 107 - return nil 108 }, 109 ) 110 l.Info("issues indexed", "count", count) 111 return err 112 } 113 114 - // IssueData data stored and will be indexed 115 - type IssueData struct { 116 ID int64 `json:"id"` 117 IssueID int `json:"issue_id"` 118 Title string `json:"title"` 119 Body string `json:"body"` ··· 122 Comments []IssueCommentData `json:"comments"` 123 } 124 125 type IssueCommentData struct { 126 Body string `json:"body"` 127 } ··· 133 134 const maxBatchSize = 20 135 136 - func (ix *Indexer) Index(ctx context.Context, issues ...*IssueData) error { 137 batch := bleveutil.NewFlushingBatch(ix.indexer, maxBatchSize) 138 for _, issue := range issues { 139 - if err := batch.Index(base36.Encode(issue.ID), issue); err != nil { 140 return err 141 } 142 } ··· 153 matchAndQuery(opts.Keyword, "body"), 154 )) 155 } 156 queries = append(queries, boolFieldQuery(opts.IsOpen, "is_open")) 157 // TODO: append more queries 158 ··· 184 185 func boolFieldQuery(val bool, field string) query.Query { 186 q := bleve.NewBoolFieldQuery(val) 187 q.FieldVal = field 188 return q 189 }
··· 1 + // heavily inspired by gitea's model (basically copy-pasted) 2 package issues_indexer 3 4 import ( ··· 91 return db.GetIssuesPaginated(e, page) 92 }, 93 func(issues []models.Issue) error { 94 + count += len(issues) 95 + return ix.Index(ctx, issues...) 96 }, 97 ) 98 l.Info("issues indexed", "count", count) 99 return err 100 } 101 102 + // issueData data stored and will be indexed 103 + type issueData struct { 104 ID int64 `json:"id"` 105 + RepoAt string `json:"repo_at"` 106 IssueID int `json:"issue_id"` 107 Title string `json:"title"` 108 Body string `json:"body"` ··· 111 Comments []IssueCommentData `json:"comments"` 112 } 113 114 + func makeIssueData(issue *models.Issue) issueData { 115 + return issueData{ 116 + ID: issue.Id, 117 + RepoAt: issue.RepoAt.String(), 118 + IssueID: issue.IssueId, 119 + Title: issue.Title, 120 + Body: issue.Body, 121 + IsOpen: issue.Open, 122 + } 123 + } 124 + 125 type IssueCommentData struct { 126 Body string `json:"body"` 127 } ··· 133 134 const maxBatchSize = 20 135 136 + func (ix *Indexer) Index(ctx context.Context, issues ...models.Issue) error { 137 batch := bleveutil.NewFlushingBatch(ix.indexer, maxBatchSize) 138 for _, issue := range issues { 139 + issueData := makeIssueData(&issue) 140 + if err := batch.Index(base36.Encode(issue.Id), issueData); err != nil { 141 return err 142 } 143 } ··· 154 matchAndQuery(opts.Keyword, "body"), 155 )) 156 } 157 + queries = append(queries, keywordFieldQuery(opts.RepoAt, "repo_at")) 158 queries = append(queries, boolFieldQuery(opts.IsOpen, "is_open")) 159 // TODO: append more queries 160 ··· 186 187 func boolFieldQuery(val bool, field string) query.Query { 188 q := bleve.NewBoolFieldQuery(val) 189 + q.FieldVal = field 190 + return q 191 + } 192 + 193 + func keywordFieldQuery(keyword, field string) query.Query { 194 + q := bleve.NewTermQuery(keyword) 195 q.FieldVal = field 196 return q 197 }
+7 -13
appview/indexer/notifier.go
··· 3 import ( 4 "context" 5 6 - issues_indexer "tangled.org/core/appview/indexer/issues" 7 "tangled.org/core/appview/models" 8 "tangled.org/core/appview/notify" 9 ) 10 11 var _ notify.Notifier = &Indexer{} 12 13 func (ix *Indexer) NewIssue(ctx context.Context, issue *models.Issue) { 14 - ix.Issues.Index(ctx, &issues_indexer.IssueData{ 15 - ID: issue.Id, 16 - IssueID: issue.IssueId, 17 - Title: issue.Title, 18 - Body: issue.Body, 19 - IsOpen: issue.Open, 20 - Comments: []issues_indexer.IssueCommentData{}, 21 - }) 22 - } 23 - 24 - func (ix *Indexer) NewPullComment(ctx context.Context, comment *models.PullComment) { 25 - panic("unimplemented") 26 }
··· 3 import ( 4 "context" 5 6 "tangled.org/core/appview/models" 7 "tangled.org/core/appview/notify" 8 + "tangled.org/core/log" 9 ) 10 11 var _ notify.Notifier = &Indexer{} 12 13 func (ix *Indexer) NewIssue(ctx context.Context, issue *models.Issue) { 14 + l := log.FromContext(ctx).With("notifier", "indexer.NewIssue", "issue", issue) 15 + l.Debug("indexing new issue") 16 + err := ix.Issues.Index(ctx, *issue) 17 + if err != nil { 18 + l.Error("failed to index an issue", "err", err) 19 + } 20 }
+1
appview/issues/issues.go
··· 851 Rkey: tid.TID(), 852 Title: r.FormValue("title"), 853 Body: r.FormValue("body"), 854 Did: user.Did, 855 Created: time.Now(), 856 Repo: &f.Repo,
··· 851 Rkey: tid.TID(), 852 Title: r.FormValue("title"), 853 Body: r.FormValue("body"), 854 + Open: true, 855 Did: user.Did, 856 Created: time.Now(), 857 Repo: &f.Repo,
appview/models/search.go

This file has not been changed.

appview/pages/pages.go

This file has not been changed.

appview/pagination/page.go

This file has not been changed.

appview/state/router.go

This file has not been changed.

appview/state/state.go

This file has not been changed.

History

15 rounds 2 comments
sign up or login to add to the discussion
boltless.me submitted #14
diff interdiff
1 commit
expand
bb301662
appview: add basic issue indexer
expand 0 comments
pull request successfully merged
boltless.me submitted #13
diff interdiff
1 commit
expand
bb301662
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #12
diff interdiff
1 commit
expand
47b55ade
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #11
diff interdiff
1 commit
expand
b335f72a
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #10
diff interdiff
1 commit
expand
429a2f09
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #9
diff interdiff
1 commit
expand
429a2f09
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #8
diff interdiff
1 commit
expand
429a2f09
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #7
diff interdiff
1 commit
expand
3e70c814
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #6
diff interdiff
1 commit
expand
3e70c814
appview: add basic issue indexer
expand 1 comment
oppi.li

this changeset largely lgtm!

boltless.me submitted #5
diff
1 commit
expand
b9a6ec9d
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #4
diff interdiff
1 commit
expand
66f42aa9
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #3
diff interdiff
1 commit
expand
e1d4d340
appview: add basic issue indexer
expand 1 comment
oppi.li
  • this is an N+1 query, because we first get all issues and then make one query for each issue. understandably this only needs to happen when populating the indices but we could improve this by just making one query
  • we can use a pagination free API for getting issues with comments while populating indexer
  • more of a note to self: search index should also handle issue deletions
boltless.me submitted #2
diff interdiff
1 commit
expand
c83d9601
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #1
diff interdiff
1 commit
expand
af090ec1
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #0
diff
1 commit
expand
31b2eff1
appview: add basic issue indexer (wip)
expand 0 comments