tangled.org / core
Monorepo for Tangled tangled.org
fork atom

appview: add basic issue indexer (wip) #494

merged opened by boltless.me targeting master from boltless.me/core: feat/search
  • Heavily inspired by gitea
  • add GetAllIssues which only receives a paginator and gathers all issues ignoring repoAt field

Signed-off-by: Seongmin Lee boltlessengineer@proton.me

STACK 13
#711 appview/notify: merge NewPullClosed/Merged/Reopen to NewPullState
1
#6
#710 appview: add pull_reopen event
0
#6
#709 appview/pulls: add search endpoint
0
#6
#708 appview/indexer: add pulls indexer
0
#6
#707 appview: add issue_reopen event
0
#6
#706 appview/indexer: put bleve index dir under indexes/ dir
0
#6
#693 appview/notify: pass logger with mergedLogger
2
#9
#695 appview/indexer: handle NewIssueClosed
1
#9
#694 appview/notify: add DeleteIssue event
1
#9
#496 appview: add issue search endpoint
5
#13
#495 wip: add indexer mappings
1
#14
#494 appview: add basic issue indexer (wip)
2
#14
#685 go.mod: add bleve as a dependency
1
#10
Labels

None yet.

assignee

None yet.

Participants 2
AT URI
at://did:plc:xasnlahkri4ewmbuzly2rlc5/sh.tangled.repo.pull/3lwecnvoz4z22
+35 -32
Interdiff #4 → #5
unified split
.gitignore

This file has not been changed.

appview/indexer/base36/base36.go

This file has not been changed.

appview/indexer/bleve/batch.go

This file has not been changed.

appview/indexer/indexer.go

This file has not been changed.

+27 -19
appview/indexer/issues/indexer.go
··· 1 + // heavily inspired by gitea's model (basically copy-pasted) 1 2 package issues_indexer 2 3 3 4 import ( ··· 90 91 return db.GetIssuesPaginated(e, page) 91 92 }, 92 93 func(issues []models.Issue) error { 93 - var dataList []*IssueData 94 - for _, issue := range issues { 95 - dataList = append(dataList, &IssueData{ 96 - ID: issue.Id, 97 - IssueID: issue.IssueId, 98 - Title: issue.Title, 99 - Body: issue.Body, 100 - IsOpen: issue.Open, 101 - }) 102 - err := ix.Index(ctx, dataList...) 103 - if err != nil { 104 - return err 105 - } 106 - } 107 - return nil 94 + count += len(issues) 95 + return ix.Index(ctx, issues...) 108 96 }, 109 97 ) 110 98 l.Info("issues indexed", "count", count) 111 99 return err 112 100 } 113 101 114 - // IssueData data stored and will be indexed 115 - type IssueData struct { 102 + // issueData data stored and will be indexed 103 + type issueData struct { 116 104 ID int64 `json:"id"` 105 + RepoAt string `json:"repo_at"` 117 106 IssueID int `json:"issue_id"` 118 107 Title string `json:"title"` 119 108 Body string `json:"body"` ··· 122 111 Comments []IssueCommentData `json:"comments"` 123 112 } 124 113 114 + func makeIssueData(issue *models.Issue) issueData { 115 + return issueData{ 116 + ID: issue.Id, 117 + RepoAt: issue.RepoAt.String(), 118 + IssueID: issue.IssueId, 119 + Title: issue.Title, 120 + Body: issue.Body, 121 + IsOpen: issue.Open, 122 + } 123 + } 124 + 125 125 type IssueCommentData struct { 126 126 Body string `json:"body"` 127 127 } ··· 133 133 134 134 const maxBatchSize = 20 135 135 136 - func (ix *Indexer) Index(ctx context.Context, issues ...*IssueData) error { 136 + func (ix *Indexer) Index(ctx context.Context, issues ...models.Issue) error { 137 137 batch := bleveutil.NewFlushingBatch(ix.indexer, maxBatchSize) 138 138 for _, issue := range issues { 139 - if err := batch.Index(base36.Encode(issue.ID), issue); err != nil { 139 + issueData := makeIssueData(&issue) 140 + if err := batch.Index(base36.Encode(issue.Id), issueData); err != nil { 140 141 return err 141 142 } 142 143 } ··· 153 154 matchAndQuery(opts.Keyword, "body"), 154 155 )) 155 156 } 157 + queries = append(queries, keywordFieldQuery(opts.RepoAt, "repo_at")) 156 158 queries = append(queries, boolFieldQuery(opts.IsOpen, "is_open")) 157 159 // TODO: append more queries 158 160 ··· 184 186 185 187 func boolFieldQuery(val bool, field string) query.Query { 186 188 q := bleve.NewBoolFieldQuery(val) 189 + q.FieldVal = field 190 + return q 191 + } 192 + 193 + func keywordFieldQuery(keyword, field string) query.Query { 194 + q := bleve.NewTermQuery(keyword) 187 195 q.FieldVal = field 188 196 return q 189 197 }
+7 -13
appview/indexer/notifier.go
··· 3 3 import ( 4 4 "context" 5 5 6 - issues_indexer "tangled.org/core/appview/indexer/issues" 7 6 "tangled.org/core/appview/models" 8 7 "tangled.org/core/appview/notify" 8 + "tangled.org/core/log" 9 9 ) 10 10 11 11 var _ notify.Notifier = &Indexer{} 12 12 13 13 func (ix *Indexer) NewIssue(ctx context.Context, issue *models.Issue) { 14 - ix.Issues.Index(ctx, &issues_indexer.IssueData{ 15 - ID: issue.Id, 16 - IssueID: issue.IssueId, 17 - Title: issue.Title, 18 - Body: issue.Body, 19 - IsOpen: issue.Open, 20 - Comments: []issues_indexer.IssueCommentData{}, 21 - }) 22 - } 23 - 24 - func (ix *Indexer) NewPullComment(ctx context.Context, comment *models.PullComment) { 25 - panic("unimplemented") 14 + l := log.FromContext(ctx).With("notifier", "indexer.NewIssue", "issue", issue) 15 + l.Debug("indexing new issue") 16 + err := ix.Issues.Index(ctx, *issue) 17 + if err != nil { 18 + l.Error("failed to index an issue", "err", err) 19 + } 26 20 }
+1
appview/issues/issues.go
··· 851 851 Rkey: tid.TID(), 852 852 Title: r.FormValue("title"), 853 853 Body: r.FormValue("body"), 854 + Open: true, 854 855 Did: user.Did, 855 856 Created: time.Now(), 856 857 Repo: &f.Repo,
appview/models/search.go

This file has not been changed.

appview/pages/pages.go

This file has not been changed.

appview/pagination/page.go

This file has not been changed.

appview/state/router.go

This file has not been changed.

appview/state/state.go

This file has not been changed.

History

15 rounds 2 comments
sign up or login to add to the discussion
boltless.me submitted #14
diff interdiff
1 commit
expand
bb301662
appview: add basic issue indexer
expand 0 comments
pull request successfully merged
boltless.me submitted #13
diff interdiff
1 commit
expand
bb301662
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #12
diff interdiff
1 commit
expand
47b55ade
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #11
diff interdiff
1 commit
expand
b335f72a
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #10
diff interdiff
1 commit
expand
429a2f09
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #9
diff interdiff
1 commit
expand
429a2f09
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #8
diff interdiff
1 commit
expand
429a2f09
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #7
diff interdiff
1 commit
expand
3e70c814
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #6
diff interdiff
1 commit
expand
3e70c814
appview: add basic issue indexer
expand 1 comment
oppi.li

this changeset largely lgtm!

boltless.me submitted #5
diff
1 commit
expand
b9a6ec9d
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #4
diff interdiff
1 commit
expand
66f42aa9
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #3
diff interdiff
1 commit
expand
e1d4d340
appview: add basic issue indexer
expand 1 comment
oppi.li
  • this is an N+1 query, because we first get all issues and then make one query for each issue. understandably this only needs to happen when populating the indices but we could improve this by just making one query
  • we can use a pagination free API for getting issues with comments while populating indexer
  • more of a note to self: search index should also handle issue deletions
boltless.me submitted #2
diff interdiff
1 commit
expand
c83d9601
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #1
diff interdiff
1 commit
expand
af090ec1
appview: add basic issue indexer
expand 0 comments
boltless.me submitted #0
diff
1 commit
expand
31b2eff1
appview: add basic issue indexer (wip)
expand 0 comments