this repo has no description
tangled.org
1// heavily inspired by gitea's model (basically copy-pasted)
2package issues_indexer
3
4import (
5 "context"
6 "errors"
7 "log"
8 "os"
9
10 "github.com/blevesearch/bleve/v2"
11 "github.com/blevesearch/bleve/v2/index/upsidedown"
12 "github.com/blevesearch/bleve/v2/search/query"
13 "tangled.org/core/appview/db"
14 "tangled.org/core/appview/indexer/base36"
15 "tangled.org/core/appview/indexer/bleve"
16 "tangled.org/core/appview/models"
17 "tangled.org/core/appview/pagination"
18 tlog "tangled.org/core/log"
19)
20
21type Indexer struct {
22 indexer bleve.Index
23 path string
24}
25
26func NewIndexer(indexDir string) *Indexer {
27 return &Indexer{
28 path: indexDir,
29 }
30}
31
32// Init initializes the indexer
33func (ix *Indexer) Init(ctx context.Context, e db.Execer) {
34 l := tlog.FromContext(ctx)
35 existed, err := ix.intialize(ctx)
36 if err != nil {
37 log.Fatalln("failed to initialize issue indexer", err)
38 }
39 if !existed {
40 l.Debug("Populating the issue indexer")
41 err := PopulateIndexer(ctx, ix, e)
42 if err != nil {
43 log.Fatalln("failed to populate issue indexer", err)
44 }
45 }
46 l.Info("Initialized the issue indexer")
47}
48
49func (ix *Indexer) intialize(ctx context.Context) (bool, error) {
50 if ix.indexer != nil {
51 return false, errors.New("indexer is already initialized")
52 }
53
54 indexer, err := openIndexer(ctx, ix.path)
55 if err != nil {
56 return false, err
57 }
58 if indexer != nil {
59 ix.indexer = indexer
60 return true, nil
61 }
62
63 mapping := bleve.NewIndexMapping()
64 indexer, err = bleve.New(ix.path, mapping)
65 if err != nil {
66 return false, err
67 }
68
69 ix.indexer = indexer
70
71 return false, nil
72}
73
74func openIndexer(ctx context.Context, path string) (bleve.Index, error) {
75 l := tlog.FromContext(ctx)
76 indexer, err := bleve.Open(path)
77 if err != nil {
78 if errors.Is(err, upsidedown.IncompatibleVersion) {
79 l.Info("Indexer was built with a previous version of bleve, deleting and rebuilding")
80 return nil, os.RemoveAll(path)
81 }
82 return nil, nil
83 }
84 return indexer, nil
85}
86
87func PopulateIndexer(ctx context.Context, ix *Indexer, e db.Execer) error {
88 l := tlog.FromContext(ctx)
89 count := 0
90 err := pagination.IterateAll(
91 func(page pagination.Page) ([]models.Issue, error) {
92 return db.GetIssuesPaginated(e, page)
93 },
94 func(issues []models.Issue) error {
95 count += len(issues)
96 return ix.Index(ctx, issues...)
97 },
98 )
99 l.Info("issues indexed", "count", count)
100 return err
101}
102
103// issueData data stored and will be indexed
104type issueData struct {
105 ID int64 `json:"id"`
106 RepoAt string `json:"repo_at"`
107 IssueID int `json:"issue_id"`
108 Title string `json:"title"`
109 Body string `json:"body"`
110
111 IsOpen bool `json:"is_open"`
112 Comments []IssueCommentData `json:"comments"`
113}
114
115func makeIssueData(issue *models.Issue) *issueData {
116 return &issueData{
117 ID: issue.Id,
118 RepoAt: issue.RepoAt.String(),
119 IssueID: issue.IssueId,
120 Title: issue.Title,
121 Body: issue.Body,
122 IsOpen: issue.Open,
123 }
124}
125
126type IssueCommentData struct {
127 Body string `json:"body"`
128}
129
130type SearchResult struct {
131 Hits []int64
132 Total uint64
133}
134
135const maxBatchSize = 20
136
137func (ix *Indexer) Index(ctx context.Context, issues ...models.Issue) error {
138 batch := bleveutil.NewFlushingBatch(ix.indexer, maxBatchSize)
139 for _, issue := range issues {
140 issueData := makeIssueData(&issue)
141 if err := batch.Index(base36.Encode(issue.Id), issueData); err != nil {
142 return err
143 }
144 }
145 return batch.Flush()
146}
147
148// Search searches for issues
149func (ix *Indexer) Search(ctx context.Context, opts models.IssueSearchOptions) (*SearchResult, error) {
150 var queries []query.Query
151
152 if opts.Keyword != "" {
153 queries = append(queries, bleve.NewDisjunctionQuery(
154 bleveutil.MatchAndQuery("title", opts.Keyword),
155 bleveutil.MatchAndQuery("body", opts.Keyword),
156 ))
157 }
158 queries = append(queries, bleveutil.KeywordFieldQuery("repo_at", opts.RepoAt))
159 queries = append(queries, bleveutil.BoolFieldQuery("is_open", opts.IsOpen))
160 // TODO: append more queries
161
162 var indexerQuery query.Query = bleve.NewConjunctionQuery(queries...)
163 searchReq := bleve.NewSearchRequestOptions(indexerQuery, opts.Page.Limit, opts.Page.Offset, false)
164 res, err := ix.indexer.SearchInContext(ctx, searchReq)
165 if err != nil {
166 return nil, nil
167 }
168 ret := &SearchResult{
169 Total: res.Total,
170 Hits: make([]int64, len(res.Hits)),
171 }
172 for i, hit := range res.Hits {
173 id, err := base36.Decode(hit.ID)
174 if err != nil {
175 return nil, err
176 }
177 ret.Hits[i] = id
178 }
179 return ret, nil
180}