tangled.org
- Heavily inspired by gitea
- add
GetAllIssueswhich only receives a paginator and gathers all issues ignoringrepoAtfield
Signed-off-by: Seongmin Lee boltlessengineer@proton.me
+1
.gitignore
+1
.gitignore
+20
appview/indexer/base36/base36.go
+20
appview/indexer/base36/base36.go
···
1
+
// mostly copied from gitea/modules/indexer/internal/base32
2
+
3
+
package base36
4
+
5
+
import (
6
+
"fmt"
7
+
"strconv"
8
+
)
9
+
10
+
func Encode(i int64) string {
11
+
return strconv.FormatInt(i, 36)
12
+
}
13
+
14
+
func Decode(s string) (int64, error) {
15
+
i, err := strconv.ParseInt(s, 36, 64)
16
+
if err != nil {
17
+
return 0, fmt.Errorf("invalid base36 integer %q: %w", s, err)
18
+
}
19
+
return i, nil
20
+
}
+58
appview/indexer/bleve/batch.go
+58
appview/indexer/bleve/batch.go
···
1
+
// Copyright 2021 The Gitea Authors. All rights reserved.
2
+
// SPDX-License-Identifier: MIT
3
+
4
+
package bleveutil
5
+
6
+
import (
7
+
"github.com/blevesearch/bleve/v2"
8
+
)
9
+
10
+
// FlushingBatch is a batch of operations that automatically flushes to the
11
+
// underlying index once it reaches a certain size.
12
+
type FlushingBatch struct {
13
+
maxBatchSize int
14
+
batch *bleve.Batch
15
+
index bleve.Index
16
+
}
17
+
18
+
// NewFlushingBatch creates a new flushing batch for the specified index. Once
19
+
// the number of operations in the batch reaches the specified limit, the batch
20
+
// automatically flushes its operations to the index.
21
+
func NewFlushingBatch(index bleve.Index, maxBatchSize int) *FlushingBatch {
22
+
return &FlushingBatch{
23
+
maxBatchSize: maxBatchSize,
24
+
batch: index.NewBatch(),
25
+
index: index,
26
+
}
27
+
}
28
+
29
+
// Index add a new index to batch
30
+
func (b *FlushingBatch) Index(id string, data any) error {
31
+
if err := b.batch.Index(id, data); err != nil {
32
+
return err
33
+
}
34
+
return b.flushIfFull()
35
+
}
36
+
37
+
// Delete add a delete index to batch
38
+
func (b *FlushingBatch) Delete(id string) error {
39
+
b.batch.Delete(id)
40
+
return b.flushIfFull()
41
+
}
42
+
43
+
func (b *FlushingBatch) flushIfFull() error {
44
+
if b.batch.Size() < b.maxBatchSize {
45
+
return nil
46
+
}
47
+
return b.Flush()
48
+
}
49
+
50
+
// Flush submit the batch and create a new one
51
+
func (b *FlushingBatch) Flush() error {
52
+
err := b.index.Batch(b.batch)
53
+
if err != nil {
54
+
return err
55
+
}
56
+
b.batch = b.index.NewBatch()
57
+
return nil
58
+
}
+32
appview/indexer/indexer.go
+32
appview/indexer/indexer.go
···
1
+
package indexer
2
+
3
+
import (
4
+
"context"
5
+
"log/slog"
6
+
7
+
"tangled.org/core/appview/db"
8
+
issues_indexer "tangled.org/core/appview/indexer/issues"
9
+
"tangled.org/core/appview/notify"
10
+
tlog "tangled.org/core/log"
11
+
)
12
+
13
+
type Indexer struct {
14
+
Issues *issues_indexer.Indexer
15
+
logger *slog.Logger
16
+
notify.BaseNotifier
17
+
}
18
+
19
+
func New(logger *slog.Logger) *Indexer {
20
+
return &Indexer {
21
+
issues_indexer.NewIndexer("indexes.bleve"),
22
+
logger,
23
+
notify.BaseNotifier{},
24
+
}
25
+
}
26
+
27
+
// Init initializes all indexers
28
+
func (ix *Indexer) Init(ctx context.Context, db *db.DB) error {
29
+
ctx = tlog.IntoContext(ctx, ix.logger)
30
+
ix.Issues.Init(ctx, db)
31
+
return nil
32
+
}
+189
appview/indexer/issues/indexer.go
+189
appview/indexer/issues/indexer.go
···
1
+
package issues_indexer
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"os"
7
+
8
+
"github.com/blevesearch/bleve/v2"
9
+
"github.com/blevesearch/bleve/v2/index/upsidedown"
10
+
"github.com/blevesearch/bleve/v2/search/query"
11
+
"tangled.org/core/appview/db"
12
+
"tangled.org/core/appview/indexer/base36"
13
+
"tangled.org/core/appview/indexer/bleve"
14
+
"tangled.org/core/appview/models"
15
+
"tangled.org/core/appview/pagination"
16
+
tlog "tangled.org/core/log"
17
+
)
18
+
19
+
type Indexer struct {
20
+
indexer bleve.Index
21
+
path string
22
+
}
23
+
24
+
func NewIndexer(indexDir string) *Indexer {
25
+
return &Indexer{
26
+
path: indexDir,
27
+
}
28
+
}
29
+
30
+
// Init initializes the indexer
31
+
func (ix *Indexer) Init(ctx context.Context, e db.Execer) {
32
+
l := tlog.FromContext(ctx)
33
+
existed, err := ix.intialize(ctx)
34
+
if err != nil {
35
+
l.Error("failed to initialize issue indexer", "err", err)
36
+
}
37
+
if !existed {
38
+
l.Debug("Populating the issue indexer")
39
+
err := PopulateIndexer(ctx, ix, e)
40
+
if err != nil {
41
+
l.Error("failed to populate issue indexer", "err", err)
42
+
}
43
+
}
44
+
l.Info("Initialized the issue indexer")
45
+
}
46
+
47
+
func (ix *Indexer) intialize(ctx context.Context) (bool, error) {
48
+
if ix.indexer != nil {
49
+
return false, errors.New("indexer is already initialized")
50
+
}
51
+
52
+
indexer, err := openIndexer(ctx, ix.path)
53
+
if err != nil {
54
+
return false, err
55
+
}
56
+
if indexer != nil {
57
+
ix.indexer = indexer
58
+
return true, nil
59
+
}
60
+
61
+
mapping := bleve.NewIndexMapping()
62
+
indexer, err = bleve.New(ix.path, mapping)
63
+
if err != nil {
64
+
return false, err
65
+
}
66
+
67
+
ix.indexer = indexer
68
+
69
+
return false, nil
70
+
}
71
+
72
+
func openIndexer(ctx context.Context, path string) (bleve.Index, error) {
73
+
l := tlog.FromContext(ctx)
74
+
indexer, err := bleve.Open(path)
75
+
if err != nil {
76
+
if errors.Is(err, upsidedown.IncompatibleVersion) {
77
+
l.Info("Indexer was built with a previous version of bleve, deleting and rebuilding")
78
+
return nil, os.RemoveAll(path)
79
+
}
80
+
return nil, nil
81
+
}
82
+
return indexer, nil
83
+
}
84
+
85
+
func PopulateIndexer(ctx context.Context, ix *Indexer, e db.Execer) error {
86
+
l := tlog.FromContext(ctx)
87
+
count := 0
88
+
err := pagination.IterateAll(
89
+
func(page pagination.Page) ([]models.Issue, error) {
90
+
return db.GetIssuesPaginated(e, page)
91
+
},
92
+
func(issues []models.Issue) error {
93
+
var dataList []*IssueData
94
+
for _, issue := range issues {
95
+
dataList = append(dataList, &IssueData{
96
+
ID: issue.Id,
97
+
IssueID: issue.IssueId,
98
+
Title: issue.Title,
99
+
Body: issue.Body,
100
+
IsOpen: issue.Open,
101
+
})
102
+
err := ix.Index(ctx, dataList...)
103
+
if err != nil {
104
+
return err
105
+
}
106
+
}
107
+
return nil
108
+
},
109
+
)
110
+
l.Info("issues indexed", "count", count)
111
+
return err
112
+
}
113
+
114
+
// IssueData data stored and will be indexed
115
+
type IssueData struct {
116
+
ID int64 `json:"id"`
117
+
IssueID int `json:"issue_id"`
118
+
Title string `json:"title"`
119
+
Body string `json:"body"`
120
+
121
+
IsOpen bool `json:"is_open"`
122
+
Comments []IssueCommentData `json:"comments"`
123
+
}
124
+
125
+
type IssueCommentData struct {
126
+
Body string `json:"body"`
127
+
}
128
+
129
+
type SearchResult struct {
130
+
Hits []int64
131
+
Total uint64
132
+
}
133
+
134
+
const maxBatchSize = 20
135
+
136
+
func (ix *Indexer) Index(ctx context.Context, issues ...*IssueData) error {
137
+
batch := bleveutil.NewFlushingBatch(ix.indexer, maxBatchSize)
138
+
for _, issue := range issues {
139
+
if err := batch.Index(base36.Encode(issue.ID), issue); err != nil {
140
+
return err
141
+
}
142
+
}
143
+
return batch.Flush()
144
+
}
145
+
146
+
// Search searches for issues
147
+
func (ix *Indexer) Search(ctx context.Context, opts models.IssueSearchOptions) (*SearchResult, error) {
148
+
var queries []query.Query
149
+
150
+
if opts.Keyword != "" {
151
+
queries = append(queries, bleve.NewDisjunctionQuery(
152
+
matchAndQuery(opts.Keyword, "title"),
153
+
matchAndQuery(opts.Keyword, "body"),
154
+
))
155
+
}
156
+
queries = append(queries, boolFieldQuery(opts.IsOpen, "is_open"))
157
+
// TODO: append more queries
158
+
159
+
var indexerQuery query.Query = bleve.NewConjunctionQuery(queries...)
160
+
searchReq := bleve.NewSearchRequestOptions(indexerQuery, opts.Page.Limit, opts.Page.Offset, false)
161
+
res, err := ix.indexer.SearchInContext(ctx, searchReq)
162
+
if err != nil {
163
+
return nil, nil
164
+
}
165
+
ret := &SearchResult{
166
+
Total: res.Total,
167
+
Hits: make([]int64, len(res.Hits)),
168
+
}
169
+
for i, hit := range res.Hits {
170
+
id, err := base36.Decode(hit.ID)
171
+
if err != nil {
172
+
return nil, err
173
+
}
174
+
ret.Hits[i] = id
175
+
}
176
+
return ret, nil
177
+
}
178
+
179
+
func matchAndQuery(keyword, field string) query.Query {
180
+
q := bleve.NewMatchQuery(keyword)
181
+
q.FieldVal = field
182
+
return q
183
+
}
184
+
185
+
func boolFieldQuery(val bool, field string) query.Query {
186
+
q := bleve.NewBoolFieldQuery(val)
187
+
q.FieldVal = field
188
+
return q
189
+
}
+26
appview/indexer/notifier.go
+26
appview/indexer/notifier.go
···
1
+
package indexer
2
+
3
+
import (
4
+
"context"
5
+
6
+
issues_indexer "tangled.org/core/appview/indexer/issues"
7
+
"tangled.org/core/appview/models"
8
+
"tangled.org/core/appview/notify"
9
+
)
10
+
11
+
var _ notify.Notifier = &Indexer{}
12
+
13
+
func (ix *Indexer) NewIssue(ctx context.Context, issue *models.Issue) {
14
+
ix.Issues.Index(ctx, &issues_indexer.IssueData{
15
+
ID: issue.Id,
16
+
IssueID: issue.IssueId,
17
+
Title: issue.Title,
18
+
Body: issue.Body,
19
+
IsOpen: issue.Open,
20
+
Comments: []issues_indexer.IssueCommentData{},
21
+
})
22
+
}
23
+
24
+
func (ix *Indexer) NewPullComment(ctx context.Context, comment *models.PullComment) {
25
+
panic("unimplemented")
26
+
}
+4
appview/issues/issues.go
+4
appview/issues/issues.go
···
19
19
"tangled.org/core/api/tangled"
20
20
"tangled.org/core/appview/config"
21
21
"tangled.org/core/appview/db"
22
+
issues_indexer "tangled.org/core/appview/indexer/issues"
22
23
"tangled.org/core/appview/models"
23
24
"tangled.org/core/appview/notify"
24
25
"tangled.org/core/appview/oauth"
···
40
41
notifier notify.Notifier
41
42
logger *slog.Logger
42
43
validator *validator.Validator
44
+
indexer *issues_indexer.Indexer
43
45
}
44
46
45
47
func New(
···
51
53
config *config.Config,
52
54
notifier notify.Notifier,
53
55
validator *validator.Validator,
56
+
indexer *issues_indexer.Indexer,
54
57
logger *slog.Logger,
55
58
) *Issues {
56
59
return &Issues{
···
63
66
notifier: notifier,
64
67
logger: logger,
65
68
validator: validator,
69
+
indexer: indexer,
66
70
}
67
71
}
68
72
+23
appview/models/search.go
+23
appview/models/search.go
···
1
+
package models
2
+
3
+
import "tangled.org/core/appview/pagination"
4
+
5
+
type IssueSearchOptions struct {
6
+
Keyword string
7
+
RepoAt string
8
+
IsOpen bool
9
+
10
+
Page pagination.Page
11
+
}
12
+
13
+
// func (so *SearchOptions) ToFilters() []filter {
14
+
// var filters []filter
15
+
// if so.IsOpen != nil {
16
+
// openValue := 0
17
+
// if *so.IsOpen {
18
+
// openValue = 1
19
+
// }
20
+
// filters = append(filters, FilterEq("open", openValue))
21
+
// }
22
+
// return filters
23
+
// }
+1
appview/pages/pages.go
+1
appview/pages/pages.go
+23
appview/pagination/page.go
+23
appview/pagination/page.go
···
29
29
Limit: p.Limit,
30
30
}
31
31
}
32
+
33
+
func IterateAll[T any](
34
+
fetch func(page Page) ([]T, error),
35
+
handle func(items []T) error,
36
+
) error {
37
+
page := FirstPage()
38
+
for {
39
+
items, err := fetch(page)
40
+
if err != nil {
41
+
return err
42
+
}
43
+
44
+
err = handle(items)
45
+
if err != nil {
46
+
return err
47
+
}
48
+
if len(items) < page.Limit {
49
+
break
50
+
}
51
+
page = page.Next()
52
+
}
53
+
return nil
54
+
}
+1
appview/state/router.go
+1
appview/state/router.go
+10
appview/state/state.go
+10
appview/state/state.go
···
14
14
"tangled.org/core/appview"
15
15
"tangled.org/core/appview/config"
16
16
"tangled.org/core/appview/db"
17
+
"tangled.org/core/appview/indexer"
17
18
"tangled.org/core/appview/models"
18
19
"tangled.org/core/appview/notify"
19
20
dbnotify "tangled.org/core/appview/notify/db"
···
43
44
type State struct {
44
45
db *db.DB
45
46
notifier notify.Notifier
47
+
indexer *indexer.Indexer
46
48
oauth *oauth.OAuth
47
49
enforcer *rbac.Enforcer
48
50
pages *pages.Pages
···
65
67
return nil, fmt.Errorf("failed to create db: %w", err)
66
68
}
67
69
70
+
indexer := indexer.New(log.SubLogger(logger, "indexer"))
71
+
err = indexer.Init(ctx, d)
72
+
if err != nil {
73
+
return nil, fmt.Errorf("failed to create indexer: %w", err)
74
+
}
75
+
68
76
enforcer, err := rbac.NewEnforcer(config.Core.DbPath)
69
77
if err != nil {
70
78
return nil, fmt.Errorf("failed to create enforcer: %w", err)
···
159
167
if !config.Core.Dev {
160
168
notifiers = append(notifiers, phnotify.NewPosthogNotifier(posthog))
161
169
}
170
+
notifiers = append(notifiers, indexer)
162
171
notifier := notify.NewMergedNotifier(notifiers...)
163
172
164
173
state := &State{
165
174
d,
166
175
notifier,
176
+
indexer,
167
177
oauth,
168
178
enforcer,
169
179
pages,
History
15 rounds
2 comments
boltless.me
submitted
#14
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
pull request successfully merged
boltless.me
submitted
#13
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#12
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#11
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#10
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#9
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#8
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#7
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#6
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 1 comment
boltless.me
submitted
#5
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#4
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#3
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 1 comment
- this is an N+1 query, because we first get all issues and then make one query for each issue. understandably this only needs to happen when populating the indices but we could improve this by just making one query
- we can use a pagination free API for getting issues with comments while populating indexer
- more of a note to self: search index should also handle issue deletions
boltless.me
submitted
#2
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#1
1 commit
expand
collapse
appview: add basic issue indexer
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
expand 0 comments
boltless.me
submitted
#0
1 commit
expand
collapse
appview: add basic issue indexer (wip)
- Heavily inspired by gitea
- add `GetAllIssues` which only receives a paginator and gathers all
issues ignoring `repoAt` field
Signed-off-by: Seongmin Lee <boltlessengineer@proton.me>
this changeset largely lgtm!