onedeuxtriseigo.nullpo.dev
A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
1// SiYuan - Refactor your thinking
2// Copyright (c) 2020-present, b3log.org
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU Affero General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU Affero General Public License for more details.
13//
14// You should have received a copy of the GNU Affero General Public License
15// along with this program. If not, see <https://www.gnu.org/licenses/>.
16
17package model
18
19import (
20 "bytes"
21 "regexp"
22 "sort"
23 "strings"
24 "time"
25
26 "github.com/88250/gulu"
27 "github.com/88250/lute"
28 "github.com/88250/lute/ast"
29 "github.com/88250/lute/editor"
30 "github.com/88250/lute/parse"
31 "github.com/ClarkThan/ahocorasick"
32 "github.com/dgraph-io/ristretto"
33 "github.com/siyuan-note/siyuan/kernel/search"
34 "github.com/siyuan-note/siyuan/kernel/sql"
35 "github.com/siyuan-note/siyuan/kernel/task"
36 "github.com/siyuan-note/siyuan/kernel/treenode"
37 "github.com/siyuan-note/siyuan/kernel/util"
38)
39
40// virtualBlockRefCache 用于保存块关联的虚拟引用关键字。
41// 改进打开虚拟引用后加载文档的性能 https://github.com/siyuan-note/siyuan/issues/7378
42var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{
43 NumCounters: 102400,
44 MaxCost: 10240,
45 BufferItems: 64,
46})
47
48func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) {
49 val, ok := virtualBlockRefCache.Get(root.ID)
50 if !ok {
51 buf := bytes.Buffer{}
52 ast.Walk(root, func(n *ast.Node, entering bool) ast.WalkStatus {
53 if !entering || !n.IsBlock() {
54 return ast.WalkContinue
55 }
56
57 content := sql.NodeStaticContent(n, nil, false, false, false)
58 content = strings.ReplaceAll(content, editor.Zwsp, "")
59 buf.WriteString(content)
60 return ast.WalkContinue
61 })
62 content := buf.String()
63 ret = putBlockVirtualRefKeywords(content, root)
64 return
65 }
66 ret = val.([]string)
67 return
68}
69
70func putBlockVirtualRefKeywords(blockContent string, root *ast.Node) (ret []string) {
71 keywords := getVirtualRefKeywords(root)
72 if 1 > len(keywords) {
73 return
74 }
75
76 contentTmp := blockContent
77 var keywordsTmp []string
78 if !Conf.Search.CaseSensitive {
79 contentTmp = strings.ToLower(blockContent)
80 for _, keyword := range keywords {
81 keywordsTmp = append(keywordsTmp, strings.ToLower(keyword))
82 }
83 } else {
84 for _, keyword := range keywords {
85 keywordsTmp = append(keywordsTmp, keyword)
86 }
87 }
88
89 m := ahocorasick.NewMatcher()
90 m.BuildWithPatterns(keywordsTmp)
91 hits := m.Search(contentTmp)
92 for _, hit := range hits {
93 ret = append(ret, hit)
94 }
95
96 if 1 > len(ret) {
97 return
98 }
99
100 ret = gulu.Str.RemoveDuplicatedElem(ret)
101 virtualBlockRefCache.SetWithTTL(root.ID, ret, 1, 10*time.Minute)
102 return
103}
104
105func CacheVirtualBlockRefJob() {
106 if !Conf.Editor.VirtualBlockRef {
107 return
108 }
109 task.AppendTask(task.CacheVirtualBlockRef, ResetVirtualBlockRefCache)
110}
111
112func ResetVirtualBlockRefCache() {
113 virtualBlockRefCache.Clear()
114 if !Conf.Editor.VirtualBlockRef {
115 return
116 }
117
118 searchIgnoreLines := getSearchIgnoreLines()
119 refSearchIgnoreLines := getRefSearchIgnoreLines()
120 keywords := sql.QueryVirtualRefKeywords(Conf.Search.VirtualRefName, Conf.Search.VirtualRefAlias, Conf.Search.VirtualRefAnchor, Conf.Search.VirtualRefDoc, searchIgnoreLines, refSearchIgnoreLines)
121 virtualBlockRefCache.Set("virtual_ref", keywords, 1)
122}
123
124func AddVirtualBlockRefInclude(keyword []string) {
125 if 1 > len(keyword) {
126 return
127 }
128
129 include := strings.ReplaceAll(Conf.Editor.VirtualBlockRefInclude, "\\,", "__comma@sep__")
130 includes := strings.Split(include, ",")
131 includes = append(includes, keyword...)
132 includes = gulu.Str.RemoveDuplicatedElem(includes)
133 Conf.Editor.VirtualBlockRefInclude = strings.Join(includes, ",")
134 Conf.Save()
135
136 ResetVirtualBlockRefCache()
137}
138
139func AddVirtualBlockRefExclude(keyword []string) {
140 if 1 > len(keyword) {
141 return
142 }
143
144 exclude := strings.ReplaceAll(Conf.Editor.VirtualBlockRefExclude, "\\,", "__comma@sep__")
145 excludes := strings.Split(exclude, ",")
146 excludes = append(excludes, keyword...)
147 excludes = gulu.Str.RemoveDuplicatedElem(excludes)
148 Conf.Editor.VirtualBlockRefExclude = strings.Join(excludes, ",")
149 Conf.Save()
150
151 ResetVirtualBlockRefCache()
152}
153
154func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool {
155 if !Conf.Editor.VirtualBlockRef || 1 > len(virtualBlockRefKeywords) {
156 return false
157 }
158
159 if ast.NodeText != n.Type {
160 return false
161 }
162
163 parentBlock := treenode.ParentBlock(n)
164 if nil == parentBlock {
165 return false
166 }
167
168 if 0 < refCount[parentBlock.ID] {
169 // 如果块被引用过，则将其自身的文本排除在虚拟引用关键字之外
170 // Referenced blocks support rendering virtual references https://github.com/siyuan-note/siyuan/issues/10960
171 parentText := getNodeRefText(parentBlock)
172 virtualBlockRefKeywords = gulu.Str.RemoveElem(virtualBlockRefKeywords, parentText)
173 }
174
175 content := string(n.Tokens)
176 tmp := util.RemoveInvalid(content)
177 tmp = strings.TrimSpace(tmp)
178 if "" == tmp {
179 return false
180 }
181
182 newContent := markReplaceSpanWithSplit(content, virtualBlockRefKeywords, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd())
183 if content != newContent {
184 // 虚拟引用排除命中自身块命名和别名的情况 https://github.com/siyuan-note/siyuan/issues/3185
185 var blockKeys []string
186 if name := parentBlock.IALAttr("name"); "" != name {
187 blockKeys = append(blockKeys, name)
188 }
189 if alias := parentBlock.IALAttr("alias"); "" != alias {
190 blockKeys = append(blockKeys, alias)
191 }
192 if 0 < len(blockKeys) {
193 keys := gulu.Str.SubstringsBetween(newContent, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd())
194 for _, k := range keys {
195 if gulu.Str.Contains(k, blockKeys) {
196 return true
197 }
198 }
199 }
200
201 n.Tokens = []byte(newContent)
202 linkTree := parse.Inline("", n.Tokens, luteEngine.ParseOptions)
203 var children []*ast.Node
204 for c := linkTree.Root.FirstChild.FirstChild; nil != c; c = c.Next {
205 children = append(children, c)
206 }
207 for _, c := range children {
208 n.InsertBefore(c)
209 }
210 *unlinks = append(*unlinks, n)
211 return true
212 }
213 return false
214}
215
216func getVirtualRefKeywords(root *ast.Node) (ret []string) {
217 if !Conf.Editor.VirtualBlockRef {
218 return
219 }
220
221 if val, ok := virtualBlockRefCache.Get("virtual_ref"); ok {
222 ret = val.([]string)
223 }
224
225 if "" != strings.TrimSpace(Conf.Editor.VirtualBlockRefInclude) {
226 include := strings.ReplaceAll(Conf.Editor.VirtualBlockRefInclude, "\\,", "__comma@sep__")
227 includes := strings.Split(include, ",")
228 var tmp []string
229 for _, e := range includes {
230 e = strings.ReplaceAll(e, "__comma@sep__", ",")
231 tmp = append(tmp, e)
232 }
233 includes = tmp
234 ret = append(ret, includes...)
235 ret = gulu.Str.RemoveDuplicatedElem(ret)
236 }
237
238 if "" != strings.TrimSpace(Conf.Editor.VirtualBlockRefExclude) {
239 exclude := strings.ReplaceAll(Conf.Editor.VirtualBlockRefExclude, "\\,", "__comma@sep__")
240 excludes := strings.Split(exclude, ",")
241 var tmp, regexps []string
242 for _, e := range excludes {
243 e = strings.ReplaceAll(e, "__comma@sep__", ",")
244 if strings.HasPrefix(e, "/") && strings.HasSuffix(e, "/") {
245 regexps = append(regexps, e[1:len(e)-1])
246 } else {
247 tmp = append(tmp, e)
248 }
249 }
250 excludes = tmp
251 ret = gulu.Str.ExcludeElem(ret, excludes)
252 if 0 < len(regexps) {
253 tmp = nil
254 for _, str := range ret {
255 matchExclude := false
256 for _, re := range regexps {
257 if ok, _ := regexp.MatchString(re, str); ok {
258 matchExclude = true
259 break
260 }
261 }
262 if !matchExclude {
263 tmp = append(tmp, str)
264 }
265 }
266 ret = tmp
267 }
268 }
269
270 // 虚拟引用排除当前文档名 https://github.com/siyuan-note/siyuan/issues/4537
271 // Virtual references exclude the name and aliases from the current document https://github.com/siyuan-note/siyuan/issues/9204
272 title := root.IALAttr("title")
273 ret = gulu.Str.ExcludeElem(ret, []string{title})
274 if name := root.IALAttr("name"); "" != name {
275 ret = gulu.Str.ExcludeElem(ret, []string{name})
276 }
277 if alias := root.IALAttr("alias"); "" != alias {
278 for _, a := range strings.Split(alias, ",") {
279 ret = gulu.Str.ExcludeElem(ret, []string{a})
280 }
281 }
282
283 ret = prepareMarkKeywords(ret)
284 return
285}
286
287func prepareMarkKeywords(keywords []string) (ret []string) {
288 ret = gulu.Str.RemoveDuplicatedElem(keywords)
289 var tmp []string
290 for _, k := range ret {
291 if "" != k && "*" != k { // 提及和虚引排除 * Ignore `*` back mentions and virtual references https://github.com/siyuan-note/siyuan/issues/10873
292 tmp = append(tmp, k)
293 }
294 }
295 ret = tmp
296
297 sort.SliceStable(ret, func(i, j int) bool {
298 return len(ret[i]) > len(ret[j])
299 })
300 return
301}