onedeuxtriseigo.nullpo.dev
A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
1// SiYuan - Refactor your thinking
2// Copyright (c) 2020-present, b3log.org
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU Affero General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU Affero General Public License for more details.
13//
14// You should have received a copy of the GNU Affero General Public License
15// along with this program. If not, see <https://www.gnu.org/licenses/>.
16
17package model
18
19import (
20 "bytes"
21 "errors"
22 "fmt"
23 "math"
24 "os"
25 "path"
26 "path/filepath"
27 "regexp"
28 "sort"
29 "strconv"
30 "strings"
31 "sync"
32 "time"
33 "unicode/utf8"
34
35 "github.com/88250/gulu"
36 "github.com/88250/lute"
37 "github.com/88250/lute/ast"
38 "github.com/88250/lute/editor"
39 "github.com/88250/lute/html"
40 "github.com/88250/lute/lex"
41 "github.com/88250/lute/parse"
42 "github.com/88250/vitess-sqlparser/sqlparser"
43 "github.com/jinzhu/copier"
44 "github.com/siyuan-note/filelock"
45 "github.com/siyuan-note/logging"
46 "github.com/siyuan-note/siyuan/kernel/conf"
47 "github.com/siyuan-note/siyuan/kernel/search"
48 "github.com/siyuan-note/siyuan/kernel/sql"
49 "github.com/siyuan-note/siyuan/kernel/task"
50 "github.com/siyuan-note/siyuan/kernel/treenode"
51 "github.com/siyuan-note/siyuan/kernel/util"
52 "github.com/xrash/smetrics"
53)
54
55func ListInvalidBlockRefs(page, pageSize int) (ret []*Block, matchedBlockCount, matchedRootCount, pageCount int) {
56 refBlockMap := map[string][]string{}
57 blockMap := map[string]bool{}
58 var invalidBlockIDs []string
59 notebooks, err := ListNotebooks()
60 if err != nil {
61 return
62 }
63 luteEngine := util.NewLute()
64 for _, notebook := range notebooks {
65 pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
66 for _, paths := range pages {
67 var trees []*parse.Tree
68 for _, localPath := range paths {
69 tree, loadTreeErr := loadTree(localPath, luteEngine)
70 if nil != loadTreeErr {
71 continue
72 }
73 trees = append(trees, tree)
74 }
75 for _, tree := range trees {
76 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
77 if entering {
78 if n.IsBlock() {
79 blockMap[n.ID] = true
80 return ast.WalkContinue
81 }
82
83 if ast.NodeTextMark == n.Type {
84 if n.IsTextMarkType("a") {
85 if strings.HasPrefix(n.TextMarkAHref, "siyuan://blocks/") {
86 defID := strings.TrimPrefix(n.TextMarkAHref, "siyuan://blocks/")
87 if strings.Contains(defID, "?") {
88 defID = strings.Split(defID, "?")[0]
89 }
90 refID := treenode.ParentBlock(n).ID
91 if defIDs := refBlockMap[refID]; 1 > len(defIDs) {
92 refBlockMap[refID] = []string{defID}
93 } else {
94 refBlockMap[refID] = append(defIDs, defID)
95 }
96 }
97 } else if n.IsTextMarkType("block-ref") {
98 defID := n.TextMarkBlockRefID
99 refID := treenode.ParentBlock(n).ID
100 if defIDs := refBlockMap[refID]; 1 > len(defIDs) {
101 refBlockMap[refID] = []string{defID}
102 } else {
103 refBlockMap[refID] = append(defIDs, defID)
104 }
105 }
106 }
107 }
108 return ast.WalkContinue
109 })
110 }
111 }
112 }
113
114 invalidDefIDs := map[string]bool{}
115 for _, refDefIDs := range refBlockMap {
116 for _, defID := range refDefIDs {
117 invalidDefIDs[defID] = true
118 }
119 }
120
121 var toRemoves []string
122 for defID := range invalidDefIDs {
123 if _, ok := blockMap[defID]; ok {
124 toRemoves = append(toRemoves, defID)
125 }
126 }
127 for _, toRemove := range toRemoves {
128 delete(invalidDefIDs, toRemove)
129 }
130
131 toRemoves = nil
132 for refID, defIDs := range refBlockMap {
133 var tmp []string
134 for _, defID := range defIDs {
135 if _, ok := invalidDefIDs[defID]; !ok {
136 tmp = append(tmp, defID)
137 }
138 }
139
140 for _, toRemove := range tmp {
141 defIDs = gulu.Str.RemoveElem(defIDs, toRemove)
142 }
143
144 if 1 > len(defIDs) {
145 toRemoves = append(toRemoves, refID)
146 }
147 }
148 for _, toRemove := range toRemoves {
149 delete(refBlockMap, toRemove)
150 }
151
152 for refID := range refBlockMap {
153 invalidBlockIDs = append(invalidBlockIDs, refID)
154 }
155 invalidBlockIDs = gulu.Str.RemoveDuplicatedElem(invalidBlockIDs)
156
157 sort.Strings(invalidBlockIDs)
158 allInvalidBlockIDs := invalidBlockIDs
159
160 start := (page - 1) * pageSize
161 end := page * pageSize
162 if end > len(invalidBlockIDs) {
163 end = len(invalidBlockIDs)
164 }
165 invalidBlockIDs = invalidBlockIDs[start:end]
166
167 sqlBlocks := sql.GetBlocks(invalidBlockIDs)
168 var tmp []*sql.Block
169 for _, sqlBlock := range sqlBlocks {
170 if nil != sqlBlock {
171 tmp = append(tmp, sqlBlock)
172 }
173 }
174 sqlBlocks = tmp
175
176 ret = fromSQLBlocks(&sqlBlocks, "", 36)
177 if 1 > len(ret) {
178 ret = []*Block{}
179 }
180 matchedBlockCount = len(allInvalidBlockIDs)
181 rootCount := map[string]bool{}
182 for _, id := range allInvalidBlockIDs {
183 bt := treenode.GetBlockTree(id)
184 if nil == bt {
185 continue
186 }
187 rootCount[bt.RootID] = true
188 }
189 matchedRootCount = len(rootCount)
190 pageCount = (matchedBlockCount + pageSize - 1) / pageSize
191 return
192}
193
194type EmbedBlock struct {
195 Block *Block `json:"block"`
196 BlockPaths []*BlockPath `json:"blockPaths"`
197}
198
199func UpdateEmbedBlock(id, content string) (err error) {
200 bt := treenode.GetBlockTree(id)
201 if nil == bt {
202 err = ErrBlockNotFound
203 return
204 }
205
206 if treenode.TypeAbbr(ast.NodeBlockQueryEmbed.String()) != bt.Type {
207 err = errors.New("not query embed block")
208 return
209 }
210
211 embedBlock := &EmbedBlock{
212 Block: &Block{
213 Markdown: content,
214 },
215 }
216
217 updateEmbedBlockContent(id, []*EmbedBlock{embedBlock})
218 return
219}
220
221func GetEmbedBlock(embedBlockID string, includeIDs []string, headingMode int, breadcrumb bool) (ret []*EmbedBlock) {
222 return getEmbedBlock(embedBlockID, includeIDs, headingMode, breadcrumb)
223}
224
225func getEmbedBlock(embedBlockID string, includeIDs []string, headingMode int, breadcrumb bool) (ret []*EmbedBlock) {
226 stmt := "SELECT * FROM `blocks` WHERE `id` IN ('" + strings.Join(includeIDs, "','") + "')"
227 sqlBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 1024)
228
229 // 根据 includeIDs 的顺序排序 Improve `//!js` query embed block result sorting https://github.com/siyuan-note/siyuan/issues/9977
230 m := map[string]int{}
231 for i, id := range includeIDs {
232 m[id] = i
233 }
234 sort.Slice(sqlBlocks, func(i, j int) bool {
235 return m[sqlBlocks[i].ID] < m[sqlBlocks[j].ID]
236 })
237
238 ret = buildEmbedBlock(embedBlockID, []string{}, headingMode, breadcrumb, sqlBlocks)
239 return
240}
241
242func SearchEmbedBlock(embedBlockID, stmt string, excludeIDs []string, headingMode int, breadcrumb bool) (ret []*EmbedBlock) {
243 return searchEmbedBlock(embedBlockID, stmt, excludeIDs, headingMode, breadcrumb)
244}
245
246func searchEmbedBlock(embedBlockID, stmt string, excludeIDs []string, headingMode int, breadcrumb bool) (ret []*EmbedBlock) {
247 sqlBlocks := sql.SelectBlocksRawStmtNoParse(stmt, Conf.Search.Limit)
248 ret = buildEmbedBlock(embedBlockID, excludeIDs, headingMode, breadcrumb, sqlBlocks)
249 return
250}
251
252func buildEmbedBlock(embedBlockID string, excludeIDs []string, headingMode int, breadcrumb bool, sqlBlocks []*sql.Block) (ret []*EmbedBlock) {
253 var tmp []*sql.Block
254 for _, b := range sqlBlocks {
255 if "query_embed" == b.Type { // 嵌入块不再嵌入
256 // 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
257 // 这里会导致上面的 limit 限制不准确，导致结果变少，暂时没有解决方案，只能靠用户自己调整 SQL，加上 type != 'query_embed' 的条件
258 continue
259 }
260 if !gulu.Str.Contains(b.ID, excludeIDs) {
261 tmp = append(tmp, b)
262 }
263 }
264 sqlBlocks = tmp
265
266 // 缓存最多 128 棵语法树
267 trees := map[string]*parse.Tree{}
268 count := 0
269 for _, sb := range sqlBlocks {
270 if nil == trees[sb.RootID] {
271 tree, _ := LoadTreeByBlockID(sb.RootID)
272 if nil == tree {
273 continue
274 }
275 trees[sb.RootID] = tree
276 count++
277 }
278 if 127 < count {
279 break
280 }
281 }
282
283 for _, sb := range sqlBlocks {
284 block, blockPaths := getEmbeddedBlock(trees, sb, headingMode, breadcrumb)
285 if nil == block {
286 continue
287 }
288 ret = append(ret, &EmbedBlock{
289 Block: block,
290 BlockPaths: blockPaths,
291 })
292 }
293
294 // 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
295 task.AppendTaskWithTimeout(task.DatabaseIndexEmbedBlock, 30*time.Second, updateEmbedBlockContent, embedBlockID, ret)
296
297 // 添加笔记本名称
298 var boxIDs []string
299 for _, embedBlock := range ret {
300 boxIDs = append(boxIDs, embedBlock.Block.Box)
301 }
302 boxIDs = gulu.Str.RemoveDuplicatedElem(boxIDs)
303 boxNames := Conf.BoxNames(boxIDs)
304 for _, embedBlock := range ret {
305 name := boxNames[embedBlock.Block.Box]
306 embedBlock.Block.HPath = name + embedBlock.Block.HPath
307 }
308
309 if 1 > len(ret) {
310 ret = []*EmbedBlock{}
311 }
312 return
313}
314
315func SearchRefBlock(id, rootID, keyword string, beforeLen int, isSquareBrackets, isDatabase bool) (ret []*Block, newDoc bool) {
316 cachedTrees := map[string]*parse.Tree{}
317 nodeTrees := map[string]*parse.Tree{}
318 var nodeIDs []string
319 var nodes []*ast.Node
320
321 onlyDoc := false
322 if isSquareBrackets {
323 onlyDoc = Conf.Editor.OnlySearchForDoc
324 }
325
326 if "" == keyword {
327 // 查询为空时默认的块引排序规则按最近使用优先 https://github.com/siyuan-note/siyuan/issues/3218
328
329 typeFilter := Conf.Search.TypeFilter()
330 ignoreLines := getRefSearchIgnoreLines()
331 refs := sql.QueryRefsRecent(onlyDoc, typeFilter, ignoreLines)
332 var btsID []string
333 for _, ref := range refs {
334 btsID = append(btsID, ref.DefBlockRootID)
335 }
336 btsID = gulu.Str.RemoveDuplicatedElem(btsID)
337 bts := treenode.GetBlockTrees(btsID)
338
339 for _, ref := range refs {
340 tree := cachedTrees[ref.DefBlockRootID]
341 if nil == tree {
342 tree, _ = loadTreeByBlockTree(bts[ref.DefBlockRootID])
343 }
344 if nil == tree {
345 continue
346 }
347 cachedTrees[ref.RootID] = tree
348
349 node := treenode.GetNodeInTree(tree, ref.DefBlockID)
350 if nil == node {
351 continue
352 }
353
354 nodes = append(nodes, node)
355 nodeIDs = append(nodeIDs, node.ID)
356 nodeTrees[node.ID] = tree
357 }
358
359 refCount := sql.QueryRefCount(nodeIDs)
360
361 for _, node := range nodes {
362 tree := nodeTrees[node.ID]
363 sqlBlock := sql.BuildBlockFromNode(node, tree)
364 if nil == sqlBlock {
365 return
366 }
367
368 block := fromSQLBlock(sqlBlock, "", 0)
369 block.RefText = getNodeRefText(node)
370 block.RefText = maxContent(block.RefText, Conf.Editor.BlockRefDynamicAnchorTextMaxLen)
371 block.RefCount = refCount[node.ID]
372 ret = append(ret, block)
373 }
374
375 if 1 > len(ret) {
376 ret = []*Block{}
377 }
378
379 prependNotebookNameInHPath(ret)
380 filterSelfHPath(ret)
381 return
382 }
383
384 ret = fullTextSearchRefBlock(keyword, beforeLen, onlyDoc)
385 tmp := ret[:0]
386 var btsID []string
387 for _, b := range ret {
388 btsID = append(btsID, b.RootID)
389 }
390 btsID = gulu.Str.RemoveDuplicatedElem(btsID)
391 bts := treenode.GetBlockTrees(btsID)
392 for _, b := range ret {
393 tree := cachedTrees[b.RootID]
394 if nil == tree {
395 tree, _ = loadTreeByBlockTree(bts[b.RootID])
396 }
397 if nil == tree {
398 continue
399 }
400 cachedTrees[b.RootID] = tree
401 b.RefText = getBlockRefText(b.ID, tree)
402
403 hitFirstChildID := false
404 if b.IsContainerBlock() && "NodeDocument" != b.Type {
405 // `((` 引用候选中排除当前块的父块 https://github.com/siyuan-note/siyuan/issues/4538
406 tree = cachedTrees[b.RootID]
407 if nil == tree {
408 tree, _ = loadTreeByBlockTree(bts[b.RootID])
409 cachedTrees[b.RootID] = tree
410 }
411 if nil != tree {
412 bNode := treenode.GetNodeInTree(tree, b.ID)
413 if fc := treenode.FirstLeafBlock(bNode); nil != fc && fc.ID == id {
414 hitFirstChildID = true
415 }
416 }
417 }
418
419 if "NodeAttributeView" == b.Type {
420 // 数据库块可以添加到自身数据库块中，当前文档也可以添加到自身数据库块中
421 tmp = append(tmp, b)
422 nodeIDs = append(nodeIDs, b.ID)
423 nodeTrees[b.ID] = tree
424 } else {
425 // 排除自身块、父块和根块
426 if b.ID != id && !hitFirstChildID && b.ID != rootID {
427 tmp = append(tmp, b)
428 nodeIDs = append(nodeIDs, b.ID)
429 nodeTrees[b.ID] = tree
430 }
431 }
432
433 }
434 ret = tmp
435
436 refCount := sql.QueryRefCount(nodeIDs)
437 for _, b := range ret {
438 b.RefCount = refCount[b.ID]
439 }
440
441 if !isDatabase {
442 // 如果非数据库中搜索块引，则不允许新建重名文档
443 if block := treenode.GetBlockTree(id); nil != block {
444 p := path.Join(block.HPath, keyword)
445 newDoc = nil == treenode.GetBlockTreeRootByHPath(block.BoxID, p)
446 }
447 } else { // 如果是数据库中搜索绑定块，则允许新建重名文档 https://github.com/siyuan-note/siyuan/issues/11713
448 newDoc = true
449 }
450
451 prependNotebookNameInHPath(ret)
452 filterSelfHPath(ret)
453 return
454}
455
456func filterSelfHPath(blocks []*Block) {
457 // 简化搜索结果列表中的文档块路径 Simplify document block paths in search results https://github.com/siyuan-note/siyuan/issues/13364
458 // 文档块不显示自己的路径（最后一层）
459
460 for _, b := range blocks {
461 if b.IsDoc() {
462 b.HPath = strings.TrimSuffix(b.HPath, path.Base(b.HPath))
463 }
464 }
465}
466
467func prependNotebookNameInHPath(blocks []*Block) {
468 // 在 hPath 中加入笔记本名 Show notebooks in hpath of block ref search list results https://github.com/siyuan-note/siyuan/issues/9378
469
470 var boxIDs []string
471 for _, b := range blocks {
472 boxIDs = append(boxIDs, b.Box)
473 }
474 boxIDs = gulu.Str.RemoveDuplicatedElem(boxIDs)
475 boxNames := Conf.BoxNames(boxIDs)
476 for _, b := range blocks {
477 name := boxNames[b.Box]
478 b.HPath = util.EscapeHTML(name) + b.HPath
479 }
480}
481
482func FindReplace(keyword, replacement string, replaceTypes map[string]bool, ids []string, paths, boxes []string, types map[string]bool, method, orderBy, groupBy int) (err error) {
483 // method：0：文本，1：查询语法，2：SQL，3：正则表达式
484 if 2 == method {
485 err = errors.New(Conf.Language(132))
486 return
487 }
488
489 if 1 == method {
490 // 将查询语法等价于关键字，因为 keyword 参数已经是结果关键字了
491 // Find and replace supports query syntax https://github.com/siyuan-note/siyuan/issues/14937
492 method = 0
493 }
494
495 if 0 != groupBy {
496 // 按文档分组后不支持替换 Need to be reminded that replacement operations are not supported after grouping by doc https://github.com/siyuan-note/siyuan/issues/10161
497 // 因为分组条件传入以后搜索只能命中文档块，会导致 全部替换 失效
498 err = errors.New(Conf.Language(221))
499 return
500 }
501
502 // No longer trim spaces for the keyword and replacement https://github.com/siyuan-note/siyuan/issues/9229
503 if keyword == replacement {
504 return
505 }
506
507 r, _ := regexp.Compile(keyword)
508 escapedKey := util.EscapeHTML(keyword)
509 escapedKey = strings.ReplaceAll(escapedKey, """, """)
510 escapedKey = strings.ReplaceAll(escapedKey, "'", "'")
511 escapedR, _ := regexp.Compile(escapedKey)
512 ids = gulu.Str.RemoveDuplicatedElem(ids)
513 var renameRoots []*ast.Node
514 renameRootTitles := map[string]string{}
515 cachedTrees := map[string]*parse.Tree{}
516
517 historyDir, err := getHistoryDir(HistoryOpReplace, time.Now())
518 if err != nil {
519 logging.LogErrorf("get history dir failed: %s", err)
520 return
521 }
522
523 if 1 > len(ids) {
524 // `Replace All` is no longer affected by pagination https://github.com/siyuan-note/siyuan/issues/8265
525 blocks, _, _, _, _ := FullTextSearchBlock(keyword, boxes, paths, types, method, orderBy, groupBy, 1, math.MaxInt)
526 for _, block := range blocks {
527 ids = append(ids, block.ID)
528 }
529 }
530
531 for _, id := range ids {
532 bt := treenode.GetBlockTree(id)
533 if nil == bt {
534 continue
535 }
536
537 tree := cachedTrees[bt.RootID]
538 if nil != tree {
539 continue
540 }
541
542 tree, _ = LoadTreeByBlockID(id)
543 if nil == tree {
544 continue
545 }
546
547 historyPath := filepath.Join(historyDir, tree.Box, tree.Path)
548 if err = os.MkdirAll(filepath.Dir(historyPath), 0755); err != nil {
549 logging.LogErrorf("generate history failed: %s", err)
550 return
551 }
552
553 var data []byte
554 if data, err = filelock.ReadFile(filepath.Join(util.DataDir, tree.Box, tree.Path)); err != nil {
555 logging.LogErrorf("generate history failed: %s", err)
556 return
557 }
558
559 if err = gulu.File.WriteFileSafer(historyPath, data, 0644); err != nil {
560 logging.LogErrorf("generate history failed: %s", err)
561 return
562 }
563
564 cachedTrees[bt.RootID] = tree
565 }
566 indexHistoryDir(filepath.Base(historyDir), util.NewLute())
567
568 luteEngine := util.NewLute()
569 var reloadTreeIDs []string
570 updateNodes := map[string]*ast.Node{}
571 for i, id := range ids {
572 bt := treenode.GetBlockTree(id)
573 if nil == bt {
574 continue
575 }
576
577 tree := cachedTrees[bt.RootID]
578 if nil == tree {
579 continue
580 }
581
582 node := treenode.GetNodeInTree(tree, id)
583 if nil == node {
584 continue
585 }
586
587 reloadTreeIDs = append(reloadTreeIDs, tree.ID)
588 if ast.NodeDocument == node.Type {
589 if !replaceTypes["docTitle"] {
590 continue
591 }
592
593 title := node.IALAttr("title")
594 tags := node.IALAttr("tags")
595 if 0 == method {
596 if strings.Contains(title, keyword) {
597 docTitleReplacement := strings.ReplaceAll(replacement, "/", "／")
598 renameRootTitles[node.ID] = strings.ReplaceAll(title, keyword, docTitleReplacement)
599 renameRoots = append(renameRoots, node)
600 }
601
602 if strings.Contains(tags, keyword) {
603 replacement = strings.TrimPrefix(replacement, "#")
604 replacement = strings.TrimSuffix(replacement, "#")
605 tags = strings.ReplaceAll(tags, keyword, replacement)
606 tags = strings.ReplaceAll(tags, editor.Zwsp, "")
607 node.SetIALAttr("tags", tags)
608 ReloadTag()
609 }
610 } else if 3 == method {
611 if nil != r && r.MatchString(title) {
612 docTitleReplacement := strings.ReplaceAll(replacement, "/", "／")
613 renameRootTitles[node.ID] = r.ReplaceAllString(title, docTitleReplacement)
614 renameRoots = append(renameRoots, node)
615 }
616
617 if nil != r && r.MatchString(tags) {
618 replacement = strings.TrimPrefix(replacement, "#")
619 replacement = strings.TrimSuffix(replacement, "#")
620 tags = r.ReplaceAllString(tags, replacement)
621 tags = strings.ReplaceAll(tags, editor.Zwsp, "")
622 node.SetIALAttr("tags", tags)
623 ReloadTag()
624 }
625 }
626 } else {
627 var unlinks []*ast.Node
628 ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus {
629 if !entering {
630 return ast.WalkContinue
631 }
632
633 switch n.Type {
634 case ast.NodeText:
635 if !replaceTypes["text"] {
636 return ast.WalkContinue
637 }
638
639 if replaceTextNode(n, method, keyword, replacement, r, luteEngine) {
640 if nil != n.Parent && ast.NodeBackslash == n.Parent.Type {
641 unlinks = append(unlinks, n.Parent)
642
643 prev, next := n.Parent.Previous, n.Parent.Next
644 for ; prev != nil && ((ast.NodeText == prev.Type && prev.Tokens == nil) || ast.NodeBackslash == prev.Type); prev = prev.Previous {
645 // Tokens 为空的节点或者转义节点之前已经处理，需要跳过
646 }
647 if nil != prev && ast.NodeText == prev.Type && nil != next && ast.NodeText == next.Type {
648 prev.Tokens = append(prev.Tokens, next.Tokens...)
649 next.Tokens = nil // 将 Tokens 设置为空，表示该节点已经被处理过
650 unlinks = append(unlinks, next)
651 }
652 } else {
653 unlinks = append(unlinks, n)
654 }
655 }
656 case ast.NodeLinkDest:
657 if !replaceTypes["imgSrc"] {
658 return ast.WalkContinue
659 }
660
661 replaceNodeTokens(n, method, keyword, strings.TrimSpace(replacement), r)
662 if 1 > len(n.Tokens) {
663 unlinks = append(unlinks, n.Parent)
664 mergeSamePreNext(n)
665 }
666 case ast.NodeLinkText:
667 if !replaceTypes["imgText"] {
668 return ast.WalkContinue
669 }
670
671 replaceNodeTokens(n, method, keyword, replacement, r)
672 case ast.NodeLinkTitle:
673 if !replaceTypes["imgTitle"] {
674 return ast.WalkContinue
675 }
676
677 replaceNodeTokens(n, method, keyword, replacement, r)
678 case ast.NodeCodeBlockCode:
679 if !replaceTypes["codeBlock"] {
680 return ast.WalkContinue
681 }
682
683 replaceNodeTokens(n, method, keyword, replacement, r)
684 case ast.NodeMathBlockContent:
685 if !replaceTypes["mathBlock"] {
686 return ast.WalkContinue
687 }
688
689 replaceNodeTokens(n, method, keyword, replacement, r)
690 case ast.NodeHTMLBlock:
691 if !replaceTypes["htmlBlock"] {
692 return ast.WalkContinue
693 }
694
695 replaceNodeTokens(n, method, keyword, replacement, r)
696 case ast.NodeTextMark:
697 if n.IsTextMarkType("code") {
698 if !replaceTypes["code"] {
699 return ast.WalkContinue
700 }
701
702 if 0 == method {
703 if strings.Contains(n.TextMarkTextContent, escapedKey) {
704 n.TextMarkTextContent = strings.ReplaceAll(n.TextMarkTextContent, escapedKey, util.EscapeHTML(replacement))
705 }
706 } else if 3 == method {
707 if nil != escapedR && escapedR.MatchString(n.TextMarkTextContent) {
708 n.TextMarkTextContent = escapedR.ReplaceAllString(n.TextMarkTextContent, util.EscapeHTML(replacement))
709 }
710 }
711
712 if "" == n.TextMarkTextContent {
713 unlinks = append(unlinks, n)
714 mergeSamePreNext(n)
715 }
716 } else if n.IsTextMarkType("a") {
717 if replaceTypes["aText"] {
718 if 0 == method {
719 content := util.UnescapeHTML(n.TextMarkTextContent)
720 if strings.Contains(content, escapedKey) {
721 n.TextMarkTextContent = strings.ReplaceAll(content, escapedKey, replacement)
722 } else if strings.Contains(content, keyword) {
723 n.TextMarkTextContent = strings.ReplaceAll(content, keyword, replacement)
724 }
725 } else if 3 == method {
726 if nil != r && r.MatchString(n.TextMarkTextContent) {
727 n.TextMarkTextContent = r.ReplaceAllString(n.TextMarkTextContent, replacement)
728 }
729 }
730 if "" == n.TextMarkTextContent {
731 unlinks = append(unlinks, n)
732 mergeSamePreNext(n)
733 }
734 }
735
736 if replaceTypes["aTitle"] {
737 if 0 == method {
738 title := util.UnescapeHTML(n.TextMarkATitle)
739 if strings.Contains(title, escapedKey) {
740 n.TextMarkATitle = strings.ReplaceAll(title, escapedKey, replacement)
741 } else if strings.Contains(n.TextMarkATitle, keyword) {
742 n.TextMarkATitle = strings.ReplaceAll(title, keyword, replacement)
743 }
744 } else if 3 == method {
745 if nil != r && r.MatchString(n.TextMarkATitle) {
746 n.TextMarkATitle = r.ReplaceAllString(n.TextMarkATitle, replacement)
747 }
748 }
749 }
750
751 if replaceTypes["aHref"] {
752 if 0 == method {
753 href := util.UnescapeHTML(n.TextMarkAHref)
754 if strings.Contains(href, escapedKey) {
755 n.TextMarkAHref = strings.ReplaceAll(href, escapedKey, util.EscapeHTML(replacement))
756 } else if strings.Contains(href, keyword) {
757 n.TextMarkAHref = strings.ReplaceAll(href, keyword, strings.TrimSpace(replacement))
758 }
759 } else if 3 == method {
760 if nil != r && r.MatchString(n.TextMarkAHref) {
761 n.TextMarkAHref = r.ReplaceAllString(n.TextMarkAHref, strings.TrimSpace(replacement))
762 }
763 }
764
765 if "" == n.TextMarkAHref {
766 if "" == n.TextMarkTextContent {
767 unlinks = append(unlinks, n)
768 mergeSamePreNext(n)
769 } else {
770 n.Type = ast.NodeText
771 n.Tokens = []byte(n.TextMarkTextContent)
772 }
773 }
774 }
775 } else if n.IsTextMarkType("em") {
776 if !replaceTypes["em"] {
777 return ast.WalkContinue
778 }
779
780 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "em", luteEngine)
781 if "" == n.TextMarkTextContent {
782 unlinks = append(unlinks, n)
783 mergeSamePreNext(n)
784 }
785 } else if n.IsTextMarkType("strong") {
786 if !replaceTypes["strong"] {
787 return ast.WalkContinue
788 }
789
790 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "strong", luteEngine)
791 if "" == n.TextMarkTextContent {
792 unlinks = append(unlinks, n)
793 mergeSamePreNext(n)
794 }
795 } else if n.IsTextMarkType("kbd") {
796 if !replaceTypes["kbd"] {
797 return ast.WalkContinue
798 }
799
800 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "kbd", luteEngine)
801 if "" == n.TextMarkTextContent {
802 unlinks = append(unlinks, n)
803 }
804 } else if n.IsTextMarkType("mark") {
805 if !replaceTypes["mark"] {
806 return ast.WalkContinue
807 }
808
809 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "mark", luteEngine)
810 if "" == n.TextMarkTextContent {
811 unlinks = append(unlinks, n)
812 mergeSamePreNext(n)
813 }
814 } else if n.IsTextMarkType("s") {
815 if !replaceTypes["s"] {
816 return ast.WalkContinue
817 }
818
819 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "s", luteEngine)
820 if "" == n.TextMarkTextContent {
821 unlinks = append(unlinks, n)
822 mergeSamePreNext(n)
823 }
824 } else if n.IsTextMarkType("sub") {
825 if !replaceTypes["sub"] {
826 return ast.WalkContinue
827 }
828
829 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "sub", luteEngine)
830 if "" == n.TextMarkTextContent {
831 unlinks = append(unlinks, n)
832 }
833 } else if n.IsTextMarkType("sup") {
834 if !replaceTypes["sup"] {
835 return ast.WalkContinue
836 }
837
838 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "sup", luteEngine)
839 if "" == n.TextMarkTextContent {
840 unlinks = append(unlinks, n)
841 }
842 } else if n.IsTextMarkType("tag") {
843 if !replaceTypes["tag"] {
844 return ast.WalkContinue
845 }
846
847 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "tag", luteEngine)
848 if "" == n.TextMarkTextContent {
849 unlinks = append(unlinks, n)
850 }
851
852 ReloadTag()
853 } else if n.IsTextMarkType("u") {
854 if !replaceTypes["u"] {
855 return ast.WalkContinue
856 }
857
858 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "u", luteEngine)
859 if "" == n.TextMarkTextContent {
860 unlinks = append(unlinks, n)
861 mergeSamePreNext(n)
862 }
863 } else if n.IsTextMarkType("inline-math") {
864 if !replaceTypes["inlineMath"] {
865 return ast.WalkContinue
866 }
867
868 if 0 == method {
869 if strings.Contains(n.TextMarkInlineMathContent, keyword) {
870 n.TextMarkInlineMathContent = strings.ReplaceAll(n.TextMarkInlineMathContent, keyword, replacement)
871 }
872 } else if 3 == method {
873 if nil != r && r.MatchString(n.TextMarkInlineMathContent) {
874 n.TextMarkInlineMathContent = r.ReplaceAllString(n.TextMarkInlineMathContent, replacement)
875 }
876 }
877
878 if "" == n.TextMarkInlineMathContent {
879 unlinks = append(unlinks, n)
880 }
881 } else if n.IsTextMarkType("inline-memo") {
882 if !replaceTypes["inlineMemo"] {
883 return ast.WalkContinue
884 }
885
886 if 0 == method {
887 if strings.Contains(n.TextMarkInlineMemoContent, keyword) {
888 n.TextMarkInlineMemoContent = strings.ReplaceAll(n.TextMarkInlineMemoContent, keyword, replacement)
889 n.TextMarkTextContent = strings.ReplaceAll(n.TextMarkTextContent, keyword, replacement)
890 }
891 } else if 3 == method {
892 if nil != r && r.MatchString(n.TextMarkInlineMemoContent) {
893 n.TextMarkInlineMemoContent = r.ReplaceAllString(n.TextMarkInlineMemoContent, replacement)
894 n.TextMarkTextContent = r.ReplaceAllString(n.TextMarkTextContent, replacement)
895 }
896 }
897
898 if "" == n.TextMarkTextContent {
899 unlinks = append(unlinks, n)
900 }
901 } else if n.IsTextMarkType("text") {
902 // Search and replace fails in some cases https://github.com/siyuan-note/siyuan/issues/10016
903 if !replaceTypes["text"] {
904 return ast.WalkContinue
905 }
906
907 replaceNodeTextMarkTextContent(n, method, keyword, escapedKey, replacement, r, "text", luteEngine)
908 if "" == n.TextMarkTextContent {
909 unlinks = append(unlinks, n)
910 mergeSamePreNext(n)
911 }
912 } else if n.IsTextMarkType("block-ref") {
913 if !replaceTypes["blockRef"] {
914 return ast.WalkContinue
915 }
916
917 if 0 == method {
918 if strings.Contains(n.TextMarkTextContent, keyword) {
919 n.TextMarkTextContent = strings.ReplaceAll(n.TextMarkTextContent, keyword, replacement)
920 n.TextMarkBlockRefSubtype = "s"
921 }
922 } else if 3 == method {
923 if nil != r && r.MatchString(n.TextMarkTextContent) {
924 n.TextMarkTextContent = r.ReplaceAllString(n.TextMarkTextContent, replacement)
925 n.TextMarkBlockRefSubtype = "s"
926 }
927 }
928
929 if "" == n.TextMarkTextContent {
930 unlinks = append(unlinks, n)
931 }
932 } else if n.IsTextMarkType("file-annotation-ref") {
933 if !replaceTypes["fileAnnotationRef"] {
934 return ast.WalkContinue
935 }
936
937 if 0 == method {
938 if strings.Contains(n.TextMarkTextContent, keyword) {
939 n.TextMarkTextContent = strings.ReplaceAll(n.TextMarkTextContent, keyword, replacement)
940 }
941 } else if 3 == method {
942 if nil != r && r.MatchString(n.TextMarkTextContent) {
943 n.TextMarkTextContent = r.ReplaceAllString(n.TextMarkTextContent, replacement)
944 }
945 }
946 if "" == n.TextMarkTextContent {
947 unlinks = append(unlinks, n)
948 }
949 }
950 }
951 return ast.WalkContinue
952 })
953
954 for _, unlink := range unlinks {
955 unlink.Unlink()
956 }
957 }
958
959 if err = writeTreeUpsertQueue(tree); err != nil {
960 return
961 }
962 updateNodes[id] = node
963 util.PushEndlessProgress(fmt.Sprintf(Conf.Language(206), i+1, len(ids)))
964 }
965
966 for i, renameRoot := range renameRoots {
967 newTitle := renameRootTitles[renameRoot.ID]
968 RenameDoc(renameRoot.Box, renameRoot.Path, newTitle)
969
970 util.PushEndlessProgress(fmt.Sprintf(Conf.Language(207), i+1, len(renameRoots)))
971 }
972
973 sql.FlushQueue()
974
975 reloadTreeIDs = gulu.Str.RemoveDuplicatedElem(reloadTreeIDs)
976 for _, id := range reloadTreeIDs {
977 ReloadProtyle(id)
978 }
979
980 updateAttributeViewBlockText(updateNodes)
981
982 sql.FlushQueue()
983 util.PushClearProgress()
984 return
985}
986
987func replaceNodeTextMarkTextContent(n *ast.Node, method int, keyword, escapedKey string, replacement string, r *regexp.Regexp, typ string, luteEngine *lute.Lute) {
988 if 0 == method {
989 if strings.Contains(typ, "tag") {
990 keyword = strings.TrimPrefix(keyword, "#")
991 keyword = strings.TrimSuffix(keyword, "#")
992 escapedKey = strings.TrimPrefix(escapedKey, "#")
993 escapedKey = strings.TrimSuffix(escapedKey, "#")
994 if strings.HasPrefix(replacement, "#") && strings.HasSuffix(replacement, "#") {
995 replacement = strings.TrimPrefix(replacement, "#")
996 replacement = strings.TrimSuffix(replacement, "#")
997 } else if n.TextMarkTextContent == keyword || n.TextMarkTextContent == escapedKey {
998 // 将标签转换为纯文本
999
1000 if "tag" == n.TextMarkType { // 没有其他类型，仅是标签时直接转换
1001 content := n.TextMarkTextContent
1002 if strings.Contains(content, escapedKey) {
1003 content = strings.ReplaceAll(content, escapedKey, replacement)
1004 } else if strings.Contains(content, keyword) {
1005 content = strings.ReplaceAll(content, keyword, replacement)
1006 }
1007 content = strings.ReplaceAll(content, editor.Zwsp, "")
1008
1009 tree := parse.Inline("", []byte(content), luteEngine.ParseOptions)
1010 if nil == tree.Root.FirstChild {
1011 return
1012 }
1013 parse.NestedInlines2FlattedSpans(tree, false)
1014
1015 var replaceNodes []*ast.Node
1016 for rNode := tree.Root.FirstChild.FirstChild; nil != rNode; rNode = rNode.Next {
1017 replaceNodes = append(replaceNodes, rNode)
1018 if blockRefID, _, _ := treenode.GetBlockRef(rNode); "" != blockRefID {
1019 task.AppendAsyncTaskWithDelay(task.SetDefRefCount, util.SQLFlushInterval, refreshRefCount, blockRefID)
1020 }
1021 }
1022
1023 for _, rNode := range replaceNodes {
1024 n.InsertBefore(rNode)
1025 }
1026 n.TextMarkTextContent = ""
1027 return
1028 }
1029
1030 // 存在其他类型时仅移除标签类型
1031 n.TextMarkType = strings.ReplaceAll(n.TextMarkType, "tag", "")
1032 n.TextMarkType = strings.TrimSpace(n.TextMarkType)
1033 } else if strings.Contains(n.TextMarkTextContent, keyword) || strings.Contains(n.TextMarkTextContent, escapedKey) { // 标签包含了部分关键字的情况
1034 if "tag" == n.TextMarkType { // 没有其他类型，仅是标签时保持标签类型不变，仅替换标签部分内容
1035 content := n.TextMarkTextContent
1036 if strings.Contains(content, escapedKey) {
1037 content = strings.ReplaceAll(content, escapedKey, replacement)
1038 } else if strings.Contains(content, keyword) {
1039 content = strings.ReplaceAll(content, keyword, replacement)
1040 }
1041 content = strings.ReplaceAll(content, editor.Zwsp, "")
1042 n.TextMarkTextContent = content
1043 return
1044 }
1045 }
1046 }
1047
1048 if strings.Contains(n.TextMarkTextContent, escapedKey) {
1049 n.TextMarkTextContent = strings.ReplaceAll(n.TextMarkTextContent, escapedKey, util.EscapeHTML(replacement))
1050 } else if strings.Contains(n.TextMarkTextContent, keyword) {
1051 n.TextMarkTextContent = strings.ReplaceAll(n.TextMarkTextContent, keyword, replacement)
1052 }
1053 n.TextMarkTextContent = strings.ReplaceAll(n.TextMarkTextContent, editor.Zwsp, "")
1054 } else if 3 == method {
1055 if nil != r && r.MatchString(n.TextMarkTextContent) {
1056 n.TextMarkTextContent = r.ReplaceAllString(n.TextMarkTextContent, replacement)
1057 }
1058 n.TextMarkTextContent = strings.ReplaceAll(n.TextMarkTextContent, editor.Zwsp, "")
1059 }
1060}
1061
1062// replaceTextNode 替换文本节点为其他节点。
1063// Supports replacing text elements with other elements https://github.com/siyuan-note/siyuan/issues/11058
1064func replaceTextNode(text *ast.Node, method int, keyword string, replacement string, r *regexp.Regexp, luteEngine *lute.Lute) bool {
1065 if 0 == method {
1066 newContent := text.Tokens
1067 if Conf.Search.CaseSensitive {
1068 if bytes.Contains(text.Tokens, []byte(keyword)) {
1069 newContent = bytes.ReplaceAll(text.Tokens, []byte(keyword), []byte(replacement))
1070 }
1071 } else {
1072 if "" != strings.TrimSpace(keyword) {
1073 // 当搜索结果中的文本元素包含大小写混合时替换失败
1074 // Replace fails when search results contain mixed case in text elements https://github.com/siyuan-note/siyuan/issues/9171
1075 keywords := strings.Split(keyword, " ")
1076 // keyword 可能是 "foo Foo" 使用空格分隔的大小写命中情况，这里统一转换小写后去重
1077 if 0 < len(keywords) {
1078 var lowerKeywords []string
1079 for _, k := range keywords {
1080 lowerKeywords = append(lowerKeywords, strings.ToLower(k))
1081 }
1082 keyword = strings.Join(lowerKeywords, " ")
1083 }
1084 }
1085
1086 if bytes.Contains(bytes.ToLower(text.Tokens), []byte(keyword)) {
1087 newContent = replaceCaseInsensitive(text.Tokens, []byte(keyword), []byte(replacement))
1088 }
1089 }
1090 if !bytes.Equal(newContent, text.Tokens) {
1091 tree := parse.Inline("", newContent, luteEngine.ParseOptions)
1092 if nil == tree.Root.FirstChild {
1093 return false
1094 }
1095 parse.NestedInlines2FlattedSpans(tree, false)
1096
1097 var replaceNodes []*ast.Node
1098 for rNode := tree.Root.FirstChild.FirstChild; nil != rNode; rNode = rNode.Next {
1099 replaceNodes = append(replaceNodes, rNode)
1100 }
1101
1102 for _, rNode := range replaceNodes {
1103 text.InsertBefore(rNode)
1104 }
1105 return true
1106 }
1107 } else if 3 == method {
1108 if nil != r && r.MatchString(string(text.Tokens)) {
1109 newContent := []byte(r.ReplaceAllString(string(text.Tokens), replacement))
1110 tree := parse.Inline("", newContent, luteEngine.ParseOptions)
1111 if nil == tree.Root.FirstChild {
1112 return false
1113 }
1114
1115 var replaceNodes []*ast.Node
1116 for rNode := tree.Root.FirstChild.FirstChild; nil != rNode; rNode = rNode.Next {
1117 replaceNodes = append(replaceNodes, rNode)
1118 }
1119
1120 for _, rNode := range replaceNodes {
1121 text.InsertBefore(rNode)
1122 }
1123 return true
1124 }
1125 }
1126 return false
1127}
1128
1129func replaceNodeTokens(n *ast.Node, method int, keyword string, replacement string, r *regexp.Regexp) {
1130 if 0 == method {
1131 if bytes.Contains(n.Tokens, []byte(keyword)) {
1132 n.Tokens = bytes.ReplaceAll(n.Tokens, []byte(keyword), []byte(replacement))
1133 }
1134 } else if 3 == method {
1135 if nil != r && r.MatchString(string(n.Tokens)) {
1136 n.Tokens = []byte(r.ReplaceAllString(string(n.Tokens), replacement))
1137 }
1138 }
1139}
1140
1141func mergeSamePreNext(n *ast.Node) {
1142 prev, next := n.Previous, n.Next
1143 if nil != n.Parent && ast.NodeImage == n.Parent.Type {
1144 prev = n.Parent.Previous
1145 next = n.Parent.Next
1146 }
1147
1148 if nil == prev || nil == next || prev.Type != next.Type || ast.NodeKramdownSpanIAL == prev.Type {
1149 return
1150 }
1151
1152 switch prev.Type {
1153 case ast.NodeText:
1154 prev.Tokens = append(prev.Tokens, next.Tokens...)
1155 next.Unlink()
1156 case ast.NodeTextMark:
1157 if prev.TextMarkType != next.TextMarkType {
1158 break
1159 }
1160
1161 switch prev.TextMarkType {
1162 case "em", "strong", "mark", "s", "u", "text":
1163 prev.TextMarkTextContent += next.TextMarkTextContent
1164 next.Unlink()
1165 }
1166 }
1167}
1168
1169// FullTextSearchBlock 搜索内容块。
1170//
1171// method：0：关键字，1：查询语法，2：SQL，3：正则表达式
1172// orderBy: 0：按块类型（默认），1：按创建时间升序，2：按创建时间降序，3：按更新时间升序，4：按更新时间降序，5：按内容顺序（仅在按文档分组时），6：按相关度升序，7：按相关度降序
1173// groupBy：0：不分组，1：按文档分组
1174func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method, orderBy, groupBy, page, pageSize int) (ret []*Block, matchedBlockCount, matchedRootCount, pageCount int, docMode bool) {
1175 ret = []*Block{}
1176 if "" == query {
1177 return
1178 }
1179
1180 query = filterQueryInvisibleChars(query)
1181 var ignoreFilter string
1182 if ignoreLines := getSearchIgnoreLines(); 0 < len(ignoreLines) {
1183 // Support ignore search results https://github.com/siyuan-note/siyuan/issues/10089
1184 buf := bytes.Buffer{}
1185 for _, line := range ignoreLines {
1186 buf.WriteString(" AND ")
1187 buf.WriteString(line)
1188 }
1189 ignoreFilter += buf.String()
1190 }
1191
1192 beforeLen := 36
1193 var blocks []*Block
1194 orderByClause := buildOrderBy(query, method, orderBy)
1195 switch method {
1196 case 1: // 查询语法
1197 typeFilter := buildTypeFilter(types)
1198 boxFilter := buildBoxesFilter(boxes)
1199 pathFilter := buildPathsFilter(paths)
1200 if ast.IsNodeIDPattern(query) {
1201 blocks, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen, page, pageSize)
1202 } else {
1203 blocks, matchedBlockCount, matchedRootCount = fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, ignoreFilter, orderByClause, beforeLen, page, pageSize)
1204 }
1205 case 2: // SQL
1206 blocks, matchedBlockCount, matchedRootCount = searchBySQL(query, beforeLen, page, pageSize)
1207 case 3: // 正则表达式
1208 typeFilter := buildTypeFilter(types)
1209 boxFilter := buildBoxesFilter(boxes)
1210 pathFilter := buildPathsFilter(paths)
1211 blocks, matchedBlockCount, matchedRootCount = fullTextSearchByRegexp(query, boxFilter, pathFilter, typeFilter, ignoreFilter, orderByClause, beforeLen, page, pageSize)
1212 default: // 关键字
1213 typeFilter := buildTypeFilter(types)
1214 boxFilter := buildBoxesFilter(boxes)
1215 pathFilter := buildPathsFilter(paths)
1216 if ast.IsNodeIDPattern(query) {
1217 blocks, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen, page, pageSize)
1218 } else {
1219 if 2 > len(strings.Split(strings.TrimSpace(query), " ")) {
1220 query = stringQuery(query)
1221 blocks, matchedBlockCount, matchedRootCount = fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, ignoreFilter, orderByClause, beforeLen, page, pageSize)
1222 } else {
1223 docMode = true // 文档全文搜索模式 https://github.com/siyuan-note/siyuan/issues/10584
1224 blocks, matchedBlockCount, matchedRootCount = fullTextSearchByLikeWithRoot(query, boxFilter, pathFilter, typeFilter, ignoreFilter, orderByClause, beforeLen, page, pageSize)
1225 }
1226 }
1227 }
1228 pageCount = (matchedBlockCount + pageSize - 1) / pageSize
1229
1230 switch groupBy {
1231 case 0: // 不分组
1232 ret = blocks
1233 case 1: // 按文档分组
1234 rootMap := map[string]bool{}
1235 var rootIDs []string
1236 contentSorts := map[string]int{}
1237 var btsID []string
1238 for _, b := range blocks {
1239 btsID = append(btsID, b.RootID)
1240 }
1241 btsID = gulu.Str.RemoveDuplicatedElem(btsID)
1242 bts := treenode.GetBlockTrees(btsID)
1243 for _, b := range blocks {
1244 if _, ok := rootMap[b.RootID]; !ok {
1245 rootMap[b.RootID] = true
1246 rootIDs = append(rootIDs, b.RootID)
1247 tree, _ := loadTreeByBlockTree(bts[b.RootID])
1248 if nil == tree {
1249 continue
1250 }
1251
1252 if 5 == orderBy { // 按内容顺序（仅在按文档分组时）
1253 sortVal := 0
1254 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
1255 if !entering || !n.IsBlock() {
1256 return ast.WalkContinue
1257 }
1258
1259 contentSorts[n.ID] = sortVal
1260 sortVal++
1261 return ast.WalkContinue
1262 })
1263 }
1264 }
1265 }
1266
1267 sqlRoots := sql.GetBlocks(rootIDs)
1268 roots := fromSQLBlocks(&sqlRoots, "", beforeLen)
1269 for _, root := range roots {
1270 for _, b := range blocks {
1271 if 5 == orderBy { // 按内容顺序（仅在按文档分组时）
1272 b.Sort = contentSorts[b.ID]
1273 }
1274 if b.RootID == root.ID {
1275 root.Children = append(root.Children, b)
1276 }
1277 }
1278
1279 switch orderBy {
1280 case 1: //按创建时间升序
1281 sort.Slice(root.Children, func(i, j int) bool { return root.Children[i].Created < root.Children[j].Created })
1282 case 2: // 按创建时间降序
1283 sort.Slice(root.Children, func(i, j int) bool { return root.Children[i].Created > root.Children[j].Created })
1284 case 3: // 按更新时间升序
1285 sort.Slice(root.Children, func(i, j int) bool { return root.Children[i].Updated < root.Children[j].Updated })
1286 case 4: // 按更新时间降序
1287 sort.Slice(root.Children, func(i, j int) bool { return root.Children[i].Updated > root.Children[j].Updated })
1288 case 5: // 按内容顺序（仅在按文档分组时）
1289 sort.Slice(root.Children, func(i, j int) bool { return root.Children[i].Sort < root.Children[j].Sort })
1290 default: // 按块类型（默认）
1291 sort.Slice(root.Children, func(i, j int) bool { return root.Children[i].Sort < root.Children[j].Sort })
1292 }
1293 }
1294
1295 switch orderBy {
1296 case 1: //按创建时间升序
1297 sort.Slice(roots, func(i, j int) bool { return roots[i].Created < roots[j].Created })
1298 case 2: // 按创建时间降序
1299 sort.Slice(roots, func(i, j int) bool { return roots[i].Created > roots[j].Created })
1300 case 3: // 按更新时间升序
1301 sort.Slice(roots, func(i, j int) bool { return roots[i].Updated < roots[j].Updated })
1302 case 4: // 按更新时间降序
1303 sort.Slice(roots, func(i, j int) bool { return roots[i].Updated > roots[j].Updated })
1304 case 5: // 按内容顺序（仅在按文档分组时）
1305 // 都是文档，按更新时间降序
1306 sort.Slice(roots, func(i, j int) bool { return roots[i].IAL["updated"] > roots[j].IAL["updated"] })
1307 case 6, 7: // 按相关度
1308 // 已在 ORDER BY 中处理
1309 default: // 按块类型（默认）
1310 // 都是文档，不需要再次排序
1311 }
1312 ret = roots
1313 default:
1314 ret = blocks
1315 }
1316 if 1 > len(ret) {
1317 ret = []*Block{}
1318 }
1319
1320 if 0 == groupBy {
1321 filterSelfHPath(ret)
1322 }
1323
1324 var nodeIDs []string
1325 for _, b := range ret {
1326 if 0 == groupBy {
1327 nodeIDs = append(nodeIDs, b.ID)
1328 } else {
1329 for _, c := range b.Children {
1330 nodeIDs = append(nodeIDs, c.ID)
1331 }
1332 }
1333 }
1334
1335 refCount := sql.QueryRefCount(nodeIDs)
1336 for _, b := range ret {
1337 if 0 == groupBy {
1338 b.RefCount = refCount[b.ID]
1339 } else {
1340 for _, c := range b.Children {
1341 c.RefCount = refCount[c.ID]
1342 }
1343 }
1344 }
1345 return
1346}
1347
1348func buildBoxesFilter(boxes []string) string {
1349 if 0 == len(boxes) {
1350 return ""
1351 }
1352 builder := bytes.Buffer{}
1353 builder.WriteString(" AND (")
1354 for i, box := range boxes {
1355 builder.WriteString(fmt.Sprintf("box = '%s'", box))
1356 if i < len(boxes)-1 {
1357 builder.WriteString(" OR ")
1358 }
1359 }
1360 builder.WriteString(")")
1361 return builder.String()
1362}
1363
1364func buildPathsFilter(paths []string) string {
1365 if 0 == len(paths) {
1366 return ""
1367 }
1368 builder := bytes.Buffer{}
1369 builder.WriteString(" AND (")
1370 for i, path := range paths {
1371 builder.WriteString(fmt.Sprintf("path LIKE '%s%%'", path))
1372 if i < len(paths)-1 {
1373 builder.WriteString(" OR ")
1374 }
1375 }
1376 builder.WriteString(")")
1377 return builder.String()
1378}
1379
1380func buildOrderBy(query string, method, orderBy int) string {
1381 switch orderBy {
1382 case 1:
1383 return "ORDER BY created ASC"
1384 case 2:
1385 return "ORDER BY created DESC"
1386 case 3:
1387 return "ORDER BY updated ASC"
1388 case 4:
1389 return "ORDER BY updated DESC"
1390 case 6:
1391 if 0 != method && 1 != method {
1392 // 只有关键字搜索和查询语法搜索才支持按相关度升序 https://github.com/siyuan-note/siyuan/issues/7861
1393 return "ORDER BY sort DESC, updated DESC"
1394 }
1395 return "ORDER BY rank DESC" // 默认是按相关度降序，所以按相关度升序要反过来使用 DESC
1396 case 7:
1397 if 0 != method && 1 != method {
1398 return "ORDER BY sort ASC, updated DESC"
1399 }
1400 return "ORDER BY rank" // 默认是按相关度降序
1401 default:
1402 clause := "ORDER BY CASE " +
1403 "WHEN name = '${keyword}' THEN 10 " +
1404 "WHEN alias = '${keyword}' THEN 20 " +
1405 "WHEN name LIKE '%${keyword}%' THEN 50 " +
1406 "WHEN alias LIKE '%${keyword}%' THEN 60 " +
1407 "ELSE 65535 END ASC, sort ASC, updated DESC"
1408 clause = strings.ReplaceAll(clause, "${keyword}", strings.ReplaceAll(query, "'", "''"))
1409 return clause
1410 }
1411}
1412
1413func buildTypeFilter(types map[string]bool) string {
1414 s := conf.NewSearch()
1415 if err := copier.Copy(s, Conf.Search); err != nil {
1416 logging.LogErrorf("copy search conf failed: %s", err)
1417 }
1418 if nil != types {
1419 s.Document = types["document"]
1420 s.Heading = types["heading"]
1421 s.List = types["list"]
1422 s.ListItem = types["listItem"]
1423 s.CodeBlock = types["codeBlock"]
1424 s.MathBlock = types["mathBlock"]
1425 s.Table = types["table"]
1426 s.Blockquote = types["blockquote"]
1427 s.SuperBlock = types["superBlock"]
1428 s.Paragraph = types["paragraph"]
1429 s.HTMLBlock = types["htmlBlock"]
1430 s.EmbedBlock = types["embedBlock"]
1431 s.DatabaseBlock = types["databaseBlock"]
1432 s.AudioBlock = types["audioBlock"]
1433 s.VideoBlock = types["videoBlock"]
1434 s.IFrameBlock = types["iframeBlock"]
1435 s.WidgetBlock = types["widgetBlock"]
1436 } else {
1437 s.Document = Conf.Search.Document
1438 s.Heading = Conf.Search.Heading
1439 s.List = Conf.Search.List
1440 s.ListItem = Conf.Search.ListItem
1441 s.CodeBlock = Conf.Search.CodeBlock
1442 s.MathBlock = Conf.Search.MathBlock
1443 s.Table = Conf.Search.Table
1444 s.Blockquote = Conf.Search.Blockquote
1445 s.SuperBlock = Conf.Search.SuperBlock
1446 s.Paragraph = Conf.Search.Paragraph
1447 s.HTMLBlock = Conf.Search.HTMLBlock
1448 s.EmbedBlock = Conf.Search.EmbedBlock
1449 s.DatabaseBlock = Conf.Search.DatabaseBlock
1450 s.AudioBlock = Conf.Search.AudioBlock
1451 s.VideoBlock = Conf.Search.VideoBlock
1452 s.IFrameBlock = Conf.Search.IFrameBlock
1453 s.WidgetBlock = Conf.Search.WidgetBlock
1454 }
1455 return s.TypeFilter()
1456}
1457
1458func searchBySQL(stmt string, beforeLen, page, pageSize int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
1459 stmt = strings.TrimSpace(stmt)
1460 blocks := sql.SelectBlocksRawStmt(stmt, page, pageSize)
1461 ret = fromSQLBlocks(&blocks, "", beforeLen)
1462 if 1 > len(ret) {
1463 ret = []*Block{}
1464 return
1465 }
1466
1467 stmt = strings.ToLower(stmt)
1468 stdQuery := !strings.Contains(stmt, "with recursive") && !strings.Contains(stmt, "union")
1469 if stdQuery {
1470 if strings.HasPrefix(stmt, "select a.* ") { // 多个搜索关键字匹配文档 https://github.com/siyuan-note/siyuan/issues/7350
1471 stmt = strings.ReplaceAll(stmt, "select a.* ", "select COUNT(a.id) AS `matches`, COUNT(DISTINCT(a.root_id)) AS `docs` ")
1472 } else {
1473 stmt = strings.ReplaceAll(stmt, "select * ", "select COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` ")
1474 }
1475 }
1476 stmt = removeLimitClause(stmt)
1477 result, _ := sql.QueryNoLimit(stmt)
1478 if 1 > len(result) {
1479 return
1480 }
1481
1482 if !stdQuery {
1483 var rootIDs, blockIDs []string
1484 for _, queryResult := range result {
1485 rootIDs = append(rootIDs, queryResult["root_id"].(string))
1486 blockIDs = append(blockIDs, queryResult["id"].(string))
1487 }
1488 rootIDs = gulu.Str.RemoveDuplicatedElem(rootIDs)
1489 blockIDs = gulu.Str.RemoveDuplicatedElem(blockIDs)
1490 matchedRootCount = len(rootIDs)
1491 matchedBlockCount = len(blockIDs)
1492 } else {
1493 matchedBlockCount = int(result[0]["matches"].(int64))
1494 matchedRootCount = int(result[0]["docs"].(int64))
1495 }
1496 return
1497}
1498
1499func removeLimitClause(stmt string) string {
1500 parsedStmt, err := sqlparser.Parse(stmt)
1501 if err != nil {
1502 return stmt
1503 }
1504
1505 switch parsedStmt.(type) {
1506 case *sqlparser.Select:
1507 slct := parsedStmt.(*sqlparser.Select)
1508 if nil != slct.Limit {
1509 slct.Limit = nil
1510 }
1511 stmt = sqlparser.String(slct)
1512 }
1513 return stmt
1514}
1515
1516func fullTextSearchRefBlock(keyword string, beforeLen int, onlyDoc bool) (ret []*Block) {
1517 keyword = filterQueryInvisibleChars(keyword)
1518
1519 if id := extractID(keyword); "" != id {
1520 ret, _, _ = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+id+"'", 36, 1, 32)
1521 return
1522 }
1523
1524 quotedKeyword := stringQuery(keyword)
1525 table := "blocks_fts" // 大小写敏感
1526 if !Conf.Search.CaseSensitive {
1527 table = "blocks_fts_case_insensitive"
1528 }
1529
1530 projections := "id, parent_id, root_id, hash, box, path, " +
1531 "snippet(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 64) AS hpath, " +
1532 "snippet(" + table + ", 7, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 64) AS name, " +
1533 "snippet(" + table + ", 8, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 64) AS alias, " +
1534 "snippet(" + table + ", 9, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 64) AS memo, " +
1535 "snippet(" + table + ", 10, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 64) AS tag, " +
1536 "snippet(" + table + ", 11, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 64) AS content, " +
1537 "fcontent, markdown, length, type, subtype, ial, sort, created, updated"
1538 stmt := "SELECT " + projections + " FROM " + table + " WHERE " + table + " MATCH '" + columnFilter() + ":(" + quotedKeyword + ")' AND type"
1539 if onlyDoc {
1540 stmt += " = 'd'"
1541 } else {
1542 stmt += " IN " + Conf.Search.TypeFilter()
1543 }
1544
1545 if ignoreLines := getRefSearchIgnoreLines(); 0 < len(ignoreLines) {
1546 // Support ignore search results https://github.com/siyuan-note/siyuan/issues/10089
1547 buf := bytes.Buffer{}
1548 for _, line := range ignoreLines {
1549 buf.WriteString(" AND ")
1550 buf.WriteString(line)
1551 }
1552 stmt += buf.String()
1553 }
1554
1555 orderBy := ` ORDER BY CASE
1556 WHEN name = '${keyword}' THEN 10
1557 WHEN alias = '${keyword}' THEN 20
1558 WHEN memo = '${keyword}' THEN 30
1559 WHEN content = '${keyword}' and type = 'd' THEN 40
1560 WHEN content LIKE '%${keyword}%' and type = 'd' THEN 41
1561 WHEN name LIKE '%${keyword}%' THEN 50
1562 WHEN alias LIKE '%${keyword}%' THEN 60
1563 WHEN content = '${keyword}' and type = 'h' THEN 70
1564 WHEN content LIKE '%${keyword}%' and type = 'h' THEN 71
1565 WHEN fcontent = '${keyword}' and type = 'i' THEN 80
1566 WHEN fcontent LIKE '%${keyword}%' and type = 'i' THEN 81
1567 WHEN memo LIKE '%${keyword}%' THEN 90
1568 WHEN content LIKE '%${keyword}%' and type != 'i' and type != 'l' THEN 100
1569 ELSE 65535 END ASC, sort ASC, length ASC`
1570 orderBy = strings.ReplaceAll(orderBy, "${keyword}", strings.ReplaceAll(keyword, "'", "''"))
1571 stmt += orderBy + " LIMIT " + strconv.Itoa(Conf.Search.Limit)
1572 blocks := sql.SelectBlocksRawStmtNoParse(stmt, Conf.Search.Limit)
1573 ret = fromSQLBlocks(&blocks, "", beforeLen)
1574 if 1 > len(ret) {
1575 ret = []*Block{}
1576 }
1577 return
1578}
1579
1580func extractID(content string) (ret string) {
1581 // Improve block ref search ID extraction https://github.com/siyuan-note/siyuan/issues/10848
1582
1583 if 22 > len(content) {
1584 return
1585 }
1586
1587 // 从第一个字符开始循环，直到找到一个合法的 ID 为止
1588 for i := 0; i < len(content)-21; i++ {
1589 if ast.IsNodeIDPattern(content[i : i+22]) {
1590 ret = content[i : i+22]
1591 return
1592 }
1593 }
1594 return
1595}
1596
1597func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter, ignoreFilter, orderBy string, beforeLen, page, pageSize int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
1598 fieldFilter := fieldRegexp(exp)
1599 stmt := "SELECT * FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter
1600 stmt += boxFilter + pathFilter + ignoreFilter + " " + orderBy
1601 regex, err := regexp.Compile(exp)
1602 if nil != err {
1603 util.PushErrMsg(err.Error(), 5000)
1604 return
1605 }
1606
1607 blocks := sql.SelectBlocksRegex(stmt, regex, Conf.Search.Name, Conf.Search.Alias, Conf.Search.Memo, Conf.Search.IAL, page, pageSize)
1608 ret = fromSQLBlocks(&blocks, "", beforeLen)
1609 if 1 > len(ret) {
1610 ret = []*Block{}
1611 }
1612
1613 matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter, ignoreFilter)
1614 return
1615}
1616
1617func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter, ignoreFilter string) (matchedBlockCount, matchedRootCount int) {
1618 fieldFilter := fieldRegexp(exp)
1619 stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter + ignoreFilter
1620 stmt += boxFilter + pathFilter
1621 result, _ := sql.QueryNoLimit(stmt)
1622 if 1 > len(result) {
1623 return
1624 }
1625 matchedBlockCount = int(result[0]["matches"].(int64))
1626 matchedRootCount = int(result[0]["docs"].(int64))
1627 return
1628}
1629
1630func fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, ignoreFilter, orderBy string, beforeLen, page, pageSize int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
1631 table := "blocks_fts" // 大小写敏感
1632 if !Conf.Search.CaseSensitive {
1633 table = "blocks_fts_case_insensitive"
1634 }
1635 projections := "id, parent_id, root_id, hash, box, path, " +
1636 // Search result content snippet returns more text https://github.com/siyuan-note/siyuan/issues/10707
1637 "snippet(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 512) AS hpath, " +
1638 "snippet(" + table + ", 7, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 512) AS name, " +
1639 "snippet(" + table + ", 8, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 512) AS alias, " +
1640 "snippet(" + table + ", 9, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 512) AS memo, " +
1641 "snippet(" + table + ", 10, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 64) AS tag, " +
1642 "snippet(" + table + ", 11, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 512) AS content, " +
1643 "fcontent, markdown, length, type, subtype, ial, sort, created, updated"
1644 stmt := "SELECT " + projections + " FROM " + table + " WHERE (`" + table + "` MATCH '" + columnFilter() + ":(" + query + ")'"
1645 stmt += ") AND type IN " + typeFilter
1646 stmt += boxFilter + pathFilter + ignoreFilter + " " + orderBy
1647 stmt += " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize)
1648 blocks := sql.SelectBlocksRawStmt(stmt, page, pageSize)
1649 ret = fromSQLBlocks(&blocks, "", beforeLen)
1650 if 1 > len(ret) {
1651 ret = []*Block{}
1652 }
1653
1654 matchedBlockCount, matchedRootCount = fullTextSearchCountByFTS(query, boxFilter, pathFilter, typeFilter, ignoreFilter)
1655 return
1656}
1657
1658func fullTextSearchCountByFTS(query, boxFilter, pathFilter, typeFilter, ignoreFilter string) (matchedBlockCount, matchedRootCount int) {
1659 table := "blocks_fts" // 大小写敏感
1660 if !Conf.Search.CaseSensitive {
1661 table = "blocks_fts_case_insensitive"
1662 }
1663
1664 stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `" + table + "` WHERE (`" + table + "` MATCH '" + columnFilter() + ":(" + query + ")'"
1665 stmt += ") AND type IN " + typeFilter
1666 stmt += boxFilter + pathFilter + ignoreFilter
1667 result, _ := sql.QueryNoLimit(stmt)
1668 if 1 > len(result) {
1669 return
1670 }
1671 matchedBlockCount = int(result[0]["matches"].(int64))
1672 matchedRootCount = int(result[0]["docs"].(int64))
1673 return
1674}
1675
1676func fullTextSearchByLikeWithRoot(query, boxFilter, pathFilter, typeFilter, ignoreFilter, orderBy string, beforeLen, page, pageSize int) (ret []*Block, matchedBlockCount, matchedRootCount int) {
1677 query = strings.ReplaceAll(query, "'", "''") // 不需要转义双引号，因为条件都是通过单引号包裹的，只需要转义单引号即可
1678 keywords := strings.Split(query, " ")
1679 contentField := columnConcat()
1680 var likeFilter string
1681 orderByLike := "("
1682 for i, keyword := range keywords {
1683 likeFilter += "GROUP_CONCAT(" + contentField + ") LIKE '%" + keyword + "%'"
1684 orderByLike += "(docContent LIKE '%" + keyword + "%')"
1685 if i < len(keywords)-1 {
1686 likeFilter += " AND "
1687 orderByLike += " + "
1688 }
1689 }
1690 orderByLike += ")"
1691 dMatchStmt := "SELECT root_id, MAX(CASE WHEN type = 'd' THEN (" + contentField + ") END) AS docContent" +
1692 " FROM blocks WHERE type IN " + typeFilter + boxFilter + pathFilter + ignoreFilter +
1693 " GROUP BY root_id HAVING " + likeFilter + "ORDER BY " + orderByLike + " DESC, MAX(updated) DESC"
1694 cteStmt := "WITH docBlocks AS (" + dMatchStmt + ")"
1695 likeFilter = strings.ReplaceAll(likeFilter, "GROUP_CONCAT("+contentField+")", "concatContent")
1696 limit := " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize)
1697 selectStmt := cteStmt + "\nSELECT *, " +
1698 "(" + contentField + ") AS concatContent, " +
1699 "(SELECT COUNT(root_id) FROM docBlocks) AS docs, " +
1700 "(CASE WHEN (root_id IN (SELECT root_id FROM docBlocks) AND (" + strings.ReplaceAll(likeFilter, "concatContent", contentField) + ")) THEN 1 ELSE 0 END) AS blockSort" +
1701 " FROM blocks WHERE type IN " + typeFilter + boxFilter + pathFilter + ignoreFilter +
1702 " AND (id IN (SELECT root_id FROM docBlocks " + limit + ") OR" +
1703 " (root_id IN (SELECT root_id FROM docBlocks" + limit + ") AND (" + likeFilter + ")))"
1704 if strings.Contains(orderBy, "ORDER BY rank DESC") {
1705 orderBy = buildOrderBy(query, 0, 0)
1706 selectStmt += " " + strings.Replace(orderBy, "END ASC, ", "END ASC, blockSort ASC, ", 1)
1707 } else if strings.Contains(orderBy, "ORDER BY rank") {
1708 orderBy = buildOrderBy(query, 0, 0)
1709 selectStmt += " " + strings.Replace(orderBy, "END ASC, ", "END ASC, blockSort DESC, ", 1)
1710 } else if strings.Contains(orderBy, "sort ASC") {
1711 selectStmt += " " + strings.Replace(orderBy, "END ASC, ", "END ASC, blockSort DESC, ", 1)
1712 } else {
1713 selectStmt += " " + orderBy
1714 }
1715 result, _ := sql.QueryNoLimit(selectStmt)
1716 resultBlocks := sql.ToBlocks(result)
1717 if 0 < len(resultBlocks) {
1718 matchedRootCount = int(result[0]["docs"].(int64))
1719 matchedBlockCount = matchedRootCount
1720 }
1721
1722 keywords = gulu.Str.RemoveDuplicatedElem(keywords)
1723 terms := strings.Join(keywords, search.TermSep)
1724 terms = strings.ReplaceAll(terms, "''", "'")
1725 ret = fromSQLBlocks(&resultBlocks, terms, beforeLen)
1726 if 1 > len(ret) {
1727 ret = []*Block{}
1728 }
1729 return
1730}
1731
1732func highlightByFTS(query, typeFilter, id string) (ret []string) {
1733 query = strings.ReplaceAll(query, " ", " OR ")
1734 const limit = 256
1735 table := "blocks_fts"
1736 if !Conf.Search.CaseSensitive {
1737 table = "blocks_fts_case_insensitive"
1738 }
1739 projections := "id, parent_id, root_id, hash, box, path, " +
1740 "highlight(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS hpath, " +
1741 "highlight(" + table + ", 7, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS name, " +
1742 "highlight(" + table + ", 8, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS alias, " +
1743 "highlight(" + table + ", 9, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS memo, " +
1744 "highlight(" + table + ", 10, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS tag, " +
1745 "highlight(" + table + ", 11, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS content, " +
1746 "fcontent, markdown, length, type, subtype, " +
1747 "highlight(" + table + ", 17, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS ial, " +
1748 "sort, created, updated"
1749 stmt := "SELECT " + projections + " FROM " + table + " WHERE (`" + table + "` MATCH '" + columnFilter() + ":(" + query + ")'"
1750 stmt += ") AND type IN " + typeFilter
1751 stmt += " AND root_id = '" + id + "'"
1752 stmt += " LIMIT " + strconv.Itoa(limit)
1753 sqlBlocks := sql.SelectBlocksRawStmt(stmt, 1, limit)
1754 for _, block := range sqlBlocks {
1755 keyword := gulu.Str.SubstringsBetween(block.HPath, search.SearchMarkLeft, search.SearchMarkRight)
1756 if 0 < len(keyword) {
1757 ret = append(ret, keyword...)
1758 }
1759 keyword = gulu.Str.SubstringsBetween(block.Name, search.SearchMarkLeft, search.SearchMarkRight)
1760 if 0 < len(keyword) {
1761 ret = append(ret, keyword...)
1762 }
1763 keyword = gulu.Str.SubstringsBetween(block.Alias, search.SearchMarkLeft, search.SearchMarkRight)
1764 if 0 < len(keyword) {
1765 ret = append(ret, keyword...)
1766 }
1767 keyword = gulu.Str.SubstringsBetween(block.Memo, search.SearchMarkLeft, search.SearchMarkRight)
1768 if 0 < len(keyword) {
1769 ret = append(ret, keyword...)
1770 }
1771 keyword = gulu.Str.SubstringsBetween(block.Tag, search.SearchMarkLeft, search.SearchMarkRight)
1772 if 0 < len(keyword) {
1773 ret = append(ret, keyword...)
1774 }
1775 keyword = gulu.Str.SubstringsBetween(block.IAL, search.SearchMarkLeft, search.SearchMarkRight)
1776 if 0 < len(keyword) {
1777 ret = append(ret, keyword...)
1778 }
1779 keyword = gulu.Str.SubstringsBetween(block.Content, search.SearchMarkLeft, search.SearchMarkRight)
1780 if 0 < len(keyword) {
1781 ret = append(ret, keyword...)
1782 }
1783 }
1784 ret = gulu.Str.RemoveDuplicatedElem(ret)
1785 return
1786}
1787
1788func highlightByRegexp(query, typeFilter, id string) (ret []string) {
1789 fieldFilter := fieldRegexp(query)
1790 stmt := "SELECT * FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter
1791 stmt += " AND root_id = '" + id + "'"
1792 regex, _ := regexp.Compile(query)
1793 if nil == regex {
1794 return
1795 }
1796 sqlBlocks := sql.SelectBlocksRegex(stmt, regex, Conf.Search.Name, Conf.Search.Alias, Conf.Search.Memo, Conf.Search.IAL, 1, 256)
1797 for _, block := range sqlBlocks {
1798 keyword := gulu.Str.SubstringsBetween(block.HPath, search.SearchMarkLeft, search.SearchMarkRight)
1799 if 0 < len(keyword) {
1800 ret = append(ret, keyword...)
1801 }
1802 keyword = gulu.Str.SubstringsBetween(block.Name, search.SearchMarkLeft, search.SearchMarkRight)
1803 if 0 < len(keyword) {
1804 ret = append(ret, keyword...)
1805 }
1806 keyword = gulu.Str.SubstringsBetween(block.Alias, search.SearchMarkLeft, search.SearchMarkRight)
1807 if 0 < len(keyword) {
1808 ret = append(ret, keyword...)
1809 }
1810 keyword = gulu.Str.SubstringsBetween(block.Memo, search.SearchMarkLeft, search.SearchMarkRight)
1811 if 0 < len(keyword) {
1812 ret = append(ret, keyword...)
1813 }
1814 keyword = gulu.Str.SubstringsBetween(block.Tag, search.SearchMarkLeft, search.SearchMarkRight)
1815 if 0 < len(keyword) {
1816 ret = append(ret, keyword...)
1817 }
1818 keyword = gulu.Str.SubstringsBetween(block.Content, search.SearchMarkLeft, search.SearchMarkRight)
1819 if 0 < len(keyword) {
1820 ret = append(ret, keyword...)
1821 }
1822 }
1823 ret = gulu.Str.RemoveDuplicatedElem(ret)
1824 return
1825}
1826
1827func markSearch(text string, keyword string, beforeLen int) (marked string, score float64) {
1828 if 0 == len(keyword) {
1829 if strings.Contains(text, search.SearchMarkLeft) { // 使用 FTS snippet() 处理过高亮片段，这里简单替换后就返回
1830 marked = util.EscapeHTML(text)
1831 marked = strings.ReplaceAll(marked, search.SearchMarkLeft, "<mark>")
1832 marked = strings.ReplaceAll(marked, search.SearchMarkRight, "</mark>")
1833 return
1834 }
1835
1836 keywords := gulu.Str.SubstringsBetween(text, search.SearchMarkLeft, search.SearchMarkRight)
1837 keywords = gulu.Str.RemoveDuplicatedElem(keywords)
1838 keyword = strings.Join(keywords, search.TermSep)
1839 marked = strings.ReplaceAll(text, search.SearchMarkLeft, "")
1840 marked = strings.ReplaceAll(marked, search.SearchMarkRight, "")
1841 _, marked = search.MarkText(marked, keyword, beforeLen, Conf.Search.CaseSensitive)
1842 marked = util.EscapeHTML(marked)
1843 return
1844 }
1845
1846 pos, marked := search.MarkText(text, keyword, beforeLen, Conf.Search.CaseSensitive)
1847 if -1 < pos {
1848 if 0 == pos {
1849 score = 1
1850 }
1851 score += float64(strings.Count(marked, "<mark>"))
1852 winkler := smetrics.JaroWinkler(text, keyword, 0.7, 4)
1853 score += winkler
1854 }
1855 score = -score // 分越小排序越靠前
1856 return
1857}
1858
1859func fromSQLBlocks(sqlBlocks *[]*sql.Block, terms string, beforeLen int) (ret []*Block) {
1860 for _, sqlBlock := range *sqlBlocks {
1861 ret = append(ret, fromSQLBlock(sqlBlock, terms, beforeLen))
1862 }
1863 return
1864}
1865
1866func fromSQLBlock(sqlBlock *sql.Block, terms string, beforeLen int) (block *Block) {
1867 if nil == sqlBlock {
1868 return
1869 }
1870
1871 id := sqlBlock.ID
1872 content := sqlBlock.Content
1873 if 1 < strings.Count(content, search.SearchMarkRight) && strings.HasSuffix(content, search.SearchMarkRight+"...") {
1874 // 返回多个关键字命中时需要检查最后一个关键字是否被截断
1875 firstKeyword := gulu.Str.SubStringBetween(content, search.SearchMarkLeft, search.SearchMarkRight)
1876 lastKeyword := gulu.Str.LastSubStringBetween(content, search.SearchMarkLeft, search.SearchMarkRight)
1877 if firstKeyword != lastKeyword {
1878 // 如果第一个关键字和最后一个关键字不相同，说明最后一个关键字被截断了
1879 // 此时需要将 content 中的最后一个关键字替换为完整的关键字
1880 content = strings.TrimSuffix(content, search.SearchMarkLeft+lastKeyword+search.SearchMarkRight+"...")
1881 content += search.SearchMarkLeft + firstKeyword + search.SearchMarkRight + "..."
1882 }
1883 }
1884
1885 content, _ = markSearch(content, terms, beforeLen)
1886 content = maxContent(content, 5120)
1887 tag, _ := markSearch(sqlBlock.Tag, terms, beforeLen)
1888 markdown := maxContent(sqlBlock.Markdown, 5120)
1889 fContent := sqlBlock.FContent
1890 block = &Block{
1891 Box: sqlBlock.Box,
1892 Path: sqlBlock.Path,
1893 ID: id,
1894 RootID: sqlBlock.RootID,
1895 ParentID: sqlBlock.ParentID,
1896 Alias: sqlBlock.Alias,
1897 Name: sqlBlock.Name,
1898 Memo: sqlBlock.Memo,
1899 Tag: tag,
1900 Content: content,
1901 FContent: fContent,
1902 Markdown: markdown,
1903 Type: treenode.FromAbbrType(sqlBlock.Type),
1904 SubType: sqlBlock.SubType,
1905 Sort: sqlBlock.Sort,
1906 }
1907 if "" != sqlBlock.IAL {
1908 block.IAL = map[string]string{}
1909 ialStr := strings.TrimPrefix(sqlBlock.IAL, "{:")
1910 ialStr = strings.TrimSuffix(ialStr, "}")
1911 ial := parse.Tokens2IAL([]byte(ialStr))
1912 for _, kv := range ial {
1913 block.IAL[kv[0]] = kv[1]
1914 }
1915 }
1916
1917 hPath, _ := markSearch(sqlBlock.HPath, "", 18)
1918 if !strings.HasPrefix(hPath, "/") {
1919 hPath = "/" + hPath
1920 }
1921 block.HPath = hPath
1922
1923 if "" != block.Name {
1924 block.Name, _ = markSearch(block.Name, terms, 256)
1925 }
1926 if "" != block.Alias {
1927 block.Alias, _ = markSearch(block.Alias, terms, 256)
1928 }
1929 if "" != block.Memo {
1930 block.Memo, _ = markSearch(block.Memo, terms, 256)
1931 }
1932 return
1933}
1934
1935func maxContent(content string, maxLen int) string {
1936 idx := strings.Index(content, "<mark>")
1937 if 128 < maxLen && maxLen <= idx {
1938 head := bytes.Buffer{}
1939 for i := 0; i < 512; i++ {
1940 r, size := utf8.DecodeLastRuneInString(content[:idx])
1941 head.WriteRune(r)
1942 idx -= size
1943 if 64 < head.Len() {
1944 break
1945 }
1946 }
1947
1948 content = util.Reverse(head.String()) + content[idx:]
1949 }
1950
1951 if maxLen < utf8.RuneCountInString(content) {
1952 return gulu.Str.SubStr(content, maxLen) + "..."
1953 }
1954 return content
1955}
1956
1957func fieldRegexp(regexp string) string {
1958 regexp = strings.ReplaceAll(regexp, "'", "''") // 不需要转义双引号，因为条件都是通过单引号包裹的，只需要转义单引号即可
1959 buf := bytes.Buffer{}
1960 buf.WriteString("(")
1961 buf.WriteString("content REGEXP '")
1962 buf.WriteString(regexp)
1963 buf.WriteString("'")
1964 if Conf.Search.Name {
1965 buf.WriteString(" OR name REGEXP '")
1966 buf.WriteString(regexp)
1967 buf.WriteString("'")
1968 }
1969 if Conf.Search.Alias {
1970 buf.WriteString(" OR alias REGEXP '")
1971 buf.WriteString(regexp)
1972 buf.WriteString("'")
1973 }
1974 if Conf.Search.Memo {
1975 buf.WriteString(" OR memo REGEXP '")
1976 buf.WriteString(regexp)
1977 buf.WriteString("'")
1978 }
1979 if Conf.Search.IAL {
1980 buf.WriteString(" OR ial REGEXP '")
1981 buf.WriteString(regexp)
1982 buf.WriteString("'")
1983 }
1984 buf.WriteString(" OR tag REGEXP '")
1985 buf.WriteString(regexp)
1986 buf.WriteString("')")
1987 return buf.String()
1988}
1989
1990func columnFilter() string {
1991 buf := bytes.Buffer{}
1992 buf.WriteString("{content")
1993 if Conf.Search.Name {
1994 buf.WriteString(" name")
1995 }
1996 if Conf.Search.Alias {
1997 buf.WriteString(" alias")
1998 }
1999 if Conf.Search.Memo {
2000 buf.WriteString(" memo")
2001 }
2002 if Conf.Search.IAL {
2003 buf.WriteString(" ial")
2004 }
2005 buf.WriteString(" tag}")
2006 return buf.String()
2007}
2008
2009func columnConcat() string {
2010 buf := bytes.Buffer{}
2011 buf.WriteString("content")
2012 if Conf.Search.Name {
2013 buf.WriteString("||name")
2014 }
2015 if Conf.Search.Alias {
2016 buf.WriteString("||alias")
2017 }
2018 if Conf.Search.Memo {
2019 buf.WriteString("||memo")
2020 }
2021 if Conf.Search.IAL {
2022 buf.WriteString("||ial")
2023 }
2024 buf.WriteString("||tag")
2025 return buf.String()
2026}
2027
2028func stringQuery(query string) string {
2029 trimmedQuery := strings.TrimSpace(query)
2030 if "" == trimmedQuery {
2031 return "\"" + query + "\""
2032 }
2033
2034 query = strings.ReplaceAll(query, "\"", "\"\"")
2035 query = strings.ReplaceAll(query, "'", "''")
2036
2037 if strings.Contains(trimmedQuery, " ") {
2038 buf := bytes.Buffer{}
2039 parts := strings.Split(query, " ")
2040 for _, part := range parts {
2041 part = strings.TrimSpace(part)
2042 part = "\"" + part + "\""
2043 buf.WriteString(part)
2044 buf.WriteString(" ")
2045 }
2046 return strings.TrimSpace(buf.String())
2047 }
2048 return "\"" + query + "\""
2049}
2050
2051// markReplaceSpan 用于处理搜索高亮。
2052func markReplaceSpan(n *ast.Node, unlinks *[]*ast.Node, keywords []string, markSpanDataType string, luteEngine *lute.Lute) bool {
2053 if ast.NodeText == n.Type {
2054 text := n.Content()
2055 escapedText := util.EscapeHTML(text)
2056 escapedKeywords := make([]string, len(keywords))
2057 for i, keyword := range keywords {
2058 escapedKeywords[i] = util.EscapeHTML(keyword)
2059 }
2060 hText := search.EncloseHighlighting(escapedText, escapedKeywords, search.GetMarkSpanStart(markSpanDataType), search.GetMarkSpanEnd(), Conf.Search.CaseSensitive, false)
2061 if hText != escapedText {
2062 text = hText
2063 }
2064 n.Tokens = gulu.Str.ToBytes(text)
2065 if bytes.Contains(n.Tokens, []byte(search.MarkDataType)) {
2066 linkTree := parse.Inline("", n.Tokens, luteEngine.ParseOptions)
2067 var children []*ast.Node
2068 for c := linkTree.Root.FirstChild.FirstChild; nil != c; c = c.Next {
2069 children = append(children, c)
2070 }
2071 for _, c := range children {
2072 n.InsertBefore(c)
2073 }
2074 *unlinks = append(*unlinks, n)
2075 return true
2076 }
2077 } else if ast.NodeTextMark == n.Type {
2078 // 搜索结果高亮支持大部分行级元素 https://github.com/siyuan-note/siyuan/issues/6745
2079
2080 if n.IsTextMarkType("inline-math") || n.IsTextMarkType("inline-memo") {
2081 return false
2082 }
2083
2084 var text string
2085 if n.IsTextMarkType("code") {
2086 // code 在前面的 n.
2087 for i, k := range keywords {
2088 keywords[i] = html.EscapeString(k)
2089 }
2090 text = n.TextMarkTextContent
2091 } else {
2092 text = n.Content()
2093 }
2094
2095 startTag := search.GetMarkSpanStart(markSpanDataType)
2096 text = search.EncloseHighlighting(text, keywords, startTag, search.GetMarkSpanEnd(), Conf.Search.CaseSensitive, false)
2097 if strings.Contains(text, search.MarkDataType) {
2098 dataType := search.GetMarkSpanStart(n.TextMarkType + " " + search.MarkDataType)
2099 text = strings.ReplaceAll(text, startTag, dataType)
2100 tokens := gulu.Str.ToBytes(text)
2101 linkTree := parse.Inline("", tokens, luteEngine.ParseOptions)
2102 var children []*ast.Node
2103 for c := linkTree.Root.FirstChild.FirstChild; nil != c; c = c.Next {
2104 if ast.NodeText == c.Type {
2105 c.Type = ast.NodeTextMark
2106 c.TextMarkType = n.TextMarkType
2107 c.TextMarkTextContent = string(c.Tokens)
2108 if n.IsTextMarkType("a") {
2109 c.TextMarkAHref, c.TextMarkATitle = n.TextMarkAHref, n.TextMarkATitle
2110 } else if treenode.IsBlockRef(n) {
2111 c.TextMarkBlockRefID = n.TextMarkBlockRefID
2112 c.TextMarkBlockRefSubtype = n.TextMarkBlockRefSubtype
2113 } else if treenode.IsFileAnnotationRef(n) {
2114 c.TextMarkFileAnnotationRefID = n.TextMarkFileAnnotationRefID
2115 }
2116 } else if ast.NodeTextMark == c.Type {
2117 if n.IsTextMarkType("a") {
2118 c.TextMarkAHref, c.TextMarkATitle = n.TextMarkAHref, n.TextMarkATitle
2119 } else if treenode.IsBlockRef(n) {
2120 c.TextMarkBlockRefID = n.TextMarkBlockRefID
2121 c.TextMarkBlockRefSubtype = n.TextMarkBlockRefSubtype
2122 } else if treenode.IsFileAnnotationRef(n) {
2123 c.TextMarkFileAnnotationRefID = n.TextMarkFileAnnotationRefID
2124 }
2125 }
2126
2127 children = append(children, c)
2128 if nil != n.Next && ast.NodeKramdownSpanIAL == n.Next.Type {
2129 c.KramdownIAL = n.KramdownIAL
2130 ial := &ast.Node{Type: ast.NodeKramdownSpanIAL, Tokens: n.Next.Tokens}
2131 children = append(children, ial)
2132 }
2133 }
2134 for _, c := range children {
2135 n.InsertBefore(c)
2136 }
2137 *unlinks = append(*unlinks, n)
2138 return true
2139 }
2140 }
2141 return false
2142}
2143
2144// markReplaceSpanWithSplit 用于处理虚拟引用和反链提及高亮。
2145func markReplaceSpanWithSplit(text string, keywords []string, replacementStart, replacementEnd string) (ret string) {
2146 // 虚拟引用和反链提及关键字按最长匹配优先 https://github.com/siyuan-note/siyuan/issues/7465
2147 sort.Slice(keywords, func(i, j int) bool { return len(keywords[i]) > len(keywords[j]) })
2148
2149 tmp := search.EncloseHighlighting(text, keywords, replacementStart, replacementEnd, Conf.Search.CaseSensitive, true)
2150 parts := strings.Split(tmp, replacementEnd)
2151 buf := bytes.Buffer{}
2152 for i := 0; i < len(parts); i++ {
2153 if i >= len(parts)-1 {
2154 buf.WriteString(parts[i])
2155 break
2156 }
2157
2158 if nextPart := parts[i+1]; 0 < len(nextPart) && lex.IsASCIILetter(nextPart[0]) {
2159 // 取消已经高亮的部分
2160 part := strings.ReplaceAll(parts[i], replacementStart, "")
2161 buf.WriteString(part)
2162 continue
2163 }
2164
2165 buf.WriteString(parts[i])
2166 buf.WriteString(replacementEnd)
2167 }
2168 ret = buf.String()
2169 return
2170}
2171
2172var (
2173 searchIgnoreLastModified int64
2174 searchIgnore []string
2175 searchIgnoreLock = sync.Mutex{}
2176)
2177
2178func getSearchIgnoreLines() (ret []string) {
2179 // Support ignore search results https://github.com/siyuan-note/siyuan/issues/10089
2180
2181 now := time.Now().UnixMilli()
2182 if now-searchIgnoreLastModified < 30*1000 {
2183 return searchIgnore
2184 }
2185
2186 searchIgnoreLock.Lock()
2187 defer searchIgnoreLock.Unlock()
2188
2189 searchIgnoreLastModified = now
2190
2191 searchIgnorePath := filepath.Join(util.DataDir, ".siyuan", "searchignore")
2192 err := os.MkdirAll(filepath.Dir(searchIgnorePath), 0755)
2193 if err != nil {
2194 return
2195 }
2196 if !gulu.File.IsExist(searchIgnorePath) {
2197 if err = gulu.File.WriteFileSafer(searchIgnorePath, nil, 0644); err != nil {
2198 logging.LogErrorf("create searchignore [%s] failed: %s", searchIgnorePath, err)
2199 return
2200 }
2201 }
2202 data, err := os.ReadFile(searchIgnorePath)
2203 if err != nil {
2204 logging.LogErrorf("read searchignore [%s] failed: %s", searchIgnorePath, err)
2205 return
2206 }
2207 dataStr := string(data)
2208 dataStr = strings.ReplaceAll(dataStr, "\r\n", "\n")
2209 ret = strings.Split(dataStr, "\n")
2210
2211 ret = gulu.Str.RemoveDuplicatedElem(ret)
2212 if 0 < len(ret) && "" == ret[0] {
2213 ret = ret[1:]
2214 }
2215 searchIgnore = nil
2216 for _, line := range ret {
2217 searchIgnore = append(searchIgnore, line)
2218 }
2219 return
2220}
2221
2222var (
2223 refSearchIgnoreLastModified int64
2224 refSearchIgnore []string
2225 refSearchIgnoreLock = sync.Mutex{}
2226)
2227
2228func getRefSearchIgnoreLines() (ret []string) {
2229 // Support ignore search results https://github.com/siyuan-note/siyuan/issues/10089
2230
2231 now := time.Now().UnixMilli()
2232 if now-refSearchIgnoreLastModified < 30*1000 {
2233 return refSearchIgnore
2234 }
2235
2236 refSearchIgnoreLock.Lock()
2237 defer refSearchIgnoreLock.Unlock()
2238
2239 refSearchIgnoreLastModified = now
2240
2241 searchIgnorePath := filepath.Join(util.DataDir, ".siyuan", "refsearchignore")
2242 err := os.MkdirAll(filepath.Dir(searchIgnorePath), 0755)
2243 if err != nil {
2244 return
2245 }
2246 if !gulu.File.IsExist(searchIgnorePath) {
2247 if err = gulu.File.WriteFileSafer(searchIgnorePath, nil, 0644); err != nil {
2248 logging.LogErrorf("create refsearchignore [%s] failed: %s", searchIgnorePath, err)
2249 return
2250 }
2251 }
2252 data, err := os.ReadFile(searchIgnorePath)
2253 if err != nil {
2254 logging.LogErrorf("read refsearchignore [%s] failed: %s", searchIgnorePath, err)
2255 return
2256 }
2257 dataStr := string(data)
2258 dataStr = strings.ReplaceAll(dataStr, "\r\n", "\n")
2259 ret = strings.Split(dataStr, "\n")
2260
2261 ret = gulu.Str.RemoveDuplicatedElem(ret)
2262 if 0 < len(ret) && "" == ret[0] {
2263 ret = ret[1:]
2264 }
2265 refSearchIgnore = nil
2266 for _, line := range ret {
2267 refSearchIgnore = append(refSearchIgnore, line)
2268 }
2269 return
2270}
2271
2272func filterQueryInvisibleChars(query string) string {
2273 query = strings.ReplaceAll(query, " ", "_@full_width_space@_")
2274 query = strings.ReplaceAll(query, "\t", "_@tab@_")
2275 query = strings.ReplaceAll(query, string(gulu.ZWJ), "__@ZWJ@__")
2276 query = util.RemoveInvalid(query)
2277 query = strings.ReplaceAll(query, "_@full_width_space@_", " ")
2278 query = strings.ReplaceAll(query, "_@tab@_", "\t")
2279 query = strings.ReplaceAll(query, "__@ZWJ@__", string(gulu.ZWJ))
2280 query = strings.ReplaceAll(query, string(gulu.ZWJ)+"#", "#")
2281 return query
2282}
2283
2284func replaceCaseInsensitive(input, old, new []byte) []byte {
2285 re := regexp.MustCompile("(?i)" + regexp.QuoteMeta(string(old)))
2286 return []byte(re.ReplaceAllString(string(input), string(new)))
2287}