onedeuxtriseigo.nullpo.dev
A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
1// SiYuan - Refactor your thinking
2// Copyright (c) 2020-present, b3log.org
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU Affero General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU Affero General Public License for more details.
13//
14// You should have received a copy of the GNU Affero General Public License
15// along with this program. If not, see <https://www.gnu.org/licenses/>.
16
17package sql
18
19import (
20 "bytes"
21 "crypto/sha256"
22 "database/sql"
23 "fmt"
24 "os"
25 "path"
26 "path/filepath"
27 "strings"
28 "sync"
29
30 "github.com/88250/gulu"
31 "github.com/88250/lute/parse"
32 "github.com/emirpasic/gods/sets/hashset"
33 ignore "github.com/sabhiram/go-gitignore"
34 "github.com/siyuan-note/eventbus"
35 "github.com/siyuan-note/logging"
36 "github.com/siyuan-note/siyuan/kernel/util"
37)
38
39var luteEngine = util.NewLute()
40
41func init() {
42 luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial
43}
44
45const (
46 BlocksInsert = "INSERT INTO blocks (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
47 BlocksFTSInsert = "INSERT INTO blocks_fts (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
48 BlocksFTSCaseInsensitiveInsert = "INSERT INTO blocks_fts_case_insensitive (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
49 BlocksPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
50
51 SpansInsert = "INSERT INTO spans (id, block_id, root_id, box, path, content, markdown, type, ial) VALUES %s"
52 SpansPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
53
54 AssetsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
55 AttributesPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?)"
56 RefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
57 FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
58)
59
60func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (err error) {
61 if 1 > len(blocks) {
62 return
63 }
64
65 var bulk []*Block
66 for _, block := range blocks {
67 bulk = append(bulk, block)
68 if 512 > len(bulk) {
69 continue
70 }
71
72 if err = insertBlocks0(tx, bulk, context); err != nil {
73 return
74 }
75 bulk = []*Block{}
76 }
77 if 0 < len(bulk) {
78 if err = insertBlocks0(tx, bulk, context); err != nil {
79 return
80 }
81 }
82 return
83}
84
85func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (err error) {
86 valueStrings := make([]string, 0, len(bulk))
87 valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?"))
88 hashBuf := bytes.Buffer{}
89 for _, b := range bulk {
90 valueStrings = append(valueStrings, BlocksPlaceholder)
91 valueArgs = append(valueArgs, b.ID)
92 valueArgs = append(valueArgs, b.ParentID)
93 valueArgs = append(valueArgs, b.RootID)
94 valueArgs = append(valueArgs, b.Hash)
95 valueArgs = append(valueArgs, b.Box)
96 valueArgs = append(valueArgs, b.Path)
97 valueArgs = append(valueArgs, b.HPath)
98 valueArgs = append(valueArgs, b.Name)
99 valueArgs = append(valueArgs, b.Alias)
100 valueArgs = append(valueArgs, b.Memo)
101 valueArgs = append(valueArgs, b.Tag)
102 valueArgs = append(valueArgs, b.Content)
103 valueArgs = append(valueArgs, b.FContent)
104 valueArgs = append(valueArgs, b.Markdown)
105 valueArgs = append(valueArgs, b.Length)
106 valueArgs = append(valueArgs, b.Type)
107 valueArgs = append(valueArgs, b.SubType)
108 valueArgs = append(valueArgs, b.IAL)
109 valueArgs = append(valueArgs, b.Sort)
110 valueArgs = append(valueArgs, b.Created)
111 valueArgs = append(valueArgs, b.Updated)
112 putBlockCache(b)
113
114 hashBuf.WriteString(b.Hash)
115 }
116
117 stmt := fmt.Sprintf(BlocksInsert, strings.Join(valueStrings, ","))
118 if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil {
119 return
120 }
121 hashBuf.WriteString("blocks")
122 evtHash := fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
123 // 使用下面的 EvtSQLInsertBlocksFTS 就可以了
124 //eventbus.Publish(eventbus.EvtSQLInsertBlocks, context, current, total, len(bulk), evtHash)
125
126 stmt = fmt.Sprintf(BlocksFTSInsert, strings.Join(valueStrings, ","))
127 if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil {
128 return
129 }
130
131 if !caseSensitive {
132 stmt = fmt.Sprintf(BlocksFTSCaseInsensitiveInsert, strings.Join(valueStrings, ","))
133 if err = prepareExecInsertTx(tx, stmt, valueArgs); err != nil {
134 return
135 }
136 }
137 hashBuf.WriteString("fts")
138 evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
139 eventbus.Publish(eventbus.EvtSQLInsertBlocksFTS, context, len(bulk), evtHash)
140 return
141}
142
143func insertAttributes(tx *sql.Tx, attributes []*Attribute) (err error) {
144 if 1 > len(attributes) {
145 return
146 }
147
148 var bulk []*Attribute
149 for _, attr := range attributes {
150 bulk = append(bulk, attr)
151 if 512 > len(bulk) {
152 continue
153 }
154
155 if err = insertAttribute0(tx, bulk); err != nil {
156 return
157 }
158 bulk = []*Attribute{}
159 }
160 if 0 < len(bulk) {
161 if err = insertAttribute0(tx, bulk); err != nil {
162 return
163 }
164 }
165 return
166}
167
168func insertAttribute0(tx *sql.Tx, bulk []*Attribute) (err error) {
169 if 1 > len(bulk) {
170 return
171 }
172
173 valueStrings := make([]string, 0, len(bulk))
174 valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AttributesPlaceholder, "?"))
175 for _, attr := range bulk {
176 valueStrings = append(valueStrings, AttributesPlaceholder)
177 valueArgs = append(valueArgs, attr.ID)
178 valueArgs = append(valueArgs, attr.Name)
179 valueArgs = append(valueArgs, attr.Value)
180 valueArgs = append(valueArgs, attr.Type)
181 valueArgs = append(valueArgs, attr.BlockID)
182 valueArgs = append(valueArgs, attr.RootID)
183 valueArgs = append(valueArgs, attr.Box)
184 valueArgs = append(valueArgs, attr.Path)
185 }
186 stmt := fmt.Sprintf("INSERT INTO attributes (id, name, value, type, block_id, root_id, box, path) VALUES %s", strings.Join(valueStrings, ","))
187 err = prepareExecInsertTx(tx, stmt, valueArgs)
188 return
189}
190
191func insertAssets(tx *sql.Tx, assets []*Asset) (err error) {
192 if 1 > len(assets) {
193 return
194 }
195
196 var bulk []*Asset
197 for _, asset := range assets {
198 bulk = append(bulk, asset)
199 if 512 > len(bulk) {
200 continue
201 }
202
203 if err = insertAsset0(tx, bulk); err != nil {
204 return
205 }
206 bulk = []*Asset{}
207 }
208 if 0 < len(bulk) {
209 if err = insertAsset0(tx, bulk); err != nil {
210 return
211 }
212 }
213 return
214}
215
216func insertAsset0(tx *sql.Tx, bulk []*Asset) (err error) {
217 if 1 > len(bulk) {
218 return
219 }
220
221 valueStrings := make([]string, 0, len(bulk))
222 valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AssetsPlaceholder, "?"))
223 for _, asset := range bulk {
224 valueStrings = append(valueStrings, AssetsPlaceholder)
225 valueArgs = append(valueArgs, asset.ID)
226 valueArgs = append(valueArgs, asset.BlockID)
227 valueArgs = append(valueArgs, asset.RootID)
228 valueArgs = append(valueArgs, asset.Box)
229 valueArgs = append(valueArgs, asset.DocPath)
230 valueArgs = append(valueArgs, asset.Path)
231 valueArgs = append(valueArgs, asset.Name)
232 valueArgs = append(valueArgs, asset.Title)
233 valueArgs = append(valueArgs, asset.Hash)
234 }
235 stmt := fmt.Sprintf("INSERT INTO assets (id, block_id, root_id, box, docpath, path, name, title, hash) VALUES %s", strings.Join(valueStrings, ","))
236 err = prepareExecInsertTx(tx, stmt, valueArgs)
237 return
238}
239
240func insertSpans(tx *sql.Tx, spans []*Span) (err error) {
241 if 1 > len(spans) {
242 return
243 }
244
245 var bulk []*Span
246 for _, span := range spans {
247 bulk = append(bulk, span)
248 if 512 > len(bulk) {
249 continue
250 }
251
252 if err = insertSpans0(tx, bulk); err != nil {
253 return
254 }
255 bulk = []*Span{}
256 }
257 if 0 < len(bulk) {
258 if err = insertSpans0(tx, bulk); err != nil {
259 return
260 }
261 }
262 return
263}
264
265func insertSpans0(tx *sql.Tx, bulk []*Span) (err error) {
266 if 1 > len(bulk) {
267 return
268 }
269
270 valueStrings := make([]string, 0, len(bulk))
271 valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(SpansPlaceholder, "?"))
272 for _, span := range bulk {
273 valueStrings = append(valueStrings, SpansPlaceholder)
274 valueArgs = append(valueArgs, span.ID)
275 valueArgs = append(valueArgs, span.BlockID)
276 valueArgs = append(valueArgs, span.RootID)
277 valueArgs = append(valueArgs, span.Box)
278 valueArgs = append(valueArgs, span.Path)
279 valueArgs = append(valueArgs, span.Content)
280 valueArgs = append(valueArgs, span.Markdown)
281 valueArgs = append(valueArgs, span.Type)
282 valueArgs = append(valueArgs, span.IAL)
283 }
284 stmt := fmt.Sprintf(SpansInsert, strings.Join(valueStrings, ","))
285 err = prepareExecInsertTx(tx, stmt, valueArgs)
286 return
287}
288
289func insertBlockRefs(tx *sql.Tx, refs []*Ref) (err error) {
290 if 1 > len(refs) {
291 return
292 }
293
294 var bulk []*Ref
295 for _, ref := range refs {
296 bulk = append(bulk, ref)
297 if 512 > len(bulk) {
298 continue
299 }
300
301 if err = insertRefs0(tx, bulk); err != nil {
302 return
303 }
304 bulk = []*Ref{}
305 }
306 if 0 < len(bulk) {
307 if err = insertRefs0(tx, bulk); err != nil {
308 return
309 }
310 }
311 return
312}
313
314func insertRefs0(tx *sql.Tx, bulk []*Ref) (err error) {
315 if 1 > len(bulk) {
316 return
317 }
318
319 valueStrings := make([]string, 0, len(bulk))
320 valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(RefsPlaceholder, "?"))
321 for _, ref := range bulk {
322 valueStrings = append(valueStrings, RefsPlaceholder)
323 valueArgs = append(valueArgs, ref.ID)
324 valueArgs = append(valueArgs, ref.DefBlockID)
325 valueArgs = append(valueArgs, ref.DefBlockParentID)
326 valueArgs = append(valueArgs, ref.DefBlockRootID)
327 valueArgs = append(valueArgs, ref.DefBlockPath)
328 valueArgs = append(valueArgs, ref.BlockID)
329 valueArgs = append(valueArgs, ref.RootID)
330 valueArgs = append(valueArgs, ref.Box)
331 valueArgs = append(valueArgs, ref.Path)
332 valueArgs = append(valueArgs, ref.Content)
333 valueArgs = append(valueArgs, ref.Markdown)
334 valueArgs = append(valueArgs, ref.Type)
335
336 putRefCache(ref)
337 }
338 stmt := fmt.Sprintf("INSERT INTO refs (id, def_block_id, def_block_parent_id, def_block_root_id, def_block_path, block_id, root_id, box, path, content, markdown, type) VALUES %s", strings.Join(valueStrings, ","))
339 err = prepareExecInsertTx(tx, stmt, valueArgs)
340 return
341}
342
343func insertFileAnnotationRefs(tx *sql.Tx, refs []*FileAnnotationRef) (err error) {
344 if 1 > len(refs) {
345 return
346 }
347
348 var bulk []*FileAnnotationRef
349 for _, ref := range refs {
350 bulk = append(bulk, ref)
351 if 512 > len(bulk) {
352 continue
353 }
354
355 if err = insertFileAnnotationRefs0(tx, bulk); err != nil {
356 return
357 }
358 bulk = []*FileAnnotationRef{}
359 }
360 if 0 < len(bulk) {
361 if err = insertFileAnnotationRefs0(tx, bulk); err != nil {
362 return
363 }
364 }
365 return
366}
367
368func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error) {
369 if 1 > len(bulk) {
370 return
371 }
372
373 valueStrings := make([]string, 0, len(bulk))
374 valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(FileAnnotationRefsPlaceholder, "?"))
375 for _, ref := range bulk {
376 valueStrings = append(valueStrings, FileAnnotationRefsPlaceholder)
377 valueArgs = append(valueArgs, ref.ID)
378 valueArgs = append(valueArgs, ref.FilePath)
379 valueArgs = append(valueArgs, ref.AnnotationID)
380 valueArgs = append(valueArgs, ref.BlockID)
381 valueArgs = append(valueArgs, ref.RootID)
382 valueArgs = append(valueArgs, ref.Box)
383 valueArgs = append(valueArgs, ref.Path)
384 valueArgs = append(valueArgs, ref.Content)
385 valueArgs = append(valueArgs, ref.Type)
386 }
387 stmt := fmt.Sprintf("INSERT INTO file_annotation_refs (id, file_path, annotation_id, block_id, root_id, box, path, content, type) VALUES %s", strings.Join(valueStrings, ","))
388 err = prepareExecInsertTx(tx, stmt, valueArgs)
389 return
390}
391
392func indexTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
393 blocks, spans, assets, attributes := fromTree(tree.Root, tree)
394 refs, fileAnnotationRefs := refsFromTree(tree)
395 err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs)
396 return
397}
398
399func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
400 oldBlockHashes := queryBlockHashes(tree.ID)
401 blocks, spans, assets, attributes := fromTree(tree.Root, tree)
402 newBlockHashes := map[string]string{}
403 for _, block := range blocks {
404 newBlockHashes[block.ID] = block.Hash
405 }
406 unChanges := hashset.New()
407 var toRemoves []string
408 for id, hash := range oldBlockHashes {
409 if newHash, ok := newBlockHashes[id]; ok {
410 if newHash == hash {
411 unChanges.Add(id)
412 }
413 } else {
414 toRemoves = append(toRemoves, id)
415 }
416 }
417 tmp := blocks[:0]
418 for _, b := range blocks {
419 if !unChanges.Contains(b.ID) {
420 tmp = append(tmp, b)
421 }
422 }
423 blocks = tmp
424 for _, b := range blocks {
425 toRemoves = append(toRemoves, b.ID)
426 }
427
428 if err = deleteBlocksByIDs(tx, toRemoves); err != nil {
429 return
430 }
431
432 if err = deleteSpansByRootID(tx, tree.ID); err != nil {
433 return
434 }
435 if err = deleteAssetsByRootID(tx, tree.ID); err != nil {
436 return
437 }
438 if err = deleteAttributesByRootID(tx, tree.ID); err != nil {
439 return
440 }
441 if err = deleteRefsByPathTx(tx, tree.Box, tree.Path); err != nil {
442 return
443 }
444 if err = deleteFileAnnotationRefsByPathTx(tx, tree.Box, tree.Path); err != nil {
445 return
446 }
447
448 refs, fileAnnotationRefs := refsFromTree(tree)
449 if err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); err != nil {
450 return
451 }
452 return err
453}
454
455func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{},
456 blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute,
457 refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) {
458 if ignoreLines := getIndexIgnoreLines(); 0 < len(ignoreLines) {
459 // Support ignore index https://github.com/siyuan-note/siyuan/issues/9198
460 matcher := ignore.CompileIgnoreLines(ignoreLines...)
461 if matcher.MatchesPath("/" + path.Join(tree.Box, tree.Path)) {
462 return
463 }
464 }
465
466 if err = insertBlocks(tx, blocks, context); err != nil {
467 return
468 }
469
470 if err = insertBlockRefs(tx, refs); err != nil {
471 return
472 }
473 if err = insertFileAnnotationRefs(tx, fileAnnotationRefs); err != nil {
474 return
475 }
476
477 if 0 < len(spans) {
478 // 移除文档标签，否则会重复添加 https://github.com/siyuan-note/siyuan/issues/3723
479 if err = deleteSpansByRootID(tx, tree.Root.ID); err != nil {
480 return
481 }
482 if err = insertSpans(tx, spans); err != nil {
483 return
484 }
485 }
486 if err = insertAssets(tx, assets); err != nil {
487 return
488 }
489 if err = insertAttributes(tx, attributes); err != nil {
490 return
491 }
492 return
493}
494
495var (
496 IndexIgnoreCached bool
497 indexIgnore []string
498 indexIgnoreLock = sync.Mutex{}
499)
500
501func getIndexIgnoreLines() (ret []string) {
502 // Support ignore index https://github.com/siyuan-note/siyuan/issues/9198
503
504 if IndexIgnoreCached {
505 return indexIgnore
506 }
507
508 indexIgnoreLock.Lock()
509 defer indexIgnoreLock.Unlock()
510
511 IndexIgnoreCached = true
512 indexIgnorePath := filepath.Join(util.DataDir, ".siyuan", "indexignore")
513 err := os.MkdirAll(filepath.Dir(indexIgnorePath), 0755)
514 if err != nil {
515 return
516 }
517 if !gulu.File.IsExist(indexIgnorePath) {
518 if err = gulu.File.WriteFileSafer(indexIgnorePath, nil, 0644); err != nil {
519 logging.LogErrorf("create indexignore [%s] failed: %s", indexIgnorePath, err)
520 return
521 }
522 }
523 data, err := os.ReadFile(indexIgnorePath)
524 if err != nil {
525 logging.LogErrorf("read indexignore [%s] failed: %s", indexIgnorePath, err)
526 return
527 }
528 dataStr := string(data)
529 dataStr = strings.ReplaceAll(dataStr, "\r\n", "\n")
530 ret = strings.Split(dataStr, "\n")
531
532 ret = gulu.Str.RemoveDuplicatedElem(ret)
533 if 0 < len(ret) && "" == ret[0] {
534 ret = ret[1:]
535 }
536 indexIgnore = nil
537 for _, line := range ret {
538 indexIgnore = append(indexIgnore, line)
539 }
540 return
541}