onedeuxtriseigo.nullpo.dev
A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
1// SiYuan - Refactor your thinking
2// Copyright (c) 2020-present, b3log.org
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU Affero General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU Affero General Public License for more details.
13//
14// You should have received a copy of the GNU Affero General Public License
15// along with this program. If not, see <https://www.gnu.org/licenses/>.
16
17package api
18
19import (
20 "bytes"
21 "io"
22 "net/url"
23 "os"
24 "path"
25 "path/filepath"
26 "regexp"
27 "strconv"
28 "strings"
29
30 "github.com/88250/gulu"
31 "github.com/88250/lute"
32 "github.com/88250/lute/ast"
33 "github.com/88250/lute/parse"
34 "github.com/gabriel-vasile/mimetype"
35 "github.com/gin-gonic/gin"
36 "github.com/siyuan-note/filelock"
37 "github.com/siyuan-note/httpclient"
38 "github.com/siyuan-note/logging"
39 "github.com/siyuan-note/siyuan/kernel/model"
40 "github.com/siyuan-note/siyuan/kernel/util"
41)
42
43func extensionCopy(c *gin.Context) {
44 ret := gulu.Ret.NewResult()
45 defer c.JSON(200, ret)
46
47 form, _ := c.MultipartForm()
48 dom := form.Value["dom"][0]
49 assets := filepath.Join(util.DataDir, "assets")
50 if notebookVal := form.Value["notebook"]; 0 < len(notebookVal) {
51 assets = filepath.Join(util.DataDir, notebookVal[0], "assets")
52 if !gulu.File.IsDir(assets) {
53 assets = filepath.Join(util.DataDir, "assets")
54 }
55 }
56
57 if err := os.MkdirAll(assets, 0755); err != nil {
58 logging.LogErrorf("create assets folder [%s] failed: %s", assets, err)
59 ret.Msg = err.Error()
60 return
61 }
62
63 clippingSym := false
64 symArticleHref := ""
65 hasHref := nil != form.Value["href"]
66 isPartClip := nil != form.Value["clipType"] && form.Value["clipType"][0] == "part"
67 if hasHref && !isPartClip {
68 // 剪藏链滴帖子时直接使用 Markdown 接口的返回
69 // https://ld246.com/article/raw/1724850322251
70 symArticleHref = form.Value["href"][0]
71
72 var baseURL, originalPrefix string
73 if strings.HasPrefix(symArticleHref, "https://ld246.com/article/") {
74 baseURL = "https://ld246.com/article/raw/"
75 originalPrefix = "https://ld246.com/article/"
76 } else if strings.HasPrefix(symArticleHref, "https://liuyun.io/article/") {
77 baseURL = "https://liuyun.io/article/raw/"
78 originalPrefix = "https://liuyun.io/article/"
79 }
80
81 if "" != baseURL {
82 articleID := strings.TrimPrefix(symArticleHref, originalPrefix)
83 if idx := strings.IndexAny(articleID, "/?#"); -1 != idx {
84 articleID = articleID[:idx]
85 }
86
87 symArticleHref = baseURL + articleID
88 clippingSym = true
89 }
90 }
91
92 uploaded := map[string]string{}
93 for originalName, file := range form.File {
94 oName, err := url.PathUnescape(originalName)
95 unescaped := oName
96
97 if clippingSym && strings.Contains(oName, "img-loading.svg") {
98 continue
99 }
100
101 if err != nil {
102 if strings.Contains(originalName, "%u") {
103 originalName = strings.ReplaceAll(originalName, "%u", "\\u")
104 originalName, err = strconv.Unquote("\"" + originalName + "\"")
105 if err != nil {
106 continue
107 }
108 oName, err = url.PathUnescape(originalName)
109 if err != nil {
110 continue
111 }
112 } else {
113 continue
114 }
115 }
116 if strings.Contains(oName, "%") {
117 unescaped, _ := url.PathUnescape(oName)
118 if "" != unescaped {
119 oName = unescaped
120 }
121 }
122
123 u, _ := url.Parse(oName)
124 if "" == u.Path {
125 continue
126 }
127 fName := path.Base(u.Path)
128
129 f, err := file[0].Open()
130 if err != nil {
131 ret.Code = -1
132 ret.Msg = err.Error()
133 break
134 }
135
136 data, err := io.ReadAll(f)
137 if err != nil {
138 ret.Code = -1
139 ret.Msg = err.Error()
140 break
141 }
142
143 fName = util.FilterUploadFileName(fName)
144 ext := util.Ext(fName)
145 if !util.IsCommonExt(ext) || strings.Contains(ext, "!") {
146 // 改进浏览器剪藏扩展转换本地图片后缀 https://github.com/siyuan-note/siyuan/issues/7467 https://github.com/siyuan-note/siyuan/issues/15320
147 if mtype := mimetype.Detect(data); nil != mtype {
148 ext = mtype.Extension()
149 fName += ext
150 }
151 }
152 if "" == ext && bytes.HasPrefix(data, []byte("<svg ")) && bytes.HasSuffix(data, []byte("</svg>")) {
153 ext = ".svg"
154 fName += ext
155 }
156
157 fName = util.AssetName(fName, ast.NewNodeID())
158 writePath := filepath.Join(assets, fName)
159 if err = filelock.WriteFile(writePath, data); err != nil {
160 ret.Code = -1
161 ret.Msg = err.Error()
162 break
163 }
164
165 uploaded[unescaped] = "assets/" + fName
166 }
167
168 luteEngine := util.NewLute()
169 luteEngine.SetHTMLTag2TextMark(true)
170 var md string
171 var withMath bool
172
173 if clippingSym {
174 resp, err := httpclient.NewCloudRequest30s().Get(symArticleHref)
175 if err != nil {
176 logging.LogWarnf("get [%s] failed: %s", symArticleHref, err)
177 } else {
178 bodyData, readErr := io.ReadAll(resp.Body)
179 if nil != readErr {
180 ret.Code = -1
181 ret.Msg = "read response body failed: " + readErr.Error()
182 return
183 }
184
185 md = string(bodyData)
186 luteEngine.SetIndentCodeBlock(true) // 链滴支持缩进代码块，因此需要开启
187 tree := parse.Parse("", []byte(md), luteEngine.ParseOptions)
188 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
189 if ast.NodeInlineMath == n.Type {
190 withMath = true
191 return ast.WalkStop
192 } else if ast.NodeCodeBlock == n.Type {
193 if !n.IsFencedCodeBlock {
194 // 将缩进代码块转换为围栏代码块
195 n.IsFencedCodeBlock = true
196 n.CodeBlockFenceChar = '`'
197 n.PrependChild(&ast.Node{Type: ast.NodeCodeBlockFenceInfoMarker})
198 n.PrependChild(&ast.Node{Type: ast.NodeCodeBlockFenceOpenMarker, Tokens: []byte("```"), CodeBlockFenceLen: 3})
199 n.LastChild.InsertAfter(&ast.Node{Type: ast.NodeCodeBlockFenceCloseMarker, Tokens: []byte("```"), CodeBlockFenceLen: 3})
200 code := n.ChildByType(ast.NodeCodeBlockCode)
201 if nil != code {
202 code.Tokens = bytes.TrimPrefix(code.Tokens, []byte(" "))
203 code.Tokens = bytes.ReplaceAll(code.Tokens, []byte("\n "), []byte("\n"))
204 code.Tokens = bytes.TrimPrefix(code.Tokens, []byte("\t"))
205 code.Tokens = bytes.ReplaceAll(code.Tokens, []byte("\n\t"), []byte("\n"))
206 }
207 }
208 }
209 return ast.WalkContinue
210 })
211
212 md, _ = lute.FormatNodeSync(tree.Root, luteEngine.ParseOptions, luteEngine.RenderOptions)
213 }
214 }
215
216 var tree *parse.Tree
217 if "" == md {
218 // 通过正则将 <iframe>.*</iframe> 标签中间包含的换行去掉
219 regx, _ := regexp.Compile(`(?i)<iframe[^>]*>([\s\S]*?)<\/iframe>`)
220 dom = regx.ReplaceAllStringFunc(dom, func(s string) string {
221 s = strings.ReplaceAll(s, "\n", "")
222 s = strings.ReplaceAll(s, "\r", "")
223 return s
224 })
225
226 tree, withMath = model.HTML2Tree(dom, luteEngine)
227 if nil == tree {
228 md, withMath, _ = model.HTML2Markdown(dom, luteEngine)
229 if withMath {
230 luteEngine.SetInlineMath(true)
231 }
232 tree = parse.Parse("", []byte(md), luteEngine.ParseOptions)
233 }
234 } else {
235 tree = parse.Parse("", []byte(md), luteEngine.ParseOptions)
236 }
237
238 var unlinks []*ast.Node
239 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
240 if !entering {
241 return ast.WalkContinue
242 }
243
244 if ast.NodeText == n.Type {
245 // 剔除行首空白
246 if ast.NodeParagraph == n.Parent.Type && n.Parent.FirstChild == n {
247 n.Tokens = bytes.TrimLeft(n.Tokens, " \t\n")
248 }
249 } else if ast.NodeImage == n.Type {
250 if dest := n.ChildByType(ast.NodeLinkDest); nil != dest {
251 assetPath := uploaded[string(dest.Tokens)]
252 if "" == assetPath {
253 assetPath = uploaded[string(dest.Tokens)+"?imageView2/2/interlace/1/format/webp"]
254 }
255 if "" != assetPath {
256 dest.Tokens = []byte(assetPath)
257 }
258
259 // 检测 alt 和 title 格式，如果不是文本的话转换为文本 https://github.com/siyuan-note/siyuan/issues/14233
260 if linkText := n.ChildByType(ast.NodeLinkText); nil != linkText {
261 if inlineTree := parse.Inline("", linkText.Tokens, luteEngine.ParseOptions); nil != inlineTree && nil != inlineTree.Root && nil != inlineTree.Root.FirstChild {
262 if fc := inlineTree.Root.FirstChild.FirstChild; nil != fc {
263 if ast.NodeText != fc.Type {
264 linkText.Tokens = []byte(fc.Text())
265 }
266 }
267 }
268 }
269 if title := n.ChildByType(ast.NodeLinkTitle); nil != title {
270 if inlineTree := parse.Inline("", title.Tokens, luteEngine.ParseOptions); nil != inlineTree && nil != inlineTree.Root && nil != inlineTree.Root.FirstChild {
271 if fc := inlineTree.Root.FirstChild.FirstChild; nil != fc {
272 if ast.NodeText != fc.Type {
273 title.Tokens = []byte(fc.Text())
274 }
275 }
276 }
277 }
278 }
279 }
280 return ast.WalkContinue
281 })
282 for _, unlink := range unlinks {
283 unlink.Unlink()
284 }
285
286 parse.TextMarks2Inlines(tree) // 先将 TextMark 转换为 Inlines https://github.com/siyuan-note/siyuan/issues/13056
287 parse.NestedInlines2FlattedSpansHybrid(tree, false)
288
289 md, _ = lute.FormatNodeSync(tree.Root, luteEngine.ParseOptions, luteEngine.RenderOptions)
290 ret.Data = map[string]interface{}{
291 "md": md,
292 "withMath": withMath,
293 }
294 ret.Msg = model.Conf.Language(72)
295}