From 31be236557b4d4b3cb7f39f159ad9e0a4b858454 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 22 Nov 2022 10:14:23 +0800 Subject: [PATCH 1/4] =?UTF-8?q?:art:=20=E5=AF=BC=E5=85=A5=20Markdown=20?= =?UTF-8?q?=E6=97=B6=E5=B0=86=20Base64=20=E7=BC=96=E7=A0=81=E7=9A=84?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E8=BD=AC=E6=8D=A2=E4=B8=BA=E6=96=87=E4=BB=B6?= =?UTF-8?q?=20Fix=20https://github.com/siyuan-note/siyuan/issues/6671?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/box.go | 25 +++++++++++- kernel/model/import.go | 90 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/kernel/model/box.go b/kernel/model/box.go index 0da197d69..caa715c69 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -30,6 +30,7 @@ import ( "time" "github.com/88250/gulu" + "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" "github.com/dustin/go-humanize" @@ -439,10 +440,30 @@ func moveTree(tree *parse.Tree) { } } +func parseStdMd(markdown []byte) (ret *parse.Tree) { + luteEngine := lute.New() + luteEngine.SetFootnotes(false) + luteEngine.SetToC(false) + luteEngine.SetIndentCodeBlock(false) + luteEngine.SetAutoSpace(false) + luteEngine.SetHeadingID(false) + luteEngine.SetSetext(false) + luteEngine.SetYamlFrontMatter(false) + luteEngine.SetLinkRef(false) + ret = parse.Parse("", markdown, luteEngine.ParseOptions) + genTreeID(ret) + return +} + func parseKTree(kramdown []byte) (ret *parse.Tree) { luteEngine := NewLute() ret = parse.Parse("", kramdown, luteEngine.ParseOptions) - ast.Walk(ret.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + genTreeID(ret) + return +} + +func genTreeID(tree *parse.Tree) { + ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering { return ast.WalkContinue } @@ -475,7 +496,7 @@ func parseKTree(kramdown []byte) (ret *parse.Tree) { } return ast.WalkContinue }) - ret.Root.KramdownIAL = parse.Tokens2IAL(ret.Root.LastChild.Tokens) + tree.Root.KramdownIAL = parse.Tokens2IAL(tree.Root.LastChild.Tokens) return } diff --git a/kernel/model/import.go b/kernel/model/import.go index e7fbc73a1..1336b8f84 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -18,9 +18,13 @@ package model import ( "bytes" + "encoding/base64" "encoding/json" "errors" "fmt" + "image" + "image/jpeg" + "image/png" "io" "io/fs" "math/rand" @@ -403,6 +407,9 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { } boxLocalPath = filepath.Join(util.DataDir, boxID) + base64TmpDir := filepath.Join(util.TempDir, "base64") + os.MkdirAll(base64TmpDir, 0755) + luteEngine := NewLute() if gulu.File.IsDir(localPath) { // 收集所有资源文件 @@ -419,8 +426,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { } if !strings.HasSuffix(info.Name(), ".md") && !strings.HasSuffix(info.Name(), ".markdown") { - dest := currentPath - assets[dest] = currentPath + assets[currentPath] = currentPath return nil } return nil @@ -480,7 +486,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { return io.EOF } - tree = parseKTree(data) + tree = parseStdMd(data) if nil == tree { logging.LogErrorf("parse tree [%s] failed", currentPath) return nil @@ -507,6 +513,11 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { } dest := n.TokensStr() + if strings.HasPrefix(dest, "data:image") && strings.Contains(dest, ";base64,") { + processBase64Img(n, dest, base64TmpDir, assetDirPath, err) + return ast.WalkContinue + } + if !util.IsRelativePath(dest) || "" == dest { return ast.WalkContinue } @@ -570,7 +581,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { if nil != err { return err } - tree := parseKTree(data) + tree := parseStdMd(data) if nil == tree { msg := fmt.Sprintf("parse tree [%s] failed", localPath) logging.LogErrorf(msg) @@ -595,6 +606,11 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { } dest := n.TokensStr() + if strings.HasPrefix(dest, "data:image") && strings.Contains(dest, ";base64,") { + processBase64Img(n, dest, base64TmpDir, assetDirPath, err) + return ast.WalkContinue + } + if !util.IsRelativePath(dest) { return ast.WalkContinue } @@ -641,6 +657,71 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { return } +func processBase64Img(n *ast.Node, dest string, base64TmpDir string, assetDirPath string, err error) { + sep := strings.Index(dest, ";base64,") + var decodeErr error + unbased, decodeErr := base64.StdEncoding.DecodeString(dest[sep+8:]) + if nil != decodeErr { + logging.LogErrorf("decode base64 image failed: %s", decodeErr) + return + } + dataReader := bytes.NewReader(unbased) + var img image.Image + var ext string + typ := dest[5:sep] + switch typ { + case "image/png": + img, decodeErr = png.Decode(dataReader) + ext = ".png" + case "image/jpeg": + img, decodeErr = jpeg.Decode(dataReader) + ext = ".jpg" + default: + logging.LogWarnf("unsupported base64 image type [%s]", typ) + return + } + if nil != decodeErr { + logging.LogErrorf("decode base64 image failed: %s", decodeErr) + return + } + + name := "image" + ext + alt := n.Parent.ChildByType(ast.NodeLinkText) + if nil != alt { + name = alt.TokensStr() + ext + } + name = util.FilterFileName(name) + name = util.AssetName(name) + + tmp := filepath.Join(base64TmpDir, name) + tmpFile, openErr := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE, 0644) + if nil != openErr { + logging.LogErrorf("open temp file [%s] failed: %s", tmp, openErr) + return + } + + var encodeErr error + switch typ { + case "image/png": + encodeErr = png.Encode(tmpFile, img) + case "image/jpeg": + encodeErr = jpeg.Encode(tmpFile, img, &jpeg.Options{Quality: 100}) + } + if nil != encodeErr { + logging.LogErrorf("encode base64 image failed: %s", encodeErr) + tmpFile.Close() + return + } + tmpFile.Close() + + assetTargetPath := filepath.Join(assetDirPath, name) + if err = filelock.Copy(tmp, assetTargetPath); nil != err { + logging.LogErrorf("copy asset from [%s] to [%s] failed: %s", tmp, assetTargetPath, err) + return + } + n.Tokens = []byte("assets/" + name) +} + func imgHtmlBlock2InlineImg(tree *parse.Tree) { imgHtmlBlocks := map[*ast.Node]*html.Node{} ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { @@ -691,7 +772,6 @@ func imgHtmlBlock2InlineImg(tree *parse.Tree) { n.InsertBefore(p) n.Unlink() } - return } From 8652c4cef780284d35f4522b5093ac17d4c21cb4 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 22 Nov 2022 10:38:13 +0800 Subject: [PATCH 2/4] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E4=B9=A6?= =?UTF-8?q?=E7=AD=BE=E9=9D=A2=E6=9D=BF=E5=8A=A0=E8=BD=BD=E9=80=9F=E5=BA=A6?= =?UTF-8?q?=20https://github.com/siyuan-note/siyuan/issues/6677?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/bookmark.go | 4 +++- kernel/sql/queue.go | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/kernel/model/bookmark.go b/kernel/model/bookmark.go index 5dd1ac861..64db0b445 100644 --- a/kernel/model/bookmark.go +++ b/kernel/model/bookmark.go @@ -160,7 +160,9 @@ func BookmarkLabels() (ret []string) { func BuildBookmark() (ret *Bookmarks) { WaitForWritingFiles() - sql.WaitForWritingDatabase() + if !sql.IsEmptyQueue() { + sql.WaitForWritingDatabase() + } ret = &Bookmarks{} sqlBlocks := sql.QueryBookmarkBlocks() diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index afc88d467..02e796089 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -81,6 +81,10 @@ func isWritingDatabase() bool { return false } +func IsEmptyQueue() bool { + return 1 > len(operationQueue) && !util.IsMutexLocked(&txLock) +} + func flushTreeQueue() { ops := mergeUpsertTrees() if 1 > len(ops) { From 443c43753fcca0c38665dd9aff32355b2db64609 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 22 Nov 2022 10:40:54 +0800 Subject: [PATCH 3/4] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E4=B9=A6?= =?UTF-8?q?=E7=AD=BE=E5=92=8C=E6=A0=87=E7=AD=BE=E9=9D=A2=E6=9D=BF=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E9=80=9F=E5=BA=A6=20https://github.com/siyuan-note/si?= =?UTF-8?q?yuan/issues/6677?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/tag.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/model/tag.go b/kernel/model/tag.go index 461d505e9..0da6a8aee 100644 --- a/kernel/model/tag.go +++ b/kernel/model/tag.go @@ -233,7 +233,9 @@ type Tags []*Tag func BuildTags() (ret *Tags) { WaitForWritingFiles() - sql.WaitForWritingDatabase() + if !sql.IsEmptyQueue() { + sql.WaitForWritingDatabase() + } ret = &Tags{} labels := labelTags() From 876e4425bb2aa25cbfde1843d1c20f5c15dc6416 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 22 Nov 2022 10:46:08 +0800 Subject: [PATCH 4/4] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E4=B9=A6?= =?UTF-8?q?=E7=AD=BE=E5=92=8C=E6=A0=87=E7=AD=BE=E9=9D=A2=E6=9D=BF=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E9=80=9F=E5=BA=A6=20https://github.com/siyuan-note/si?= =?UTF-8?q?yuan/issues/6677?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/bookmark.go | 2 ++ kernel/model/tag.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/kernel/model/bookmark.go b/kernel/model/bookmark.go index 64db0b445..1ab9551fb 100644 --- a/kernel/model/bookmark.go +++ b/kernel/model/bookmark.go @@ -162,6 +162,8 @@ func BuildBookmark() (ret *Bookmarks) { WaitForWritingFiles() if !sql.IsEmptyQueue() { sql.WaitForWritingDatabase() + } else { + util.RandomSleep(200, 500) } ret = &Bookmarks{} diff --git a/kernel/model/tag.go b/kernel/model/tag.go index 0da6a8aee..a56ce975c 100644 --- a/kernel/model/tag.go +++ b/kernel/model/tag.go @@ -235,6 +235,8 @@ func BuildTags() (ret *Tags) { WaitForWritingFiles() if !sql.IsEmptyQueue() { sql.WaitForWritingDatabase() + } else { + util.RandomSleep(200, 500) } ret = &Tags{}