From 519f0154986fb71ec5f99695177c68ec844dde8a Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 11:46:51 +0800 Subject: [PATCH 1/5] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=9B=BE?= =?UTF-8?q?=E7=89=87=20OCR=20=E6=8F=90=E5=8F=96=E6=96=87=E6=9C=AC=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E4=B8=AD=E7=9A=84=E5=A4=9A=E4=BD=99=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=20Fix=20https://github.com/siyuan-note/siyuan/issues/7109?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/util/string.go | 29 +++++++++++++++++++++++++++++ kernel/util/tesseract.go | 4 +--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/kernel/util/string.go b/kernel/util/string.go index d982cfcf6..569a4a0f4 100644 --- a/kernel/util/string.go +++ b/kernel/util/string.go @@ -17,7 +17,9 @@ package util import ( + "bytes" "strings" + "unicode" "github.com/88250/lute/html" ) @@ -39,3 +41,30 @@ func Reverse(s string) string { } return string(runes) } + +func RemoveRedundantSpace(str string) string { + buf := bytes.Buffer{} + lastIsChinese := false + lastIsSpace := false + for _, r := range str { + if unicode.IsSpace(r) { + if lastIsChinese || lastIsSpace { + continue + } + buf.WriteRune(' ') + lastIsChinese = false + lastIsSpace = true + continue + } + + lastIsSpace = false + buf.WriteRune(r) + if unicode.Is(unicode.Han, r) { + lastIsChinese = true + continue + } else { + lastIsChinese = false + } + } + return buf.String() +} diff --git a/kernel/util/tesseract.go b/kernel/util/tesseract.go index bc73ee200..04780bd83 100644 --- a/kernel/util/tesseract.go +++ b/kernel/util/tesseract.go @@ -23,7 +23,6 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "strings" "sync" "time" @@ -90,8 +89,7 @@ func Tesseract(imgAbsPath string) string { ret := string(output) ret = gulu.Str.RemoveInvisible(ret) - reg := regexp.MustCompile("\\s{2,}") - ret = reg.ReplaceAllString(ret, " ") + ret = RemoveRedundantSpace(ret) msg := fmt.Sprintf("OCR [%s] [%s]", info.Name(), ret) PushStatusBar(msg) return ret From c87bb94a6049808f96585b4935a8ffe51900b41d Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 13:58:03 +0800 Subject: [PATCH 2/5] =?UTF-8?q?:art:=20=E8=87=AA=E5=8A=A8=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E7=B4=A2=E5=BC=95=E6=97=B6=E6=8E=92=E9=99=A4=E9=9D=9E?= =?UTF-8?q?=20id=20=E6=A0=BC=E5=BC=8F=E7=9A=84=20.sy=20=E6=96=87=E4=BB=B6?= =?UTF-8?q?=20Fix=20https://github.com/siyuan-note/siyuan/issues/7111?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/repository.go | 12 ++++++++++++ kernel/model/transaction.go | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/kernel/model/repository.go b/kernel/model/repository.go index dc1170b30..374b37b42 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -501,6 +501,8 @@ func InitRepoKey() (err error) { return } +var isCheckoutRepo bool + func CheckoutRepo(id string) (err error) { if 1 > len(Conf.Repo.Key) { err = errors.New(Conf.Language(26)) @@ -523,6 +525,16 @@ func CheckoutRepo(id string) (err error) { Conf.Sync.Enabled = false Conf.Save() + if util.IsMutexLocked(&syncLock) { + err = errors.New("Sync is running, please try again later") + return + } + + isCheckoutRepo = true + defer func() { + isCheckoutRepo = false + }() + _, _, err = repo.Checkout(id, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress}) if nil != err { util.PushClearProgress() diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 96b2b2df2..17098e9a0 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -1244,6 +1244,11 @@ func autoFixIndex() { return } + if isCheckoutRepo { + logging.LogInfof("skip check index caused by checkout repo") + return + } + if util.IsMutexLocked(&autoFixLock) { return } @@ -1286,6 +1291,12 @@ func autoFixIndex() { break } + id := path.Base(p) + id = strings.TrimSuffix(id, ".sy") + if !ast.IsNodeIDPattern(id) { + continue + } + reindexTreeByPath(box.ID, p, i, size) } } From 5ada15c0ed817e9a7d123d56f6c7e18b0db9050d Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 13:58:21 +0800 Subject: [PATCH 3/5] =?UTF-8?q?:art:=20=E8=87=AA=E5=8A=A8=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E7=B4=A2=E5=BC=95=E6=97=B6=E6=8E=92=E9=99=A4=E9=9D=9E?= =?UTF-8?q?=20id=20=E6=A0=BC=E5=BC=8F=E7=9A=84=20.sy=20=E6=96=87=E4=BB=B6?= =?UTF-8?q?=20Fix=20https://github.com/siyuan-note/siyuan/issues/7111?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/repository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 374b37b42..8c9fe65a8 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -526,7 +526,7 @@ func CheckoutRepo(id string) (err error) { Conf.Save() if util.IsMutexLocked(&syncLock) { - err = errors.New("Sync is running, please try again later") + err = errors.New("sync is running, please try again later") return } From 9c626ebc0bb987eb968c423cd46f8f069ca4bb8e Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 13:58:58 +0800 Subject: [PATCH 4/5] =?UTF-8?q?:art:=20=E5=B5=8C=E5=85=A5=E5=9D=97?= =?UTF-8?q?=E7=BA=B3=E5=85=A5=E5=BC=95=E7=94=A8=E8=AE=A1=E6=95=B0=E5=92=8C?= =?UTF-8?q?=E5=8F=8D=E9=93=BE=20https://github.com/siyuan-note/siyuan/issu?= =?UTF-8?q?es/7096?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/sql/database.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/kernel/sql/database.go b/kernel/sql/database.go index a0e35aaa6..b11a12003 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -342,8 +342,7 @@ func buildRef(tree *parse.Tree, refNode *ast.Node) *Ref { } func buildEmbedRef(tree *parse.Tree, embedNode *ast.Node) *Ref { - markdown := treenode.ExportNodeStdMd(embedNode, luteEngine) - defBlockID, text := getEmbedRef(embedNode) + defBlockID := getEmbedRef(embedNode) var defBlockParentID, defBlockRootID, defBlockPath string defBlock := treenode.GetBlockTree(defBlockID) if nil != defBlock { @@ -362,19 +361,14 @@ func buildEmbedRef(tree *parse.Tree, embedNode *ast.Node) *Ref { RootID: tree.ID, Box: tree.Box, Path: tree.Path, - Content: text, - Markdown: markdown, + Content: "", + Markdown: "", Type: treenode.TypeAbbr(embedNode.Type.String()), } } -func getEmbedRef(embedNode *ast.Node) (queryBlockID, refText string) { +func getEmbedRef(embedNode *ast.Node) (queryBlockID string) { queryBlockID = treenode.GetEmbedBlockRef(embedNode) - if "" == queryBlockID { - return - } - - refText = getRefText(queryBlockID) return } From 175a938eece22154607d8c18fd247c30fdf22e5a Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 16:20:57 +0800 Subject: [PATCH 5/5] =?UTF-8?q?:art:=20=E6=95=B0=E6=8D=AE=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E5=90=8E=E9=9C=80=E8=A6=81=E9=87=8D=E6=96=B0=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E5=9B=BE=E7=89=87=20OCR=20=E6=8F=90=E5=8F=96=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=20Fix=20https://github.com/siyuan-note/siyuan/issues/?= =?UTF-8?q?7114?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/main.go | 2 +- kernel/mobile/kernel.go | 2 +- kernel/model/assets.go | 2 +- kernel/model/box.go | 2 +- kernel/model/flashcard.go | 4 ++-- kernel/model/repository.go | 16 ++++++++++++++-- kernel/util/file.go | 2 +- 7 files changed, 21 insertions(+), 9 deletions(-) diff --git a/kernel/main.go b/kernel/main.go index 20c1a98f9..ff84ace50 100644 --- a/kernel/main.go +++ b/kernel/main.go @@ -39,7 +39,7 @@ func main() { model.BootSyncData() model.InitBoxes() - model.InitFlashcards() + model.LoadFlashcards() model.LoadAssetsTexts() go model.AutoGenerateDocHistory() diff --git a/kernel/mobile/kernel.go b/kernel/mobile/kernel.go index fa3ef2690..a5feb8438 100644 --- a/kernel/mobile/kernel.go +++ b/kernel/mobile/kernel.go @@ -53,7 +53,7 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang model.BootSyncData() model.InitBoxes() - model.InitFlashcards() + model.LoadFlashcards() model.LoadAssetsTexts() go model.AutoGenerateDocHistory() diff --git a/kernel/model/assets.go b/kernel/model/assets.go index 50213850a..74139a08e 100644 --- a/kernel/model/assets.go +++ b/kernel/model/assets.go @@ -176,7 +176,7 @@ func NetImg2LocalAssets(rootID string) (err error) { } } name = strings.TrimSuffix(name, ext) - name = gulu.Str.SubStr(name, 63) // 插入资源文件时文件名长度最大限制 189 字节 https://github.com/siyuan-note/siyuan/issues/7099 + name = gulu.Str.SubStr(name, 63) // 插入资源文件时文件名长度最大限制 63 个字 https://github.com/siyuan-note/siyuan/issues/7099 name = util.FilterFileName(name) name = "net-img-" + name + "-" + ast.NewNodeID() + ext writePath := filepath.Join(assetsDirPath, name) diff --git a/kernel/model/box.go b/kernel/model/box.go index a7d93f571..a90fb5677 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -523,7 +523,7 @@ func FullReindex() { } IndexRefs() treenode.SaveBlockTree(true) - InitFlashcards() + LoadFlashcards() util.PushEndlessProgress(Conf.Language(58)) isFullReindexing = false diff --git a/kernel/model/flashcard.go b/kernel/model/flashcard.go index b782f6eb9..bd83bead1 100644 --- a/kernel/model/flashcard.go +++ b/kernel/model/flashcard.go @@ -359,7 +359,7 @@ func AddFlashcards(deckID string, blockIDs []string) (err error) { return } -func InitFlashcards() { +func LoadFlashcards() { riffSavePath := getRiffDir() if err := os.MkdirAll(riffSavePath, 0755); nil != err { logging.LogErrorf("create riff dir [%s] failed: %s", riffSavePath, err) @@ -445,7 +445,7 @@ func RemoveDeck(deckID string) (err error) { } } - InitFlashcards() + LoadFlashcards() return } diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 8c9fe65a8..767fb2489 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -989,13 +989,17 @@ func syncRepo(exit, byHand bool) (err error) { // 有数据变更，需要重建索引 var upserts, removes []string var upsertTrees int - var needReloadFlashcard bool + var needReloadFlashcard, needReloadOcrTexts bool for _, file := range mergeResult.Upserts { upserts = append(upserts, file.Path) if strings.HasPrefix(file.Path, "/storage/riff/") { needReloadFlashcard = true } + if strings.HasPrefix(file.Path, "/data/assets/ocr-texts.json") { + needReloadOcrTexts = true + } + if strings.HasSuffix(file.Path, ".sy") { upsertTrees++ } @@ -1005,10 +1009,18 @@ func syncRepo(exit, byHand bool) (err error) { if strings.HasPrefix(file.Path, "/storage/riff/") { needReloadFlashcard = true } + + if strings.HasPrefix(file.Path, "/data/assets/ocr-texts.json") { + needReloadOcrTexts = true + } } if needReloadFlashcard { - InitFlashcards() + LoadFlashcards() + } + + if needReloadOcrTexts { + LoadAssetsTexts() } cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939 diff --git a/kernel/util/file.go b/kernel/util/file.go index 61006a6c7..836a2ae2c 100644 --- a/kernel/util/file.go +++ b/kernel/util/file.go @@ -141,7 +141,7 @@ func FilterUploadFileName(name string) string { ret = strings.ReplaceAll(ret, "#", "") ret = strings.ReplaceAll(ret, "%", "") ret = strings.ReplaceAll(ret, "$", "") - // 插入资源文件时文件名长度最大限制 189 字节 https://github.com/siyuan-note/siyuan/issues/7099 + // 插入资源文件时文件名长度最大限制 63 个字 https://github.com/siyuan-note/siyuan/issues/7099 ret = gulu.Str.SubStr(ret, 63) return ret }