From 07a5e7501a8f55f76492827c668f7bfed941adf2 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 9 Jan 2023 17:02:38 +0800 Subject: [PATCH 1/5] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E9=87=8D?= =?UTF-8?q?=E5=BB=BA=E7=B4=A2=E5=BC=95=20https://github.com/siyuan-note/si?= =?UTF-8?q?yuan/issues/7018?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/filesys/tree.go | 19 +++++++++++++------ kernel/model/file.go | 9 +-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/kernel/filesys/tree.go b/kernel/filesys/tree.go index 916efa5ce..c539db0ad 100644 --- a/kernel/filesys/tree.go +++ b/kernel/filesys/tree.go @@ -29,6 +29,7 @@ import ( "github.com/88250/lute" "github.com/88250/lute/parse" "github.com/88250/lute/render" + jsoniter "github.com/json-iterator/go" "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/cache" @@ -88,13 +89,13 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro hPathBuilder.WriteString("Untitled/") continue } - parentTree, parseErr := parse.ParseJSONWithoutFix(parentData, luteEngine.ParseOptions) - if nil != parseErr { - logging.LogWarnf("parse parent tree [%s] failed: %s", parentAbsPath, parseErr) - hPathBuilder.WriteString("Untitled/") - continue + + ial := ReadDocIAL(parentData) + title := ial["title"] + if "" == title { + title = "Untitled" } - hPathBuilder.WriteString(parentTree.Root.IALAttr("title")) + hPathBuilder.WriteString(title) hPathBuilder.WriteString("/") } hPathBuilder.WriteString(ret.Root.IALAttr("title")) @@ -262,3 +263,9 @@ func parseJSON2Tree(boxID, p string, jsonData []byte, luteEngine *lute.Lute) (re } return } + +func ReadDocIAL(data []byte) (ret map[string]string) { + ret = map[string]string{} + jsoniter.Get(data, "Properties").ToVal(&ret) + return +} diff --git a/kernel/model/file.go b/kernel/model/file.go index 9c885effd..165569ebd 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -37,7 +37,6 @@ import ( "github.com/dustin/go-humanize" "github.com/facette/natsort" "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/cache" @@ -143,7 +142,7 @@ func (box *Box) docIAL(p string) (ret map[string]string) { logging.LogErrorf("read file [%s] failed: %s", p, err) return nil } - ret = readDocIAL(data) + ret = filesys.ReadDocIAL(data) if 1 > len(ret) { logging.LogWarnf("tree [%s] is corrupted", filePath) box.moveCorruptedData(filePath) @@ -167,12 +166,6 @@ func (box *Box) moveCorruptedData(filePath string) { logging.LogWarnf("moved corrupted data file [%s] to [%s]", filePath, to) } -func readDocIAL(data []byte) (ret map[string]string) { - ret = map[string]string{} - jsoniter.Get(data, "Properties").ToVal(&ret) - return -} - func SearchDocsByKeyword(keyword string) (ret []map[string]string) { ret = []map[string]string{} From 5a09669d3a4815274f5f2079f9df106fb882851e Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 9 Jan 2023 17:30:19 +0800 Subject: [PATCH 2/5] =?UTF-8?q?:art:=20=E8=87=AA=E5=8A=A8=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E7=B4=A2=E5=BC=95=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7016?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/transaction.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 4992a1b41..8b3f42be5 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -1238,6 +1238,29 @@ func autoFixIndex() { autoFixLock.Lock() defer autoFixLock.Unlock() + // 根据文件系统补全块树 + boxes := Conf.GetOpenedBoxes() + for _, box := range boxes { + boxPath := filepath.Join(util.DataDir, box.ID) + var paths []string + filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && filepath.Ext(path) == ".sy" { + p := path[len(boxPath):] + p = filepath.ToSlash(p) + paths = append(paths, p) + } + return nil + }) + + size := len(paths) + for i, p := range paths { + if nil == treenode.GetBlockTreeRootByPath(box.ID, p) { + reindexTreeByPath(box.ID, p, i, size) + } + } + } + + // 对比块树和数据库并订正数据库 rootUpdatedMap := treenode.GetRootUpdated() dbRootUpdatedMap, err := sql.GetRootUpdated() if nil == err { @@ -1273,6 +1296,7 @@ func autoFixIndex() { } } + // 去除重复的数据库块记录 duplicatedRootIDs := sql.GetDuplicatedRootIDs() size := len(duplicatedRootIDs) for i, rootID := range duplicatedRootIDs { @@ -1287,6 +1311,15 @@ func autoFixIndex() { } } +func reindexTreeByPath(box, p string, i, size int) { + tree, err := LoadTree(box, p) + if nil != err { + return + } + + reindexTree0(tree, i, size) +} + func reindexTree(rootID string, i, size int) { root := treenode.GetBlockTree(rootID) if nil == root { @@ -1303,6 +1336,10 @@ func reindexTree(rootID string, i, size int) { return } + reindexTree0(tree, i, size) +} + +func reindexTree0(tree *parse.Tree, i, size int) { updated := tree.Root.IALAttr("updated") if "" == updated { updated = util.TimeFromID(tree.Root.ID) From 9474f558adbb0464a9b9900c44f07cda4d574cb0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 9 Jan 2023 17:37:28 +0800 Subject: [PATCH 3/5] =?UTF-8?q?:art:=20=E8=87=AA=E5=8A=A8=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E7=B4=A2=E5=BC=95=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7016?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/transaction.go | 7 +++---- kernel/treenode/blocktree.go | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 8b3f42be5..aad8d7b51 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -1253,10 +1253,9 @@ func autoFixIndex() { }) size := len(paths) - for i, p := range paths { - if nil == treenode.GetBlockTreeRootByPath(box.ID, p) { - reindexTreeByPath(box.ID, p, i, size) - } + missingPaths := treenode.GetNotExistPaths(box.ID, paths) + for i, p := range missingPaths { + reindexTreeByPath(box.ID, p, i, size) } } diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index 6d29430fe..cbf59c79f 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -116,6 +116,27 @@ func CeilBlockCount(count int) int { return 10000*100 + 1 } +func GetNotExistPaths(boxID string, paths []string) (ret []string) { + blockTreesLock.Lock() + defer blockTreesLock.Unlock() + + pathsMap := map[string]bool{} + for _, path := range paths { + pathsMap[path] = true + } + + for _, blockTree := range blockTrees { + if blockTree.BoxID != boxID { + continue + } + + if !pathsMap[blockTree.Path] { + ret = append(ret, blockTree.Path) + } + } + return +} + func GetBlockTreeRootByPath(boxID, path string) *BlockTree { blockTreesLock.Lock() defer blockTreesLock.Unlock() From e8012158ea5ec6d18b084db576590af10631e6e0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 9 Jan 2023 17:38:41 +0800 Subject: [PATCH 4/5] =?UTF-8?q?:art:=20=E8=87=AA=E5=8A=A8=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E7=B4=A2=E5=BC=95=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7016?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/transaction.go | 2 ++ kernel/treenode/blocktree.go | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index aad8d7b51..60866ef1c 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -1231,6 +1231,8 @@ func AutoFixIndex() { var autoFixLock = sync.Mutex{} func autoFixIndex() { + defer logging.Recover() + if util.IsMutexLocked(&autoFixLock) || isFullReindexing { return } diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index cbf59c79f..f40064ae8 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -117,15 +117,14 @@ func CeilBlockCount(count int) int { } func GetNotExistPaths(boxID string, paths []string) (ret []string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() + tmp := blockTrees pathsMap := map[string]bool{} for _, path := range paths { pathsMap[path] = true } - for _, blockTree := range blockTrees { + for _, blockTree := range tmp { if blockTree.BoxID != boxID { continue } From 58975bf4eafd518975e82e63c088a7ff005bd9fc Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 9 Jan 2023 17:39:02 +0800 Subject: [PATCH 5/5] =?UTF-8?q?:art:=20=E8=87=AA=E5=8A=A8=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E6=95=B0=E6=8D=AE=E5=BA=93=E7=B4=A2=E5=BC=95=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7016?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/treenode/blocktree.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index f40064ae8..d1a26f2f5 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -117,13 +117,12 @@ func CeilBlockCount(count int) int { } func GetNotExistPaths(boxID string, paths []string) (ret []string) { - tmp := blockTrees - pathsMap := map[string]bool{} for _, path := range paths { pathsMap[path] = true } + tmp := blockTrees for _, blockTree := range tmp { if blockTree.BoxID != boxID { continue