Merge remote-tracking branch 'origin/dev' into dev

kernel/filesys/tree.go

@@ -29,6 +29,7 @@ import (
 	"github.com/88250/lute"
 	"github.com/88250/lute/parse"
 	"github.com/88250/lute/render"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/cache"
@@ -88,13 +89,13 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
 			hPathBuilder.WriteString("Untitled/")
 			continue
 		}
-		parentTree, parseErr := parse.ParseJSONWithoutFix(parentData, luteEngine.ParseOptions)
-		if nil != parseErr {
-			logging.LogWarnf("parse parent tree [%s] failed: %s", parentAbsPath, parseErr)
-			hPathBuilder.WriteString("Untitled/")
-			continue
+
+		ial := ReadDocIAL(parentData)
+		title := ial["title"]
+		if "" == title {
+			title = "Untitled"
 		}
-		hPathBuilder.WriteString(parentTree.Root.IALAttr("title"))
+		hPathBuilder.WriteString(title)
 		hPathBuilder.WriteString("/")
 	}
 	hPathBuilder.WriteString(ret.Root.IALAttr("title"))
@@ -262,3 +263,9 @@ func parseJSON2Tree(boxID, p string, jsonData []byte, luteEngine *lute.Lute) (re
 	}
 	return
 }
+
+func ReadDocIAL(data []byte) (ret map[string]string) {
+	ret = map[string]string{}
+	jsoniter.Get(data, "Properties").ToVal(&ret)
+	return
+}

kernel/model/file.go

@@ -37,7 +37,6 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/facette/natsort"
 	"github.com/gin-gonic/gin"
-	jsoniter "github.com/json-iterator/go"
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/cache"
@@ -143,7 +142,7 @@ func (box *Box) docIAL(p string) (ret map[string]string) {
 		logging.LogErrorf("read file [%s] failed: %s", p, err)
 		return nil
 	}
-	ret = readDocIAL(data)
+	ret = filesys.ReadDocIAL(data)
 	if 1 > len(ret) {
 		logging.LogWarnf("tree [%s] is corrupted", filePath)
 		box.moveCorruptedData(filePath)
@@ -167,12 +166,6 @@ func (box *Box) moveCorruptedData(filePath string) {
 	logging.LogWarnf("moved corrupted data file [%s] to [%s]", filePath, to)
 }
 
-func readDocIAL(data []byte) (ret map[string]string) {
-	ret = map[string]string{}
-	jsoniter.Get(data, "Properties").ToVal(&ret)
-	return
-}
-
 func SearchDocsByKeyword(keyword string) (ret []map[string]string) {
 	ret = []map[string]string{}
 

kernel/model/transaction.go

@@ -1231,6 +1231,8 @@ func AutoFixIndex() {
 var autoFixLock = sync.Mutex{}
 
 func autoFixIndex() {
+	defer logging.Recover()
+
 	if util.IsMutexLocked(&autoFixLock) || isFullReindexing {
 		return
 	}
@@ -1238,6 +1240,28 @@ func autoFixIndex() {
 	autoFixLock.Lock()
 	defer autoFixLock.Unlock()
 
+	// 根据文件系统补全块树
+	boxes := Conf.GetOpenedBoxes()
+	for _, box := range boxes {
+		boxPath := filepath.Join(util.DataDir, box.ID)
+		var paths []string
+		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
+			if !info.IsDir() && filepath.Ext(path) == ".sy" {
+				p := path[len(boxPath):]
+				p = filepath.ToSlash(p)
+				paths = append(paths, p)
+			}
+			return nil
+		})
+
+		size := len(paths)
+		missingPaths := treenode.GetNotExistPaths(box.ID, paths)
+		for i, p := range missingPaths {
+			reindexTreeByPath(box.ID, p, i, size)
+		}
+	}
+
+	// 对比块树和数据库并订正数据库
 	rootUpdatedMap := treenode.GetRootUpdated()
 	dbRootUpdatedMap, err := sql.GetRootUpdated()
 	if nil == err {
@@ -1273,6 +1297,7 @@ func autoFixIndex() {
 		}
 	}
 
+	// 去除重复的数据库块记录
 	duplicatedRootIDs := sql.GetDuplicatedRootIDs()
 	size := len(duplicatedRootIDs)
 	for i, rootID := range duplicatedRootIDs {
@@ -1287,6 +1312,15 @@ func autoFixIndex() {
 	}
 }
 
+func reindexTreeByPath(box, p string, i, size int) {
+	tree, err := LoadTree(box, p)
+	if nil != err {
+		return
+	}
+
+	reindexTree0(tree, i, size)
+}
+
 func reindexTree(rootID string, i, size int) {
 	root := treenode.GetBlockTree(rootID)
 	if nil == root {
@@ -1303,6 +1337,10 @@ func reindexTree(rootID string, i, size int) {
 		return
 	}
 
+	reindexTree0(tree, i, size)
+}
+
+func reindexTree0(tree *parse.Tree, i, size int) {
 	updated := tree.Root.IALAttr("updated")
 	if "" == updated {
 		updated = util.TimeFromID(tree.Root.ID)

kernel/treenode/blocktree.go

@@ -116,6 +116,25 @@ func CeilBlockCount(count int) int {
 	return 10000*100 + 1
 }
 
+func GetNotExistPaths(boxID string, paths []string) (ret []string) {
+	pathsMap := map[string]bool{}
+	for _, path := range paths {
+		pathsMap[path] = true
+	}
+
+	tmp := blockTrees
+	for _, blockTree := range tmp {
+		if blockTree.BoxID != boxID {
+			continue
+		}
+
+		if !pathsMap[blockTree.Path] {
+			ret = append(ret, blockTree.Path)
+		}
+	}
+	return
+}
+
 func GetBlockTreeRootByPath(boxID, path string) *BlockTree {
 	blockTreesLock.Lock()
 	defer blockTreesLock.Unlock()