diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index afbd8e53d..e243ed7c2 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -19,6 +19,7 @@ package treenode import ( "io" "os" + "path/filepath" "runtime" "strings" "sync" @@ -28,14 +29,19 @@ import ( "github.com/88250/lute/ast" "github.com/88250/lute/parse" "github.com/dustin/go-humanize" + util2 "github.com/siyuan-note/dejavu/util" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" "github.com/vmihailenco/msgpack/v5" ) -var blockTrees = map[string]*BlockTree{} -var blockTreesLock = sync.Mutex{} -var blockTreesChanged = time.Time{} +var blockTrees = sync.Map{} + +type btSlice struct { + data map[string]*BlockTree + changed time.Time + m *sync.Mutex +} type BlockTree struct { ID string // 块 ID @@ -49,46 +55,463 @@ type BlockTree struct { } func GetRootUpdated() (ret map[string]string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - ret = map[string]string{} - for _, b := range blockTrees { - if b.RootID == b.ID { - ret[b.RootID] = b.Updated + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.RootID == b.ID { + ret[b.RootID] = b.Updated + } } - } + slice.m.Unlock() + return true + }) return } -func GetBlockTreeByPath(path string) *BlockTree { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, b := range blockTrees { - if b.Path == path { - return b +func GetBlockTreeByPath(path string) (ret *BlockTree) { + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.Path == path { + ret = b + break + } } - } - return nil + slice.m.Unlock() + return nil == ret + }) + return } func CountTrees() (ret int) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - roots := map[string]bool{} - for _, b := range blockTrees { - roots[b.RootID] = true - } + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + roots[b.RootID] = true + } + slice.m.Unlock() + return true + }) ret = len(roots) return } func CountBlocks() (ret int) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - return len(blockTrees) + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + ret += len(slice.data) + slice.m.Unlock() + return true + }) + return +} + +func GetRedundantPaths(boxID string, paths []string) (ret []string) { + pathsMap := map[string]bool{} + for _, path := range paths { + pathsMap[path] = true + } + + btPathsMap := map[string]bool{} + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID { + btPathsMap[b.Path] = true + } + } + slice.m.Unlock() + return true + }) + + for p, _ := range btPathsMap { + if !pathsMap[p] { + ret = append(ret, p) + } + } + ret = gulu.Str.RemoveDuplicatedElem(ret) + return +} + +func GetNotExistPaths(boxID string, paths []string) (ret []string) { + pathsMap := map[string]bool{} + for _, path := range paths { + pathsMap[path] = true + } + + btPathsMap := map[string]bool{} + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID { + btPathsMap[b.Path] = true + } + } + slice.m.Unlock() + return true + }) + + for p, _ := range pathsMap { + if !btPathsMap[p] { + ret = append(ret, p) + } + } + ret = gulu.Str.RemoveDuplicatedElem(ret) + return +} + +func GetBlockTreeRootByPath(boxID, path string) (ret *BlockTree) { + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID && b.Path == path && b.RootID == b.ID { + ret = b + break + } + } + slice.m.Unlock() + return nil == ret + }) + return +} + +func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) { + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID { + ret = b + break + } + } + slice.m.Unlock() + return nil == ret + }) + return +} + +func GetBlockTree(id string) (ret *BlockTree) { + if "" == id { + return + } + + hash := btHash(id) + val, ok := blockTrees.Load(hash) + if !ok { + return + } + slice := val.(*btSlice) + slice.m.Lock() + ret = slice.data[id] + slice.m.Unlock() + return +} + +func SetBlockTreePath(tree *parse.Tree) { + hash := btHash(tree.ID) + val, ok := blockTrees.Load(hash) + if !ok { + val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}} + blockTrees.Store(hash, val) + } + + slice := val.(*btSlice) + slice.m.Lock() + slice.data[tree.ID] = &BlockTree{ + ID: tree.ID, + RootID: tree.Root.ID, + BoxID: tree.Box, + Path: tree.Path, + HPath: tree.HPath, + Updated: tree.Root.IALAttr("updated"), + Type: TypeAbbr(ast.NodeDocument.String()), + } + slice.m.Unlock() + slice.changed = time.Now() +} + +func RemoveBlockTreesByRootID(rootID string) { + var ids []string + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.RootID == rootID { + ids = append(ids, b.RootID) + } + } + slice.m.Unlock() + return true + }) + + ids = gulu.Str.RemoveDuplicatedElem(ids) + for _, id := range ids { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + continue + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() + } +} + +func RemoveBlockTreesByPath(path string) { + var ids []string + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.Path == path { + ids = append(ids, b.RootID) + } + } + slice.m.Unlock() + return true + }) + + ids = gulu.Str.RemoveDuplicatedElem(ids) + for _, id := range ids { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + continue + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() + } +} + +func RemoveBlockTreesByPathPrefix(pathPrefix string) { + var ids []string + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if strings.HasPrefix(b.Path, pathPrefix) { + ids = append(ids, b.RootID) + } + } + slice.m.Unlock() + return true + }) + + ids = gulu.Str.RemoveDuplicatedElem(ids) + for _, id := range ids { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + continue + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() + } +} + +func RemoveBlockTreesByBoxID(boxID string) (ids []string) { + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID { + ids = append(ids, b.RootID) + } + } + slice.m.Unlock() + return true + }) + + ids = gulu.Str.RemoveDuplicatedElem(ids) + for _, id := range ids { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + continue + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() + } + return +} + +func RemoveBlockTree(id string) { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + return + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() +} + +func IndexBlockTree(tree *parse.Tree) { + ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering || !n.IsBlock() { + return ast.WalkContinue + } + var parentID string + if nil != n.Parent { + parentID = n.Parent.ID + } + if "" == n.ID { + return ast.WalkContinue + } + + hash := btHash(n.ID) + val, ok := blockTrees.Load(hash) + if !ok { + val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}} + blockTrees.Store(hash, val) + } + slice := val.(*btSlice) + slice.m.Lock() + if bt := slice.data[n.ID]; nil != bt { + if bt.Updated != n.IALAttr("updated") { + slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())} + slice.changed = time.Now() + } + } else { + slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())} + slice.changed = time.Now() + } + slice.m.Unlock() + return ast.WalkContinue + }) +} + +func AutoFlushBlockTree() { + for { + SaveBlockTree(false) + time.Sleep(1 * time.Second) + } +} + +func InitBlockTree(force bool) { + start := time.Now() + + if force { + err := os.RemoveAll(util.BlockTreePath) + if nil != err { + logging.LogErrorf("remove blocktree file failed: %s", err) + } + return + } + + entries, err := os.ReadDir(util.BlockTreePath) + if nil != err { + logging.LogErrorf("read block tree dir failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return + } + + size := uint64(0) + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".msgpack") { + continue + } + + p := filepath.Join(util.BlockTreePath, entry.Name()) + var fh *os.File + fh, err = os.OpenFile(p, os.O_RDWR, 0644) + if nil != err { + logging.LogErrorf("open block tree file failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return + } + + var data []byte + data, err = io.ReadAll(fh) + fh.Close() + if nil != err { + logging.LogErrorf("read block tree failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return + } + + sliceData := map[string]*BlockTree{} + if err = msgpack.Unmarshal(data, &sliceData); nil != err { + logging.LogErrorf("unmarshal block tree failed: %s", err) + if err = os.RemoveAll(util.BlockTreePath); nil != err { + logging.LogErrorf("removed corrupted block tree failed: %s", err) + } + os.Exit(util.ExitCodeBlockTreeErr) + return + } + + name := entry.Name()[0:strings.Index(entry.Name(), ".")] + blockTrees.Store(name, &btSlice{data: sliceData, changed: time.Time{}, m: &sync.Mutex{}}) + size += uint64(len(data)) + } + + runtime.GC() + + if elapsed := time.Since(start).Seconds(); 2 < elapsed { + logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes((size)), util.BlockTreePath, elapsed) + } + return +} + +func SaveBlockTree(force bool) { + if force { + + return + } + + start := time.Now() + os.MkdirAll(util.BlockTreePath, 0755) + + size := uint64(0) + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + if !force && (slice.changed.IsZero() || slice.changed.After(start.Add(-7*time.Second))) { + return true + } + + slice.m.Lock() + data, err := msgpack.Marshal(slice.data) + if nil != err { + logging.LogErrorf("marshal block tree failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return false + } + slice.m.Unlock() + + p := filepath.Join(util.BlockTreePath, key.(string)) + ".msgpack" + if err = gulu.File.WriteFileSafer(p, data, 0644); nil != err { + logging.LogErrorf("write block tree failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return false + } + slice.changed = time.Time{} + size += uint64(len(data)) + return true + }) + + runtime.GC() + + if elapsed := time.Since(start).Seconds(); 2 < elapsed { + logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(size), util.BlockTreePath, elapsed) + } } func CeilTreeCount(count int) int { @@ -117,279 +540,6 @@ func CeilBlockCount(count int) int { return 10000*100 + 1 } -func GetRedundantPaths(boxID string, paths []string) (ret []string) { - pathsMap := map[string]bool{} - for _, path := range paths { - pathsMap[path] = true - } - - tmp := blockTrees - btPathsMap := map[string]bool{} - for _, blockTree := range tmp { - if blockTree.BoxID != boxID { - continue - } - - btPathsMap[blockTree.Path] = true - } - - for p, _ := range btPathsMap { - if !pathsMap[p] { - ret = append(ret, p) - } - } - ret = gulu.Str.RemoveDuplicatedElem(ret) - return -} - -func GetNotExistPaths(boxID string, paths []string) (ret []string) { - pathsMap := map[string]bool{} - for _, path := range paths { - pathsMap[path] = true - } - - tmp := blockTrees - btPathsMap := map[string]bool{} - for _, blockTree := range tmp { - if blockTree.BoxID != boxID { - continue - } - - btPathsMap[blockTree.Path] = true - } - - for p, _ := range pathsMap { - if !btPathsMap[p] { - ret = append(ret, p) - } - } - ret = gulu.Str.RemoveDuplicatedElem(ret) - return -} - -func GetBlockTreeRootByPath(boxID, path string) *BlockTree { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, blockTree := range blockTrees { - if blockTree.BoxID == boxID && blockTree.Path == path && blockTree.RootID == blockTree.ID { - return blockTree - } - } - return nil -} - -func GetBlockTreeRootByHPath(boxID, hPath string) *BlockTree { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, blockTree := range blockTrees { - if blockTree.BoxID == boxID && blockTree.HPath == hPath && blockTree.RootID == blockTree.ID { - return blockTree - } - } - return nil -} - -func GetBlockTree(id string) *BlockTree { - if "" == id { - return nil - } - - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - return blockTrees[id] -} - -func SetBlockTreePath(tree *parse.Tree) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, b := range blockTrees { - if b.RootID == tree.ID { - b.BoxID, b.Path, b.HPath, b.Updated, b.Type = tree.Box, tree.Path, tree.HPath, tree.Root.IALAttr("updated"), TypeAbbr(ast.NodeDocument.String()) - } - } - blockTreesChanged = time.Now() -} - -func RemoveBlockTreesByRootID(rootID string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - var ids []string - for _, b := range blockTrees { - if b.RootID == rootID { - ids = append(ids, b.RootID) - } - } - ids = gulu.Str.RemoveDuplicatedElem(ids) - for _, id := range ids { - delete(blockTrees, id) - } - blockTreesChanged = time.Now() -} - -func RemoveBlockTreesByPath(path string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - var ids []string - for _, b := range blockTrees { - if b.Path == path { - ids = append(ids, b.ID) - } - } - ids = gulu.Str.RemoveDuplicatedElem(ids) - for _, id := range ids { - delete(blockTrees, id) - } - blockTreesChanged = time.Now() -} - -func RemoveBlockTreesByPathPrefix(pathPrefix string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - var ids []string - for _, b := range blockTrees { - if strings.HasPrefix(b.Path, pathPrefix) { - ids = append(ids, b.ID) - } - } - ids = gulu.Str.RemoveDuplicatedElem(ids) - for _, id := range ids { - delete(blockTrees, id) - } - blockTreesChanged = time.Now() -} - -func RemoveBlockTreesByBoxID(boxID string) (ids []string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, b := range blockTrees { - if b.BoxID == boxID { - ids = append(ids, b.ID) - } - } - ids = gulu.Str.RemoveDuplicatedElem(ids) - for _, id := range ids { - delete(blockTrees, id) - } - blockTreesChanged = time.Now() - return -} - -func RemoveBlockTree(id string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - delete(blockTrees, id) - blockTreesChanged = time.Now() -} - -func IndexBlockTree(tree *parse.Tree) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering || !n.IsBlock() { - return ast.WalkContinue - } - var parentID string - if nil != n.Parent { - parentID = n.Parent.ID - } - if "" == n.ID { - return ast.WalkContinue - } - blockTrees[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: tree.Root.IALAttr("updated"), Type: TypeAbbr(n.Type.String())} - return ast.WalkContinue - }) - blockTreesChanged = time.Now() -} - -func AutoFlushBlockTree() { - for { - SaveBlockTree(false) - time.Sleep(1 * time.Second) - } -} - -func InitBlockTree(force bool) { - start := time.Now() - - if force { - err := os.RemoveAll(util.BlockTreePath) - if nil != err { - logging.LogErrorf("remove blocktree file failed: %s", err) - } - return - } - - var err error - fh, err := os.OpenFile(util.BlockTreePath, os.O_RDWR, 0644) - if nil != err { - logging.LogErrorf("open block tree file failed: %s", err) - os.Exit(util.ExitCodeBlockTreeErr) - return - } - defer fh.Close() - - data, err := io.ReadAll(fh) - if nil != err { - logging.LogErrorf("read block tree failed: %s", err) - os.Exit(util.ExitCodeBlockTreeErr) - return - } - blockTreesLock.Lock() - if err = msgpack.Unmarshal(data, &blockTrees); nil != err { - logging.LogErrorf("unmarshal block tree failed: %s", err) - if err = os.RemoveAll(util.BlockTreePath); nil != err { - logging.LogErrorf("removed corrupted block tree failed: %s", err) - } - os.Exit(util.ExitCodeBlockTreeErr) - return - } - blockTreesLock.Unlock() - runtime.GC() - - if elapsed := time.Since(start).Seconds(); 2 < elapsed { - logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed) - } - return -} - -func SaveBlockTree(force bool) { - if !force && blockTreesChanged.IsZero() { - return - } - - start := time.Now() - if blockTreesChanged.After(start.Add(-7 * time.Second)) { - return - } - - blockTreesLock.Lock() - data, err := msgpack.Marshal(blockTrees) - if nil != err { - logging.LogErrorf("marshal block tree failed: %s", err) - os.Exit(util.ExitCodeBlockTreeErr) - return - } - blockTreesLock.Unlock() - - if err = gulu.File.WriteFileSafer(util.BlockTreePath, data, 0644); nil != err { - logging.LogErrorf("write block tree failed: %s", err) - os.Exit(util.ExitCodeBlockTreeErr) - return - } - runtime.GC() - - if elapsed := time.Since(start).Seconds(); 2 < elapsed { - logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed) - } - - blockTreesChanged = time.Time{} +func btHash(id string) string { + return util2.Hash([]byte(id))[0:2] } diff --git a/kernel/util/working.go b/kernel/util/working.go index 0aaa8d80a..5b11f8b9f 100644 --- a/kernel/util/working.go +++ b/kernel/util/working.go @@ -253,7 +253,7 @@ func initWorkspaceDir(workspaceArg string) { os.Setenv("TMP", osTmpDir) DBPath = filepath.Join(TempDir, DBName) HistoryDBPath = filepath.Join(TempDir, "history.db") - BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack") + BlockTreePath = filepath.Join(TempDir, "blocktree") SnippetsPath = filepath.Join(DataDir, "snippets") } diff --git a/kernel/util/working_mobile.go b/kernel/util/working_mobile.go index e998996b8..abbcadaff 100644 --- a/kernel/util/working_mobile.go +++ b/kernel/util/working_mobile.go @@ -155,7 +155,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) { os.Setenv("TMP", osTmpDir) DBPath = filepath.Join(TempDir, DBName) HistoryDBPath = filepath.Join(TempDir, "history.db") - BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack") + BlockTreePath = filepath.Join(TempDir, "blocktree") SnippetsPath = filepath.Join(DataDir, "snippets") AppearancePath = filepath.Join(ConfDir, "appearance")