Merge remote-tracking branch 'origin/dev' into dev

2026-01-21 15:56:10 +01:00 · 2023-01-26 18:54:11 +08:00 · 2023-01-26 18:54:11 +08:00 · bf99eb6833
commit bf99eb6833
parent d333088288 e2fc3f442c
24 changed files with 833 additions and 625 deletions
--- a/app/appearance/langs/en_US.json
+++ b/app/appearance/langs/en_US.json
@ -945,7 +945,7 @@
    "55": "Indexed references of [%d] documents",
    "56": "Reindexing, please wait until rebuilding is complete before trying to open",
    "57": "Failed to create temp key",
-    "58": "TODO",
+    "58": "Verifying index...",
    "59": "Failed to set sync ignore list",
    "60": "Failed to get the update package: %s",
    "61": "⬆️ The new version installation package is ready, do you want to install the new version now?",
@ -980,7 +980,7 @@
    "90": "[%d/%d] Created [%d] of search indexes of block-level elements [%s]",
    "91": "Reading block tree data...",
    "92": "Parsing document tree [%s]",
-    "93": "TODO",
+    "93": "[%d/%d] Cleaned up the index related to document [%s]",
    "94": "Upload failed: %s",
    "95": "Exiting...",
    "96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data",
--- a/app/appearance/langs/es_ES.json
+++ b/app/appearance/langs/es_ES.json
@ -945,7 +945,7 @@
    "55": "Referencias indexadas de [%d] documentos",
    "56": "Reindexando, espere hasta que se complete la reconstrucción antes de intentar abrir",
    "57": "Fallo en la creación de la clave temporal",
-    "58": "TODO",
+    "58": "Verificando índice...",
    "59": " Falló la configuración de sincronización de la lista de ignorados",
    "60": "Fallo al obtener el paquete de actualización: %s",
    "61": "⬆️ El paquete de instalación de la nueva versión está listo, ¿quieres instalar la nueva versión ahora?",
@ -980,7 +980,7 @@
    "90": "[%d/%d] Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
    "91": "Leyendo datos del árbol de bloques...",
    "92": "Analizando el árbol del documento [%s]",
-    "93": "TODO",
+    "93": "[%d/%d] ha limpiado el índice relacionado con el documento [%s]",
    "94": "Carga fallida: %s",
    "95": "Saliendo...",
    "96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube",
--- a/app/appearance/langs/fr_FR.json
+++ b/app/appearance/langs/fr_FR.json
@ -945,7 +945,7 @@
    "55": "Références indexées de [%d] documents",
    "56": "Réindexation, veuillez attendre que la reconstruction soit terminée avant d'essayer d'ouvrir",
    "57": "Échec de la création d'une clé temporaire",
-    "58": "TODO",
+    "58": "Vérification de l'index...",
    "59": "Échec de la définition de la liste des ignores de synchronisation",
    "60": "Échec de la récupération du paquet de mise à jour : %s",
    "61": "⬆️ Le package d'installation de la nouvelle version est prêt, voulez-vous installer la nouvelle version maintenant ?",
@ -980,7 +980,7 @@
    "90": "[%d/%d] Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
    "91": "Lecture des données de l'arborescence des blocs...",
    "92": "Analyse de l'arborescence du document [%s]",
-    "93": "TODO",
+    "93": "[%d/%d] a nettoyé l'index lié au document [%s]",
    "94": "Échec du téléchargement : %s",
    "95": "Quitter le programme...",
    "96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud",
--- a/app/appearance/langs/zh_CHT.json
+++ b/app/appearance/langs/zh_CHT.json
@ -945,7 +945,7 @@
    "55": "已完成索引 [%d] 篇文檔的引用關係",
    "56": "正在重建索引，請等重建索引完畢後再嘗試打開",
    "57": "創建臨時金鑰失敗",
-    "58": "TODO",
+    "58": "正在校驗索引...",
    "59": "設置同步忽略列表失敗",
    "60": "獲取更新包失敗：%s",
    "61": "⬆️ 新版本安裝包已經準備就緒，是否現在安裝新版本？",
@ -980,7 +980,7 @@
    "90": "[%d/%d] 已經建立 [%d] 個塊級元素的搜索索引 [%s]",
    "91": "正在讀取塊樹數據...",
    "92": "正在解析文檔樹 [%s]",
-    "93": "TODO",
+    "93": "[%d/%d] 已經清理文檔 [%s] 相關的索引",
    "94": "上傳失敗：%s",
    "95": "正在退出...",
    "96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致",
--- a/app/appearance/langs/zh_CN.json
+++ b/app/appearance/langs/zh_CN.json
@ -945,7 +945,7 @@
    "55": "已完成索引 [%d] 篇文档的引用关系",
    "56": "正在重建索引，请等重建索引完毕后再尝试打开",
    "57": "创建临时密钥失败",
-    "58": "TODO",
+    "58": "正在校验索引...",
    "59": "设置同步忽略列表失败",
    "60": "获取更新包失败：%s",
    "61": "⬆️ 新版本安装包已经准备就绪，是否现在安装新版本？",
@ -980,7 +980,7 @@
    "90": "[%d/%d] 已经建立 [%d] 个块级元素的搜索索引 [%s]",
    "91": "正在读取块树数据...",
    "92": "正在解析文档树 [%s]",
-    "93": "TODO",
+    "93": "[%d/%d] 已经清理文档 [%s] 相关的索引",
    "94": "上传失败：%s",
    "95": "正在退出...",
    "96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致",
--- a/app/stage/auth.html
+++ b/app/stage/auth.html
@ -530,8 +530,7 @@
  }

  // 用于授权页保持连接，避免非常驻内存内核自动退出 https://github.com/siyuan-note/insider/issues/1099
-  new WebSocket(
-    window.location.protocol === 'https:' ? 'wss' : 'ws' + '://' + window.location.host + '/ws?app=siyuan&id=auth')
+  new WebSocket((window.location.protocol === 'https:' ? 'wss' : 'ws') + '://' + window.location.host + '/ws?app=siyuan&id=auth')
 </script>
 </body>
 </html>
--- a/kernel/go.mod
+++ b/kernel/go.mod
@ -41,7 +41,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.22.12
 	github.com/siyuan-note/dejavu v0.0.0-20230117131301-821aa3adc1e7
 	github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
-	github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da
+	github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2
 	github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e
 	github.com/siyuan-note/httpclient v0.0.0-20230116125720-ee36ddf6f223
 	github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a
--- a/kernel/go.sum
+++ b/kernel/go.sum
@ -377,6 +377,8 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29
 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
 github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da h1:/jNhl7LC+9BhkWvNxuJDdsNfA/2wvfuj9mqWx4CbV90=
 github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
+github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2 h1:njoj0265FOHRjF/O1aWDzdwEdYdLTZwhbuPd/hoLJT0=
+github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
 github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e h1:i3RKrdrddr4AuaHJtoWYAEVNuR7Y9wIsEqPmuFFbJC4=
 github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e/go.mod h1:NmpSIVtIGy8eNWapjDIiiCw5+5r5wxC76k40oG+WRXQ=
 github.com/siyuan-note/httpclient v0.0.0-20230116125720-ee36ddf6f223 h1:hG+gucj92x4Dl4lIe2G0WkPgBdlEBnnQCmYpghHeW54=
--- a/kernel/model/box.go
+++ b/kernel/model/box.go
@ -25,6 +25,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 	"sync"
@ -516,12 +517,15 @@ func fullReindex() {
 	}
 	treenode.InitBlockTree(true)

+	sql.DisableCache()
 	openedBoxes := Conf.GetOpenedBoxes()
 	for _, openedBox := range openedBoxes {
 		index(openedBox.ID)
 	}
+	sql.EnableCache()
 	treenode.SaveBlockTree(true)
 	LoadFlashcards()
+	runtime.GC()
 }

 func ChangeBoxSort(boxIDs []string) {
--- a/kernel/model/conf.go
+++ b/kernel/model/conf.go
@ -696,20 +696,7 @@ func clearCorruptedNotebooks() {
 		boxDirPath := filepath.Join(util.DataDir, dir.Name())
 		boxConfPath := filepath.Join(boxDirPath, ".siyuan", "conf.json")
 		if !gulu.File.IsExist(boxConfPath) {
-			if IsUserGuide(dir.Name()) {
-				filelock.Remove(boxDirPath)
-				continue
-			}
-			to := filepath.Join(util.WorkspaceDir, "corrupted", time.Now().Format("2006-01-02-150405"), dir.Name())
-			if copyErr := filelock.Copy(boxDirPath, to); nil != copyErr {
-				logging.LogErrorf("copy corrupted box [%s] failed: %s", boxDirPath, copyErr)
-				continue
-			}
-			if removeErr := filelock.Remove(boxDirPath); nil != removeErr {
-				logging.LogErrorf("remove corrupted box [%s] failed: %s", boxDirPath, removeErr)
-				continue
-			}
-			logging.LogWarnf("moved corrupted box [%s] to [%s]", boxDirPath, to)
+			logging.LogWarnf("found a corrupted box [%s]", boxDirPath)
 			continue
 		}
 	}
--- a/kernel/model/import.go
+++ b/kernel/model/import.go
@ -31,7 +31,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"runtime/debug"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@ -708,8 +708,9 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
 		}
 		IncSync()
 	}
-	debug.FreeOSMemory()
+
 	IncSync()
+	runtime.GC()
 	return
 }

--- a/kernel/model/index.go
+++ b/kernel/model/index.go
@ -125,6 +125,7 @@ func index(boxID string) {
 	end := time.Now()
 	elapsed := end.Sub(start).Seconds()
 	logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize)))
+	runtime.GC()
 	return
 }

@ -186,6 +187,49 @@ func IndexRefs() {
 	util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
 }

+// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
+func AutoIndexEmbedBlock() {
+	for {
+		embedBlocks := sql.QueryEmptyContentEmbedBlocks()
+		task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks)
+		time.Sleep(10 * time.Minute)
+	}
+}
+
+func autoIndexEmbedBlock(embedBlocks []*sql.Block) {
+	for i, embedBlock := range embedBlocks {
+		stmt := strings.TrimPrefix(embedBlock.Markdown, "{{")
+		stmt = strings.TrimSuffix(stmt, "}}")
+		queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400)
+		for _, block := range queryResultBlocks {
+			embedBlock.Content += block.Content
+		}
+		if "" == embedBlock.Content {
+			embedBlock.Content = "no query result"
+		}
+		sql.UpdateBlockContent(embedBlock)
+
+		if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿
+			break
+		}
+	}
+}
+
+func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) {
+	embedBlock := sql.GetBlock(embedBlockID)
+	if nil == embedBlock {
+		return
+	}
+
+	for _, block := range queryResultBlocks {
+		embedBlock.Content += block.Block.Markdown
+	}
+	if "" == embedBlock.Content {
+		embedBlock.Content = "no query result"
+	}
+	sql.UpdateBlockContent(embedBlock)
+}
+
 func init() {
 	//eventbus.Subscribe(eventbus.EvtSQLInsertBlocks, func(context map[string]interface{}, current, total, blockCount int, hash string) {
 	//	if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
@ -209,4 +253,16 @@ func init() {
 		util.SetBootDetails(msg)
 		util.ContextPushMsg(context, msg)
 	})
+	eventbus.Subscribe(eventbus.EvtSQLDeleteBlocks, func(context map[string]interface{}, rootID string) {
+		if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
+			// Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392
+			return
+		}
+
+		current := context["current"].(int) + 1
+		total := context["total"]
+		msg := fmt.Sprintf(Conf.Language(93), current, total, rootID)
+		util.SetBootDetails(msg)
+		util.ContextPushMsg(context, msg)
+	})
 }
--- a/kernel/model/index_fix.go
+++ b/kernel/model/index_fix.go
@ -0,0 +1,261 @@
+// SiYuan - Build Your Eternal Digital Garden
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package model
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/88250/gulu"
+	"github.com/88250/lute/ast"
+	"github.com/88250/lute/html"
+	"github.com/88250/lute/parse"
+	"github.com/siyuan-note/logging"
+	"github.com/siyuan-note/siyuan/kernel/sql"
+	"github.com/siyuan-note/siyuan/kernel/task"
+	"github.com/siyuan-note/siyuan/kernel/treenode"
+	"github.com/siyuan-note/siyuan/kernel/util"
+)
+
+// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
+func AutoFixIndex() {
+	for {
+		task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
+		time.Sleep(10 * time.Minute)
+	}
+}
+
+var autoFixLock = sync.Mutex{}
+
+func autoFixIndex() {
+	defer logging.Recover()
+
+	autoFixLock.Lock()
+	defer autoFixLock.Unlock()
+
+	util.PushStatusBar(Conf.Language(58))
+
+	// 去除重复的数据库块记录
+	duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks")
+	if 1 > len(duplicatedRootIDs) {
+		duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts")
+		if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive {
+			duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive")
+		}
+	}
+
+	util.PushStatusBar(Conf.Language(58))
+	roots := sql.GetBlocks(duplicatedRootIDs)
+	rootMap := map[string]*sql.Block{}
+	for _, root := range roots {
+		rootMap[root.ID] = root
+	}
+	var deletes int
+	for _, rootID := range duplicatedRootIDs {
+		root := rootMap[rootID]
+		if nil == root {
+			continue
+		}
+
+		//logging.LogWarnf("exist more than one tree [%s], reindex it", rootID)
+		sql.RemoveTreeQueue(root.Box, rootID)
+		deletes++
+		if util.IsExiting {
+			break
+		}
+	}
+	if 0 < deletes {
+		logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", deletes)
+	}
+
+	util.PushStatusBar(Conf.Language(58))
+	sql.WaitForWritingDatabase()
+	util.PushStatusBar(Conf.Language(58))
+	// 根据文件系统补全块树
+	boxes := Conf.GetOpenedBoxes()
+	for _, box := range boxes {
+		boxPath := filepath.Join(util.DataDir, box.ID)
+		var paths []string
+		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
+			if !info.IsDir() && filepath.Ext(path) == ".sy" {
+				p := path[len(boxPath):]
+				p = filepath.ToSlash(p)
+				paths = append(paths, p)
+			}
+			return nil
+		})
+
+		size := len(paths)
+
+		redundantPaths := treenode.GetRedundantPaths(box.ID, paths)
+		for _, p := range redundantPaths {
+			treenode.RemoveBlockTreesByPath(p)
+		}
+
+		missingPaths := treenode.GetNotExistPaths(box.ID, paths)
+		for i, p := range missingPaths {
+			id := path.Base(p)
+			id = strings.TrimSuffix(id, ".sy")
+			if !ast.IsNodeIDPattern(id) {
+				continue
+			}
+
+			reindexTreeByPath(box.ID, p, i, size)
+			if util.IsExiting {
+				break
+			}
+		}
+
+		if util.IsExiting {
+			break
+		}
+	}
+
+	util.PushStatusBar(Conf.Language(58))
+	sql.WaitForWritingDatabase()
+	util.PushStatusBar(Conf.Language(58))
+	// 清理已关闭的笔记本块树
+	boxes = Conf.GetClosedBoxes()
+	for _, box := range boxes {
+		treenode.RemoveBlockTreesByBoxID(box.ID)
+	}
+
+	// 对比块树和数据库并订正数据库
+	rootUpdatedMap := treenode.GetRootUpdated()
+	dbRootUpdatedMap, err := sql.GetRootUpdated()
+	if nil == err {
+		reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap)
+	}
+
+	util.PushStatusBar(Conf.Language(58))
+	sql.WaitForWritingDatabase()
+	util.PushStatusBar(Conf.Language(185))
+}
+
+func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
+	i := -1
+	size := len(rootUpdatedMap)
+	for rootID, updated := range rootUpdatedMap {
+		i++
+
+		if util.IsExiting {
+			break
+		}
+
+		rootUpdated := dbRootUpdatedMap[rootID]
+		if "" == rootUpdated {
+			//logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
+			reindexTree(rootID, i, size)
+			continue
+		}
+
+		if "" == updated {
+			// BlockTree 迁移，v2.6.3 之前没有 updated 字段
+			reindexTree(rootID, i, size)
+			continue
+		}
+
+		btUpdated, _ := time.Parse("20060102150405", updated)
+		dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
+		if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
+			logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
+			reindexTree(rootID, i, size)
+			continue
+		}
+
+		if util.IsExiting {
+			break
+		}
+	}
+
+	var rootIDs []string
+	for rootID, _ := range dbRootUpdatedMap {
+		if _, ok := rootUpdatedMap[rootID]; !ok {
+			rootIDs = append(rootIDs, rootID)
+		}
+
+		if util.IsExiting {
+			break
+		}
+	}
+	rootIDs = gulu.Str.RemoveDuplicatedElem(rootIDs)
+	roots := map[string]*sql.Block{}
+	blocks := sql.GetBlocks(rootIDs)
+	for _, block := range blocks {
+		roots[block.RootID] = block
+	}
+	for id, root := range roots {
+		if nil == root {
+			continue
+		}
+
+		logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", id, root.Box)
+		sql.RemoveTreeQueue(root.Box, root.ID)
+		if util.IsExiting {
+			break
+		}
+	}
+}
+
+func reindexTreeByPath(box, p string, i, size int) {
+	tree, err := LoadTree(box, p)
+	if nil != err {
+		return
+	}
+
+	reindexTree0(tree, i, size)
+}
+
+func reindexTree(rootID string, i, size int) {
+	root := treenode.GetBlockTree(rootID)
+	if nil == root {
+		logging.LogWarnf("root block not found", rootID)
+		return
+	}
+
+	tree, err := LoadTree(root.BoxID, root.Path)
+	if nil != err {
+		if os.IsNotExist(err) {
+			// 文件系统上没有找到该 .sy 文件，则订正块树
+			treenode.RemoveBlockTreesByRootID(rootID)
+		}
+		return
+	}
+
+	reindexTree0(tree, i, size)
+}
+
+func reindexTree0(tree *parse.Tree, i, size int) {
+	updated := tree.Root.IALAttr("updated")
+	if "" == updated {
+		updated = util.TimeFromID(tree.Root.ID)
+		tree.Root.SetIALAttr("updated", updated)
+		indexWriteJSONQueue(tree)
+	} else {
+		treenode.IndexBlockTree(tree)
+		sql.IndexTreeQueue(tree.Box, tree.Path)
+	}
+
+	if 0 == i%64 {
+		util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath))))
+	}
+}
--- a/kernel/model/mount.go
+++ b/kernel/model/mount.go
@ -21,7 +21,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"runtime/debug"
 	"strings"
 	"time"
 	"unicode/utf8"
@ -132,7 +131,6 @@ func unmount0(boxID string) {
 	boxConf.Closed = true
 	box.SaveConf(boxConf)
 	box.Unindex()
-	debug.FreeOSMemory()
 }

 func Mount(boxID string) (alreadyMount bool, err error) {
--- a/kernel/model/transaction.go
+++ b/kernel/model/transaction.go
@ -20,9 +20,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/siyuan-note/siyuan/kernel/task"
-	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"sync"
@ -31,7 +28,6 @@ import (
 	"github.com/88250/gulu"
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/editor"
-	"github.com/88250/lute/html"
 	"github.com/88250/lute/lex"
 	"github.com/88250/lute/parse"
 	"github.com/emirpasic/gods/sets/hashset"
@ -1235,241 +1231,3 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha
 	})
 	return
 }
-
-// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
-func AutoIndexEmbedBlock() {
-	for {
-		embedBlocks := sql.QueryEmptyContentEmbedBlocks()
-		task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks)
-		time.Sleep(10 * time.Minute)
-	}
-}
-
-func autoIndexEmbedBlock(embedBlocks []*sql.Block) {
-	for i, embedBlock := range embedBlocks {
-		stmt := strings.TrimPrefix(embedBlock.Markdown, "{{")
-		stmt = strings.TrimSuffix(stmt, "}}")
-		queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400)
-		for _, block := range queryResultBlocks {
-			embedBlock.Content += block.Content
-		}
-		if "" == embedBlock.Content {
-			embedBlock.Content = "no query result"
-		}
-		sql.UpdateBlockContent(embedBlock)
-
-		if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿
-			break
-		}
-	}
-}
-
-func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) {
-	embedBlock := sql.GetBlock(embedBlockID)
-	if nil == embedBlock {
-		return
-	}
-
-	for _, block := range queryResultBlocks {
-		embedBlock.Content += block.Block.Markdown
-	}
-	if "" == embedBlock.Content {
-		embedBlock.Content = "no query result"
-	}
-	sql.UpdateBlockContent(embedBlock)
-}
-
-// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
-func AutoFixIndex() {
-	for {
-		task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
-		time.Sleep(10 * time.Minute)
-	}
-}
-
-var autoFixLock = sync.Mutex{}
-
-func autoFixIndex() {
-	defer logging.Recover()
-
-	// 根据文件系统补全块树
-	boxes := Conf.GetOpenedBoxes()
-	for _, box := range boxes {
-		boxPath := filepath.Join(util.DataDir, box.ID)
-		var paths []string
-		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
-			if !info.IsDir() && filepath.Ext(path) == ".sy" {
-				p := path[len(boxPath):]
-				p = filepath.ToSlash(p)
-				paths = append(paths, p)
-			}
-			return nil
-		})
-
-		size := len(paths)
-
-		redundantPaths := treenode.GetRedundantPaths(box.ID, paths)
-		for _, p := range redundantPaths {
-			treenode.RemoveBlockTreesByPath(p)
-		}
-
-		missingPaths := treenode.GetNotExistPaths(box.ID, paths)
-		for i, p := range missingPaths {
-			id := path.Base(p)
-			id = strings.TrimSuffix(id, ".sy")
-			if !ast.IsNodeIDPattern(id) {
-				continue
-			}
-
-			reindexTreeByPath(box.ID, p, i, size)
-			if util.IsExiting {
-				break
-			}
-		}
-
-		if util.IsExiting {
-			break
-		}
-	}
-
-	// 清理已关闭的笔记本块树
-	boxes = Conf.GetClosedBoxes()
-	for _, box := range boxes {
-		treenode.RemoveBlockTreesByBoxID(box.ID)
-	}
-
-	// 对比块树和数据库并订正数据库
-	rootUpdatedMap := treenode.GetRootUpdated()
-	dbRootUpdatedMap, err := sql.GetRootUpdated("blocks")
-	if nil == err {
-		reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks")
-	}
-	dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts")
-	if nil == err {
-		reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts")
-	}
-	if !Conf.Search.CaseSensitive {
-		dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive")
-		if nil == err {
-			reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive")
-		}
-	}
-
-	// 去除重复的数据库块记录
-	duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks")
-	if 1 > len(duplicatedRootIDs) {
-		duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts")
-		if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive {
-			duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive")
-		}
-	}
-	size := len(duplicatedRootIDs)
-	for i, rootID := range duplicatedRootIDs {
-		root := sql.GetBlock(rootID)
-		if nil == root {
-			continue
-		}
-
-		logging.LogWarnf("exist more than one tree [%s], reindex it", rootID)
-		sql.RemoveTreeQueue(root.Box, rootID)
-		reindexTree(rootID, i, size)
-
-		if util.IsExiting {
-			break
-		}
-	}
-
-	util.PushStatusBar(Conf.Language(185))
-}
-
-func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) {
-	i := -1
-	size := len(rootUpdatedMap)
-	for rootID, updated := range rootUpdatedMap {
-		i++
-
-		if util.IsExiting {
-			break
-		}
-
-		rootUpdated := dbRootUpdatedMap[rootID]
-		if "" == rootUpdated {
-			logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
-			reindexTree(rootID, i, size)
-			continue
-		}
-
-		if "" == updated {
-			// BlockTree 迁移，v2.6.3 之前没有 updated 字段
-			reindexTree(rootID, i, size)
-			continue
-		}
-
-		btUpdated, _ := time.Parse("20060102150405", updated)
-		dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
-		if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
-			logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
-			reindexTree(rootID, i, size)
-			continue
-		}
-
-		if util.IsExiting {
-			break
-		}
-	}
-
-	for rootID, _ := range dbRootUpdatedMap {
-		if _, ok := rootUpdatedMap[rootID]; !ok {
-			logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable)
-			sql.DeleteTree(blocksTable, rootID)
-		}
-
-		if util.IsExiting {
-			break
-		}
-	}
-}
-
-func reindexTreeByPath(box, p string, i, size int) {
-	tree, err := LoadTree(box, p)
-	if nil != err {
-		return
-	}
-
-	reindexTree0(tree, i, size)
-}
-
-func reindexTree(rootID string, i, size int) {
-	root := treenode.GetBlockTree(rootID)
-	if nil == root {
-		logging.LogWarnf("root block not found", rootID)
-		return
-	}
-
-	tree, err := LoadTree(root.BoxID, root.Path)
-	if nil != err {
-		if os.IsNotExist(err) {
-			// 文件系统上没有找到该 .sy 文件，则订正块树
-			treenode.RemoveBlockTreesByRootID(rootID)
-		}
-		return
-	}
-
-	reindexTree0(tree, i, size)
-}
-
-func reindexTree0(tree *parse.Tree, i, size int) {
-	updated := tree.Root.IALAttr("updated")
-	if "" == updated {
-		updated = util.TimeFromID(tree.Root.ID)
-		tree.Root.SetIALAttr("updated", updated)
-		indexWriteJSONQueue(tree)
-	} else {
-		treenode.IndexBlockTree(tree)
-		sql.IndexTreeQueue(tree.Box, tree.Path)
-	}
-
-	if 0 == i%64 {
-		util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath))))
-	}
-}
--- a/kernel/server/serve.go
+++ b/kernel/server/serve.go
@ -63,10 +63,7 @@ func Serve(fastMode bool) {
 	})
 	ginServer.Use(sessions.Sessions("siyuan", cookieStore))

-	if "dev" == util.Mode {
-		serveDebug(ginServer)
-	}
-
+	serveDebug(ginServer)
 	serveAssets(ginServer)
 	serveAppearance(ginServer)
 	serveWebSocket(ginServer)
--- a/kernel/sql/block.go
+++ b/kernel/sql/block.go
@ -92,17 +92,3 @@ func UpdateBlockContent(block *Block) {
 	tx.Commit()
 	putBlockCache(block)
 }
-
-func DeleteTree(table, rootID string) {
-	tx, err := beginTx()
-	if nil != err {
-		return
-	}
-
-	stmt := "DELETE FROM `" + table + "` WHERE root_id = ?"
-	if err = execStmtTx(tx, stmt, rootID); nil != err {
-		tx.Rollback()
-		return
-	}
-	tx.Commit()
-}
--- a/kernel/sql/block_query.go
+++ b/kernel/sql/block_query.go
@ -597,8 +597,8 @@ func GetBlock(id string) (ret *Block) {
 	return
 }

-func GetRootUpdated(blocksTable string) (ret map[string]string, err error) {
-	rows, err := query("SELECT root_id, updated FROM `" + blocksTable + "` WHERE type = 'd'")
+func GetRootUpdated() (ret map[string]string, err error) {
+	rows, err := query("SELECT root_id, updated FROM `blocks` WHERE type = 'd'")
 	if nil != err {
 		logging.LogErrorf("sql query failed: %s", err)
 		return
--- a/kernel/sql/cache.go
+++ b/kernel/sql/cache.go
@ -17,7 +17,7 @@
 package sql

 import (
-	"runtime/debug"
+	"runtime"
 	"time"

 	"github.com/88250/lute/ast"
@ -45,7 +45,7 @@ func DisableCache() {

 func ClearBlockCache() {
 	memCache.Clear()
-	debug.FreeOSMemory()
+	runtime.GC()
 }

 func putBlockCache(block *Block) {
--- a/kernel/sql/database.go
+++ b/kernel/sql/database.go
@ -23,6 +23,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
 	"time"
 	"unicode/utf8"
@ -33,6 +34,7 @@ import (
 	"github.com/88250/lute/parse"
 	"github.com/mattn/go-sqlite3"
 	_ "github.com/mattn/go-sqlite3"
+	"github.com/siyuan-note/eventbus"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
@ -65,7 +67,6 @@ func InitDatabase(forceRebuild bool) (err error) {

 	if forceRebuild {
 		ClearQueue()
-		WaitForWritingDatabase()
 	}

 	initDBConnection()
@ -80,12 +81,12 @@ func InitDatabase(forceRebuild bool) (err error) {

 	// 不存在库或者版本不一致都会走到这里

-	db.Close()
+	closeDatabase()
 	if gulu.File.IsExist(util.DBPath) {
 		if err = removeDatabaseFile(); nil != err {
 			logging.LogErrorf("remove database file [%s] failed: %s", util.DBPath, err)
 			util.PushClearProgress()
-			return
+			err = nil
 		}
 	}
 	if gulu.File.IsExist(util.BlockTreePath) {
@ -209,7 +210,7 @@ func initHistoryDBTables() {

 func initDBConnection() {
 	if nil != db {
-		db.Close()
+		closeDatabase()
 	}
 	dsn := util.DBPath + "?_journal_mode=WAL" +
 		"&_synchronous=OFF" +
@ -947,11 +948,19 @@ func deleteFileAnnotationRefsByBoxTx(tx *sql.Tx, box string) (err error) {
 	return
 }

-func deleteByRootID(tx *sql.Tx, rootID string) (err error) {
+func deleteByRootID(tx *sql.Tx, rootID string, context map[string]interface{}) (err error) {
 	stmt := "DELETE FROM blocks WHERE root_id = ?"
 	if err = execStmtTx(tx, stmt, rootID); nil != err {
 		return
 	}
+	stmt = "DELETE FROM blocks_fts WHERE root_id = ?"
+	if err = execStmtTx(tx, stmt, rootID); nil != err {
+		return
+	}
+	stmt = "DELETE FROM blocks_fts_case_insensitive WHERE root_id = ?"
+	if err = execStmtTx(tx, stmt, rootID); nil != err {
+		return
+	}
 	stmt = "DELETE FROM spans WHERE root_id = ?"
 	if err = execStmtTx(tx, stmt, rootID); nil != err {
 		return
@ -969,6 +978,7 @@ func deleteByRootID(tx *sql.Tx, rootID string) (err error) {
 		return
 	}
 	ClearBlockCache()
+	eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, rootID)
 	return
 }

@ -1023,7 +1033,7 @@ func batchUpdateHPath(tx *sql.Tx, boxID, rootID, oldHPath, newHPath string) (err
 }

 func CloseDatabase() {
-	if err := db.Close(); nil != err {
+	if err := closeDatabase(); nil != err {
 		logging.LogErrorf("close database failed: %s", err)
 		return
 	}
@ -1111,7 +1121,7 @@ func execStmtTx(tx *sql.Tx, stmt string, args ...interface{}) (err error) {
 	if _, err = tx.Exec(stmt, args...); nil != err {
 		if strings.Contains(err.Error(), "database disk image is malformed") {
 			tx.Rollback()
-			db.Close()
+			closeDatabase()
 			removeDatabaseFile()
 			logging.LogFatalf("database disk image [%s] is malformed, please restart SiYuan kernel to rebuild it", util.DBPath)
 		}
@ -1180,3 +1190,13 @@ func removeDatabaseFile() (err error) {
 	}
 	return
 }
+
+func closeDatabase() (err error) {
+	if nil == db {
+		return
+	}
+
+	err = db.Close()
+	runtime.GC() // 没有这句的话文件句柄不会释放，后面就无法删除文件
+	return
+}
--- a/kernel/sql/queue.go
+++ b/kernel/sql/queue.go
@ -147,7 +147,7 @@ func execOp(op *dbQueueOperation, tx *sql.Tx, context map[string]interface{}) (e
 	case "delete":
 		err = batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath)
 	case "delete_id":
-		err = deleteByRootID(tx, op.removeTreeID)
+		err = deleteByRootID(tx, op.removeTreeID, context)
 	case "rename":
 		err = batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath)
 		if nil != err {
@ -285,16 +285,13 @@ func RemoveTreeQueue(box, rootID string) {
 	dbQueueLock.Lock()
 	defer dbQueueLock.Unlock()

-	var tmp []*dbQueueOperation
-	// 将已有的 upsert 操作去重
-	for _, op := range operationQueue {
-		if "upsert" == op.action && op.upsertTree.ID != rootID {
-			tmp = append(tmp, op)
+	newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
+	for i, op := range operationQueue {
+		if "delete_id" == op.action && op.removeTreeIDBox == box && op.removeTreeID == rootID {
+			operationQueue[i] = newOp
+			return
 		}
 	}
-	operationQueue = tmp
-
-	newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
 	operationQueue = append(operationQueue, newOp)
 }

@ -302,15 +299,12 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) {
 	dbQueueLock.Lock()
 	defer dbQueueLock.Unlock()

-	var tmp []*dbQueueOperation
-	// 将已有的 upsert 操作去重
-	for _, op := range operationQueue {
-		if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) {
-			tmp = append(tmp, op)
+	newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
+	for i, op := range operationQueue {
+		if "delete" == op.action && (op.removeTreeBox == treeBox && op.removeTreePath == treePathPrefix) {
+			operationQueue[i] = newOp
+			return
 		}
 	}
-	operationQueue = tmp
-
-	newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
 	operationQueue = append(operationQueue, newOp)
 }
--- a/kernel/treenode/blocktree.go
+++ b/kernel/treenode/blocktree.go
@ -19,7 +19,8 @@ package treenode
 import (
 	"io"
 	"os"
-	"runtime/debug"
+	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@ -28,14 +29,19 @@ import (
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/parse"
 	"github.com/dustin/go-humanize"
+	util2 "github.com/siyuan-note/dejavu/util"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/util"
 	"github.com/vmihailenco/msgpack/v5"
 )

-var blockTrees = map[string]*BlockTree{}
-var blockTreesLock = sync.Mutex{}
-var blockTreesChanged = time.Time{}
+var blockTrees = sync.Map{}
+
+type btSlice struct {
+	data    map[string]*BlockTree
+	changed time.Time
+	m       *sync.Mutex
+}

 type BlockTree struct {
 	ID       string // 块 ID
@ -49,46 +55,458 @@ type BlockTree struct {
 }

 func GetRootUpdated() (ret map[string]string) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
 	ret = map[string]string{}
-	for _, b := range blockTrees {
-		if b.RootID == b.ID {
-			ret[b.RootID] = b.Updated
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.RootID == b.ID {
+				ret[b.RootID] = b.Updated
+			}
 		}
-	}
+		slice.m.Unlock()
+		return true
+	})
 	return
 }

-func GetBlockTreeByPath(path string) *BlockTree {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	for _, b := range blockTrees {
-		if b.Path == path {
-			return b
+func GetBlockTreeByPath(path string) (ret *BlockTree) {
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.Path == path {
+				ret = b
+				break
+			}
 		}
-	}
-	return nil
+		slice.m.Unlock()
+		return nil == ret
+	})
+	return
 }

 func CountTrees() (ret int) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
 	roots := map[string]bool{}
-	for _, b := range blockTrees {
-		roots[b.RootID] = true
-	}
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			roots[b.RootID] = true
+		}
+		slice.m.Unlock()
+		return true
+	})
 	ret = len(roots)
 	return
 }

 func CountBlocks() (ret int) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-	return len(blockTrees)
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		ret += len(slice.data)
+		slice.m.Unlock()
+		return true
+	})
+	return
+}
+
+func GetRedundantPaths(boxID string, paths []string) (ret []string) {
+	pathsMap := map[string]bool{}
+	for _, path := range paths {
+		pathsMap[path] = true
+	}
+
+	btPathsMap := map[string]bool{}
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.BoxID == boxID {
+				btPathsMap[b.Path] = true
+			}
+		}
+		slice.m.Unlock()
+		return true
+	})
+
+	for p, _ := range btPathsMap {
+		if !pathsMap[p] {
+			ret = append(ret, p)
+		}
+	}
+	ret = gulu.Str.RemoveDuplicatedElem(ret)
+	return
+}
+
+func GetNotExistPaths(boxID string, paths []string) (ret []string) {
+	pathsMap := map[string]bool{}
+	for _, path := range paths {
+		pathsMap[path] = true
+	}
+
+	btPathsMap := map[string]bool{}
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.BoxID == boxID {
+				btPathsMap[b.Path] = true
+			}
+		}
+		slice.m.Unlock()
+		return true
+	})
+
+	for p, _ := range pathsMap {
+		if !btPathsMap[p] {
+			ret = append(ret, p)
+		}
+	}
+	ret = gulu.Str.RemoveDuplicatedElem(ret)
+	return
+}
+
+func GetBlockTreeRootByPath(boxID, path string) (ret *BlockTree) {
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.BoxID == boxID && b.Path == path && b.RootID == b.ID {
+				ret = b
+				break
+			}
+		}
+		slice.m.Unlock()
+		return nil == ret
+	})
+	return
+}
+
+func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) {
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID {
+				ret = b
+				break
+			}
+		}
+		slice.m.Unlock()
+		return nil == ret
+	})
+	return
+}
+
+func GetBlockTree(id string) (ret *BlockTree) {
+	if "" == id {
+		return
+	}
+
+	hash := btHash(id)
+	val, ok := blockTrees.Load(hash)
+	if !ok {
+		return
+	}
+	slice := val.(*btSlice)
+	slice.m.Lock()
+	ret = slice.data[id]
+	slice.m.Unlock()
+	return
+}
+
+func SetBlockTreePath(tree *parse.Tree) {
+	hash := btHash(tree.ID)
+	val, ok := blockTrees.Load(hash)
+	if !ok {
+		val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}}
+		blockTrees.Store(hash, val)
+	}
+
+	slice := val.(*btSlice)
+	slice.m.Lock()
+	slice.data[tree.ID] = &BlockTree{
+		ID:      tree.ID,
+		RootID:  tree.Root.ID,
+		BoxID:   tree.Box,
+		Path:    tree.Path,
+		HPath:   tree.HPath,
+		Updated: tree.Root.IALAttr("updated"),
+		Type:    TypeAbbr(ast.NodeDocument.String()),
+	}
+	slice.m.Unlock()
+	slice.changed = time.Now()
+}
+
+func RemoveBlockTreesByRootID(rootID string) {
+	var ids []string
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.RootID == rootID {
+				ids = append(ids, b.RootID)
+			}
+		}
+		slice.m.Unlock()
+		return true
+	})
+
+	ids = gulu.Str.RemoveDuplicatedElem(ids)
+	for _, id := range ids {
+		val, ok := blockTrees.Load(btHash(id))
+		if !ok {
+			continue
+		}
+		slice := val.(*btSlice)
+		slice.m.Lock()
+		delete(slice.data, id)
+		slice.m.Unlock()
+		slice.changed = time.Now()
+	}
+}
+
+func RemoveBlockTreesByPath(path string) {
+	var ids []string
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.Path == path {
+				ids = append(ids, b.RootID)
+			}
+		}
+		slice.m.Unlock()
+		return true
+	})
+
+	ids = gulu.Str.RemoveDuplicatedElem(ids)
+	for _, id := range ids {
+		val, ok := blockTrees.Load(btHash(id))
+		if !ok {
+			continue
+		}
+		slice := val.(*btSlice)
+		slice.m.Lock()
+		delete(slice.data, id)
+		slice.m.Unlock()
+		slice.changed = time.Now()
+	}
+}
+
+func RemoveBlockTreesByPathPrefix(pathPrefix string) {
+	var ids []string
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if strings.HasPrefix(b.Path, pathPrefix) {
+				ids = append(ids, b.RootID)
+			}
+		}
+		slice.m.Unlock()
+		return true
+	})
+
+	ids = gulu.Str.RemoveDuplicatedElem(ids)
+	for _, id := range ids {
+		val, ok := blockTrees.Load(btHash(id))
+		if !ok {
+			continue
+		}
+		slice := val.(*btSlice)
+		slice.m.Lock()
+		delete(slice.data, id)
+		slice.m.Unlock()
+		slice.changed = time.Now()
+	}
+}
+
+func RemoveBlockTreesByBoxID(boxID string) (ids []string) {
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		slice.m.Lock()
+		for _, b := range slice.data {
+			if b.BoxID == boxID {
+				ids = append(ids, b.RootID)
+			}
+		}
+		slice.m.Unlock()
+		return true
+	})
+
+	ids = gulu.Str.RemoveDuplicatedElem(ids)
+	for _, id := range ids {
+		val, ok := blockTrees.Load(btHash(id))
+		if !ok {
+			continue
+		}
+		slice := val.(*btSlice)
+		slice.m.Lock()
+		delete(slice.data, id)
+		slice.m.Unlock()
+		slice.changed = time.Now()
+	}
+	return
+}
+
+func RemoveBlockTree(id string) {
+	val, ok := blockTrees.Load(btHash(id))
+	if !ok {
+		return
+	}
+	slice := val.(*btSlice)
+	slice.m.Lock()
+	delete(slice.data, id)
+	slice.m.Unlock()
+	slice.changed = time.Now()
+}
+
+func IndexBlockTree(tree *parse.Tree) {
+	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
+		if !entering || !n.IsBlock() {
+			return ast.WalkContinue
+		}
+		var parentID string
+		if nil != n.Parent {
+			parentID = n.Parent.ID
+		}
+		if "" == n.ID {
+			return ast.WalkContinue
+		}
+
+		hash := btHash(n.ID)
+		val, ok := blockTrees.Load(hash)
+		if !ok {
+			val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}}
+			blockTrees.Store(hash, val)
+		}
+		slice := val.(*btSlice)
+		slice.m.Lock()
+		if bt := slice.data[n.ID]; nil != bt {
+			if bt.Updated != n.IALAttr("updated") {
+				slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())}
+				slice.changed = time.Now()
+			}
+		} else {
+			slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())}
+			slice.changed = time.Now()
+		}
+		slice.m.Unlock()
+		return ast.WalkContinue
+	})
+}
+
+func AutoFlushBlockTree() {
+	for {
+		SaveBlockTree(false)
+		time.Sleep(1 * time.Second)
+	}
+}
+
+func InitBlockTree(force bool) {
+	start := time.Now()
+
+	if force {
+		err := os.RemoveAll(util.BlockTreePath)
+		if nil != err {
+			logging.LogErrorf("remove blocktree file failed: %s", err)
+		}
+		return
+	}
+
+	entries, err := os.ReadDir(util.BlockTreePath)
+	if nil != err {
+		logging.LogErrorf("read block tree dir failed: %s", err)
+		os.Exit(util.ExitCodeBlockTreeErr)
+		return
+	}
+
+	size := uint64(0)
+	for _, entry := range entries {
+		if !strings.HasSuffix(entry.Name(), ".msgpack") {
+			continue
+		}
+
+		p := filepath.Join(util.BlockTreePath, entry.Name())
+		var fh *os.File
+		fh, err = os.OpenFile(p, os.O_RDWR, 0644)
+		if nil != err {
+			logging.LogErrorf("open block tree file failed: %s", err)
+			os.Exit(util.ExitCodeBlockTreeErr)
+			return
+		}
+
+		var data []byte
+		data, err = io.ReadAll(fh)
+		fh.Close()
+		if nil != err {
+			logging.LogErrorf("read block tree failed: %s", err)
+			os.Exit(util.ExitCodeBlockTreeErr)
+			return
+		}
+
+		sliceData := map[string]*BlockTree{}
+		if err = msgpack.Unmarshal(data, &sliceData); nil != err {
+			logging.LogErrorf("unmarshal block tree failed: %s", err)
+			if err = os.RemoveAll(util.BlockTreePath); nil != err {
+				logging.LogErrorf("removed corrupted block tree failed: %s", err)
+			}
+			os.Exit(util.ExitCodeBlockTreeErr)
+			return
+		}
+
+		name := entry.Name()[0:strings.Index(entry.Name(), ".")]
+		blockTrees.Store(name, &btSlice{data: sliceData, changed: time.Time{}, m: &sync.Mutex{}})
+		size += uint64(len(data))
+	}
+
+	runtime.GC()
+
+	if elapsed := time.Since(start).Seconds(); 2 < elapsed {
+		logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes((size)), util.BlockTreePath, elapsed)
+	}
+	return
+}
+
+func SaveBlockTree(force bool) {
+	start := time.Now()
+	os.MkdirAll(util.BlockTreePath, 0755)
+
+	size := uint64(0)
+	blockTrees.Range(func(key, value interface{}) bool {
+		slice := value.(*btSlice)
+		if !force && (slice.changed.IsZero() || slice.changed.After(start.Add(-7*time.Second))) {
+			return true
+		}
+
+		slice.m.Lock()
+		data, err := msgpack.Marshal(slice.data)
+		if nil != err {
+			logging.LogErrorf("marshal block tree failed: %s", err)
+			os.Exit(util.ExitCodeBlockTreeErr)
+			return false
+		}
+		slice.m.Unlock()
+
+		p := filepath.Join(util.BlockTreePath, key.(string)) + ".msgpack"
+		if err = gulu.File.WriteFileSafer(p, data, 0644); nil != err {
+			logging.LogErrorf("write block tree failed: %s", err)
+			os.Exit(util.ExitCodeBlockTreeErr)
+			return false
+		}
+		slice.changed = time.Time{}
+		size += uint64(len(data))
+		return true
+	})
+
+	runtime.GC()
+
+	if elapsed := time.Since(start).Seconds(); 2 < elapsed {
+		logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(size), util.BlockTreePath, elapsed)
+	}
 }

 func CeilTreeCount(count int) int {
@ -117,279 +535,6 @@ func CeilBlockCount(count int) int {
 	return 10000*100 + 1
 }

-func GetRedundantPaths(boxID string, paths []string) (ret []string) {
-	pathsMap := map[string]bool{}
-	for _, path := range paths {
-		pathsMap[path] = true
-	}
-
-	tmp := blockTrees
-	btPathsMap := map[string]bool{}
-	for _, blockTree := range tmp {
-		if blockTree.BoxID != boxID {
-			continue
-		}
-
-		btPathsMap[blockTree.Path] = true
-	}
-
-	for p, _ := range btPathsMap {
-		if !pathsMap[p] {
-			ret = append(ret, p)
-		}
-	}
-	ret = gulu.Str.RemoveDuplicatedElem(ret)
-	return
-}
-
-func GetNotExistPaths(boxID string, paths []string) (ret []string) {
-	pathsMap := map[string]bool{}
-	for _, path := range paths {
-		pathsMap[path] = true
-	}
-
-	tmp := blockTrees
-	btPathsMap := map[string]bool{}
-	for _, blockTree := range tmp {
-		if blockTree.BoxID != boxID {
-			continue
-		}
-
-		btPathsMap[blockTree.Path] = true
-	}
-
-	for p, _ := range pathsMap {
-		if !btPathsMap[p] {
-			ret = append(ret, p)
-		}
-	}
-	ret = gulu.Str.RemoveDuplicatedElem(ret)
-	return
-}
-
-func GetBlockTreeRootByPath(boxID, path string) *BlockTree {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	for _, blockTree := range blockTrees {
-		if blockTree.BoxID == boxID && blockTree.Path == path && blockTree.RootID == blockTree.ID {
-			return blockTree
-		}
-	}
-	return nil
-}
-
-func GetBlockTreeRootByHPath(boxID, hPath string) *BlockTree {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	for _, blockTree := range blockTrees {
-		if blockTree.BoxID == boxID && blockTree.HPath == hPath && blockTree.RootID == blockTree.ID {
-			return blockTree
-		}
-	}
-	return nil
-}
-
-func GetBlockTree(id string) *BlockTree {
-	if "" == id {
-		return nil
-	}
-
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-	return blockTrees[id]
-}
-
-func SetBlockTreePath(tree *parse.Tree) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	for _, b := range blockTrees {
-		if b.RootID == tree.ID {
-			b.BoxID, b.Path, b.HPath, b.Updated, b.Type = tree.Box, tree.Path, tree.HPath, tree.Root.IALAttr("updated"), TypeAbbr(ast.NodeDocument.String())
-		}
-	}
-	blockTreesChanged = time.Now()
-}
-
-func RemoveBlockTreesByRootID(rootID string) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	var ids []string
-	for _, b := range blockTrees {
-		if b.RootID == rootID {
-			ids = append(ids, b.RootID)
-		}
-	}
-	ids = gulu.Str.RemoveDuplicatedElem(ids)
-	for _, id := range ids {
-		delete(blockTrees, id)
-	}
-	blockTreesChanged = time.Now()
-}
-
-func RemoveBlockTreesByPath(path string) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	var ids []string
-	for _, b := range blockTrees {
-		if b.Path == path {
-			ids = append(ids, b.ID)
-		}
-	}
-	ids = gulu.Str.RemoveDuplicatedElem(ids)
-	for _, id := range ids {
-		delete(blockTrees, id)
-	}
-	blockTreesChanged = time.Now()
-}
-
-func RemoveBlockTreesByPathPrefix(pathPrefix string) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	var ids []string
-	for _, b := range blockTrees {
-		if strings.HasPrefix(b.Path, pathPrefix) {
-			ids = append(ids, b.ID)
-		}
-	}
-	ids = gulu.Str.RemoveDuplicatedElem(ids)
-	for _, id := range ids {
-		delete(blockTrees, id)
-	}
-	blockTreesChanged = time.Now()
-}
-
-func RemoveBlockTreesByBoxID(boxID string) (ids []string) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	for _, b := range blockTrees {
-		if b.BoxID == boxID {
-			ids = append(ids, b.ID)
-		}
-	}
-	ids = gulu.Str.RemoveDuplicatedElem(ids)
-	for _, id := range ids {
-		delete(blockTrees, id)
-	}
-	blockTreesChanged = time.Now()
-	return
-}
-
-func RemoveBlockTree(id string) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	delete(blockTrees, id)
-	blockTreesChanged = time.Now()
-}
-
-func IndexBlockTree(tree *parse.Tree) {
-	blockTreesLock.Lock()
-	defer blockTreesLock.Unlock()
-
-	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
-		if !entering || !n.IsBlock() {
-			return ast.WalkContinue
-		}
-		var parentID string
-		if nil != n.Parent {
-			parentID = n.Parent.ID
-		}
-		if "" == n.ID {
-			return ast.WalkContinue
-		}
-		blockTrees[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: tree.Root.IALAttr("updated"), Type: TypeAbbr(n.Type.String())}
-		return ast.WalkContinue
-	})
-	blockTreesChanged = time.Now()
-}
-
-func AutoFlushBlockTree() {
-	for {
-		SaveBlockTree(false)
-		time.Sleep(1 * time.Second)
-	}
-}
-
-func InitBlockTree(force bool) {
-	start := time.Now()
-
-	if force {
-		err := os.RemoveAll(util.BlockTreePath)
-		if nil != err {
-			logging.LogErrorf("remove blocktree file failed: %s", err)
-		}
-		return
-	}
-
-	var err error
-	fh, err := os.OpenFile(util.BlockTreePath, os.O_RDWR, 0644)
-	if nil != err {
-		logging.LogErrorf("open block tree file failed: %s", err)
-		os.Exit(util.ExitCodeBlockTreeErr)
-		return
-	}
-	defer fh.Close()
-
-	data, err := io.ReadAll(fh)
-	if nil != err {
-		logging.LogErrorf("read block tree failed: %s", err)
-		os.Exit(util.ExitCodeBlockTreeErr)
-		return
-	}
-	blockTreesLock.Lock()
-	if err = msgpack.Unmarshal(data, &blockTrees); nil != err {
-		logging.LogErrorf("unmarshal block tree failed: %s", err)
-		if err = os.RemoveAll(util.BlockTreePath); nil != err {
-			logging.LogErrorf("removed corrupted block tree failed: %s", err)
-		}
-		os.Exit(util.ExitCodeBlockTreeErr)
-		return
-	}
-	blockTreesLock.Unlock()
-	debug.FreeOSMemory()
-
-	if elapsed := time.Since(start).Seconds(); 2 < elapsed {
-		logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed)
-	}
-	return
-}
-
-func SaveBlockTree(force bool) {
-	if !force && blockTreesChanged.IsZero() {
-		return
-	}
-
-	start := time.Now()
-	if blockTreesChanged.After(start.Add(-7 * time.Second)) {
-		return
-	}
-
-	blockTreesLock.Lock()
-	data, err := msgpack.Marshal(blockTrees)
-	if nil != err {
-		logging.LogErrorf("marshal block tree failed: %s", err)
-		os.Exit(util.ExitCodeBlockTreeErr)
-		return
-	}
-	blockTreesLock.Unlock()
-
-	if err = gulu.File.WriteFileSafer(util.BlockTreePath, data, 0644); nil != err {
-		logging.LogErrorf("write block tree failed: %s", err)
-		os.Exit(util.ExitCodeBlockTreeErr)
-		return
-	}
-	debug.FreeOSMemory()
-
-	if elapsed := time.Since(start).Seconds(); 2 < elapsed {
-		logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed)
-	}
-
-	blockTreesChanged = time.Time{}
+func btHash(id string) string {
+	return util2.Hash([]byte(id))[0:2]
 }
--- a/kernel/util/working.go
+++ b/kernel/util/working.go
@ -253,7 +253,7 @@ func initWorkspaceDir(workspaceArg string) {
 	os.Setenv("TMP", osTmpDir)
 	DBPath = filepath.Join(TempDir, DBName)
 	HistoryDBPath = filepath.Join(TempDir, "history.db")
-	BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack")
+	BlockTreePath = filepath.Join(TempDir, "blocktree")
 	SnippetsPath = filepath.Join(DataDir, "snippets")
 }

--- a/kernel/util/working_mobile.go
+++ b/kernel/util/working_mobile.go
@ -155,7 +155,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) {
 	os.Setenv("TMP", osTmpDir)
 	DBPath = filepath.Join(TempDir, DBName)
 	HistoryDBPath = filepath.Join(TempDir, "history.db")
-	BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack")
+	BlockTreePath = filepath.Join(TempDir, "blocktree")
 	SnippetsPath = filepath.Join(DataDir, "snippets")

 	AppearancePath = filepath.Join(ConfDir, "appearance")