From bea767d4a67f394a119ecedabee33753200ef8a7 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 10:59:40 +0800 Subject: [PATCH 01/12] =?UTF-8?q?:bug:=20v2.7.1-dev3=E8=A7=A3=E9=94=81?= =?UTF-8?q?=E5=A4=B1=E6=95=88=20Fix=20https://github.com/siyuan-note/siyua?= =?UTF-8?q?n/issues/7166=20Fix=20https://github.com/siyuan-note/siyuan/pul?= =?UTF-8?q?l/7161?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/stage/auth.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/stage/auth.html b/app/stage/auth.html index 112505b45..2adb68651 100644 --- a/app/stage/auth.html +++ b/app/stage/auth.html @@ -530,8 +530,7 @@ } // 用于授权页保持连接，避免非常驻内存内核自动退出 https://github.com/siyuan-note/insider/issues/1099 - new WebSocket( - window.location.protocol === 'https:' ? 'wss' : 'ws' + '://' + window.location.host + '/ws?app=siyuan&id=auth') + new WebSocket((window.location.protocol === 'https:' ? 'wss' : 'ws') + '://' + window.location.host + '/ws?app=siyuan&id=auth') From 10407d54d6d3031edfbfcb56593ea8c8166dde0f Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 11:56:06 +0800 Subject: [PATCH 02/12] =?UTF-8?q?:bug:=202.7.1dev3=20=E9=87=8D=E5=BB=BA?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E6=97=B6=E6=8A=A5=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E8=A2=AB=E9=94=81=E5=AE=9A=20https://github.com/siyuan-note/si?= =?UTF-8?q?yuan/issues/7167?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/server/serve.go | 5 +---- kernel/sql/database.go | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/kernel/server/serve.go b/kernel/server/serve.go index 8b67e1e63..e078ce051 100644 --- a/kernel/server/serve.go +++ b/kernel/server/serve.go @@ -63,10 +63,7 @@ func Serve(fastMode bool) { }) ginServer.Use(sessions.Sessions("siyuan", cookieStore)) - if "dev" == util.Mode { - serveDebug(ginServer) - } - + serveDebug(ginServer) serveAssets(ginServer) serveAppearance(ginServer) serveWebSocket(ginServer) diff --git a/kernel/sql/database.go b/kernel/sql/database.go index bef9bd5ee..144b9fb14 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" "time" "unicode/utf8" @@ -65,7 +66,6 @@ func InitDatabase(forceRebuild bool) (err error) { if forceRebuild { ClearQueue() - WaitForWritingDatabase() } initDBConnection() @@ -80,12 +80,12 @@ func InitDatabase(forceRebuild bool) (err error) { // 不存在库或者版本不一致都会走到这里 - db.Close() + closeDatabase() if gulu.File.IsExist(util.DBPath) { if err = removeDatabaseFile(); nil != err { logging.LogErrorf("remove database file [%s] failed: %s", util.DBPath, err) util.PushClearProgress() - return + err = nil } } if gulu.File.IsExist(util.BlockTreePath) { @@ -209,7 +209,7 @@ func initHistoryDBTables() { func initDBConnection() { if nil != db { - db.Close() + closeDatabase() } dsn := util.DBPath + "?_journal_mode=WAL" + "&_synchronous=OFF" + @@ -1023,7 +1023,7 @@ func batchUpdateHPath(tx *sql.Tx, boxID, rootID, oldHPath, newHPath string) (err } func CloseDatabase() { - if err := db.Close(); nil != err { + if err := closeDatabase(); nil != err { logging.LogErrorf("close database failed: %s", err) return } @@ -1111,7 +1111,7 @@ func execStmtTx(tx *sql.Tx, stmt string, args ...interface{}) (err error) { if _, err = tx.Exec(stmt, args...); nil != err { if strings.Contains(err.Error(), "database disk image is malformed") { tx.Rollback() - db.Close() + closeDatabase() removeDatabaseFile() logging.LogFatalf("database disk image [%s] is malformed, please restart SiYuan kernel to rebuild it", util.DBPath) } @@ -1180,3 +1180,13 @@ func removeDatabaseFile() (err error) { } return } + +func closeDatabase() (err error) { + if nil == db { + return + } + + err = db.Close() + runtime.GC() // 没有这句的话文件句柄不会释放，后面就无法删除文件 + return +} From db777afc25cfa27b34aeea4ba6a7a79ef1f35c59 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 12:34:57 +0800 Subject: [PATCH 03/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/box.go | 4 + kernel/model/index.go | 44 +++++++ kernel/model/index_fix.go | 231 ++++++++++++++++++++++++++++++++++ kernel/model/transaction.go | 242 ------------------------------------ 4 files changed, 279 insertions(+), 242 deletions(-) create mode 100644 kernel/model/index_fix.go diff --git a/kernel/model/box.go b/kernel/model/box.go index 176658520..fe46e0748 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -25,6 +25,7 @@ import ( "os" "path" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -516,12 +517,15 @@ func fullReindex() { } treenode.InitBlockTree(true) + sql.DisableCache() openedBoxes := Conf.GetOpenedBoxes() for _, openedBox := range openedBoxes { index(openedBox.ID) } + sql.EnableCache() treenode.SaveBlockTree(true) LoadFlashcards() + runtime.GC() } func ChangeBoxSort(boxIDs []string) { diff --git a/kernel/model/index.go b/kernel/model/index.go index 424eedfee..d252b9a42 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -125,6 +125,7 @@ func index(boxID string) { end := time.Now() elapsed := end.Sub(start).Seconds() logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize))) + runtime.GC() return } @@ -186,6 +187,49 @@ func IndexRefs() { util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) } +// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112 +func AutoIndexEmbedBlock() { + for { + embedBlocks := sql.QueryEmptyContentEmbedBlocks() + task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks) + time.Sleep(10 * time.Minute) + } +} + +func autoIndexEmbedBlock(embedBlocks []*sql.Block) { + for i, embedBlock := range embedBlocks { + stmt := strings.TrimPrefix(embedBlock.Markdown, "{{") + stmt = strings.TrimSuffix(stmt, "}}") + queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400) + for _, block := range queryResultBlocks { + embedBlock.Content += block.Content + } + if "" == embedBlock.Content { + embedBlock.Content = "no query result" + } + sql.UpdateBlockContent(embedBlock) + + if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿 + break + } + } +} + +func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) { + embedBlock := sql.GetBlock(embedBlockID) + if nil == embedBlock { + return + } + + for _, block := range queryResultBlocks { + embedBlock.Content += block.Block.Markdown + } + if "" == embedBlock.Content { + embedBlock.Content = "no query result" + } + sql.UpdateBlockContent(embedBlock) +} + func init() { //eventbus.Subscribe(eventbus.EvtSQLInsertBlocks, func(context map[string]interface{}, current, total, blockCount int, hash string) { // if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container { diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go new file mode 100644 index 000000000..ea8850cb7 --- /dev/null +++ b/kernel/model/index_fix.go @@ -0,0 +1,231 @@ +// SiYuan - Build Your Eternal Digital Garden +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package model + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/88250/lute/ast" + "github.com/88250/lute/html" + "github.com/88250/lute/parse" + "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" + "github.com/siyuan-note/siyuan/kernel/treenode" + "github.com/siyuan-note/siyuan/kernel/util" +) + +// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016 +func AutoFixIndex() { + for { + task.AppendTask(task.DatabaseIndexFix, autoFixIndex) + time.Sleep(10 * time.Minute) + } +} + +var autoFixLock = sync.Mutex{} + +func autoFixIndex() { + defer logging.Recover() + + // 根据文件系统补全块树 + boxes := Conf.GetOpenedBoxes() + for _, box := range boxes { + boxPath := filepath.Join(util.DataDir, box.ID) + var paths []string + filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() && filepath.Ext(path) == ".sy" { + p := path[len(boxPath):] + p = filepath.ToSlash(p) + paths = append(paths, p) + } + return nil + }) + + size := len(paths) + + redundantPaths := treenode.GetRedundantPaths(box.ID, paths) + for _, p := range redundantPaths { + treenode.RemoveBlockTreesByPath(p) + } + + missingPaths := treenode.GetNotExistPaths(box.ID, paths) + for i, p := range missingPaths { + id := path.Base(p) + id = strings.TrimSuffix(id, ".sy") + if !ast.IsNodeIDPattern(id) { + continue + } + + reindexTreeByPath(box.ID, p, i, size) + if util.IsExiting { + break + } + } + + if util.IsExiting { + break + } + } + + // 清理已关闭的笔记本块树 + boxes = Conf.GetClosedBoxes() + for _, box := range boxes { + treenode.RemoveBlockTreesByBoxID(box.ID) + } + + // 对比块树和数据库并订正数据库 + rootUpdatedMap := treenode.GetRootUpdated() + dbRootUpdatedMap, err := sql.GetRootUpdated("blocks") + if nil == err { + reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks") + } + dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts") + if nil == err { + reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts") + } + if !Conf.Search.CaseSensitive { + dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive") + if nil == err { + reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive") + } + } + + // 去除重复的数据库块记录 + duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks") + if 1 > len(duplicatedRootIDs) { + duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts") + if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive { + duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive") + } + } + size := len(duplicatedRootIDs) + for i, rootID := range duplicatedRootIDs { + root := sql.GetBlock(rootID) + if nil == root { + continue + } + + logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) + sql.RemoveTreeQueue(root.Box, rootID) + reindexTree(rootID, i, size) + + if util.IsExiting { + break + } + } + + util.PushStatusBar(Conf.Language(185)) +} + +func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) { + i := -1 + size := len(rootUpdatedMap) + for rootID, updated := range rootUpdatedMap { + i++ + + if util.IsExiting { + break + } + + rootUpdated := dbRootUpdatedMap[rootID] + if "" == rootUpdated { + //logging.LogWarnf("not found tree [%s] in database, reindex it", rootID) + reindexTree(rootID, i, size) + continue + } + + if "" == updated { + // BlockTree 迁移，v2.6.3 之前没有 updated 字段 + reindexTree(rootID, i, size) + continue + } + + btUpdated, _ := time.Parse("20060102150405", updated) + dbUpdated, _ := time.Parse("20060102150405", rootUpdated) + if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) { + logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID) + reindexTree(rootID, i, size) + continue + } + + if util.IsExiting { + break + } + } + + for rootID, _ := range dbRootUpdatedMap { + if _, ok := rootUpdatedMap[rootID]; !ok { + logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable) + sql.DeleteTree(blocksTable, rootID) + } + + if util.IsExiting { + break + } + } +} + +func reindexTreeByPath(box, p string, i, size int) { + tree, err := LoadTree(box, p) + if nil != err { + return + } + + reindexTree0(tree, i, size) +} + +func reindexTree(rootID string, i, size int) { + root := treenode.GetBlockTree(rootID) + if nil == root { + logging.LogWarnf("root block not found", rootID) + return + } + + tree, err := LoadTree(root.BoxID, root.Path) + if nil != err { + if os.IsNotExist(err) { + // 文件系统上没有找到该 .sy 文件，则订正块树 + treenode.RemoveBlockTreesByRootID(rootID) + } + return + } + + reindexTree0(tree, i, size) +} + +func reindexTree0(tree *parse.Tree, i, size int) { + updated := tree.Root.IALAttr("updated") + if "" == updated { + updated = util.TimeFromID(tree.Root.ID) + tree.Root.SetIALAttr("updated", updated) + indexWriteJSONQueue(tree) + } else { + treenode.IndexBlockTree(tree) + sql.IndexTreeQueue(tree.Box, tree.Path) + } + + if 0 == i%64 { + util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath)))) + } +} diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 9eaceedea..a075d115c 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -20,9 +20,6 @@ import ( "bytes" "errors" "fmt" - "github.com/siyuan-note/siyuan/kernel/task" - "os" - "path" "path/filepath" "strings" "sync" @@ -31,7 +28,6 @@ import ( "github.com/88250/gulu" "github.com/88250/lute/ast" "github.com/88250/lute/editor" - "github.com/88250/lute/html" "github.com/88250/lute/lex" "github.com/88250/lute/parse" "github.com/emirpasic/gods/sets/hashset" @@ -1235,241 +1231,3 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha }) return } - -// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112 -func AutoIndexEmbedBlock() { - for { - embedBlocks := sql.QueryEmptyContentEmbedBlocks() - task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks) - time.Sleep(10 * time.Minute) - } -} - -func autoIndexEmbedBlock(embedBlocks []*sql.Block) { - for i, embedBlock := range embedBlocks { - stmt := strings.TrimPrefix(embedBlock.Markdown, "{{") - stmt = strings.TrimSuffix(stmt, "}}") - queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400) - for _, block := range queryResultBlocks { - embedBlock.Content += block.Content - } - if "" == embedBlock.Content { - embedBlock.Content = "no query result" - } - sql.UpdateBlockContent(embedBlock) - - if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿 - break - } - } -} - -func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) { - embedBlock := sql.GetBlock(embedBlockID) - if nil == embedBlock { - return - } - - for _, block := range queryResultBlocks { - embedBlock.Content += block.Block.Markdown - } - if "" == embedBlock.Content { - embedBlock.Content = "no query result" - } - sql.UpdateBlockContent(embedBlock) -} - -// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016 -func AutoFixIndex() { - for { - task.AppendTask(task.DatabaseIndexFix, autoFixIndex) - time.Sleep(10 * time.Minute) - } -} - -var autoFixLock = sync.Mutex{} - -func autoFixIndex() { - defer logging.Recover() - - // 根据文件系统补全块树 - boxes := Conf.GetOpenedBoxes() - for _, box := range boxes { - boxPath := filepath.Join(util.DataDir, box.ID) - var paths []string - filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() && filepath.Ext(path) == ".sy" { - p := path[len(boxPath):] - p = filepath.ToSlash(p) - paths = append(paths, p) - } - return nil - }) - - size := len(paths) - - redundantPaths := treenode.GetRedundantPaths(box.ID, paths) - for _, p := range redundantPaths { - treenode.RemoveBlockTreesByPath(p) - } - - missingPaths := treenode.GetNotExistPaths(box.ID, paths) - for i, p := range missingPaths { - id := path.Base(p) - id = strings.TrimSuffix(id, ".sy") - if !ast.IsNodeIDPattern(id) { - continue - } - - reindexTreeByPath(box.ID, p, i, size) - if util.IsExiting { - break - } - } - - if util.IsExiting { - break - } - } - - // 清理已关闭的笔记本块树 - boxes = Conf.GetClosedBoxes() - for _, box := range boxes { - treenode.RemoveBlockTreesByBoxID(box.ID) - } - - // 对比块树和数据库并订正数据库 - rootUpdatedMap := treenode.GetRootUpdated() - dbRootUpdatedMap, err := sql.GetRootUpdated("blocks") - if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks") - } - dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts") - if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts") - } - if !Conf.Search.CaseSensitive { - dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive") - if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive") - } - } - - // 去除重复的数据库块记录 - duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks") - if 1 > len(duplicatedRootIDs) { - duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts") - if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive { - duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive") - } - } - size := len(duplicatedRootIDs) - for i, rootID := range duplicatedRootIDs { - root := sql.GetBlock(rootID) - if nil == root { - continue - } - - logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) - sql.RemoveTreeQueue(root.Box, rootID) - reindexTree(rootID, i, size) - - if util.IsExiting { - break - } - } - - util.PushStatusBar(Conf.Language(185)) -} - -func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) { - i := -1 - size := len(rootUpdatedMap) - for rootID, updated := range rootUpdatedMap { - i++ - - if util.IsExiting { - break - } - - rootUpdated := dbRootUpdatedMap[rootID] - if "" == rootUpdated { - logging.LogWarnf("not found tree [%s] in database, reindex it", rootID) - reindexTree(rootID, i, size) - continue - } - - if "" == updated { - // BlockTree 迁移，v2.6.3 之前没有 updated 字段 - reindexTree(rootID, i, size) - continue - } - - btUpdated, _ := time.Parse("20060102150405", updated) - dbUpdated, _ := time.Parse("20060102150405", rootUpdated) - if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) { - logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID) - reindexTree(rootID, i, size) - continue - } - - if util.IsExiting { - break - } - } - - for rootID, _ := range dbRootUpdatedMap { - if _, ok := rootUpdatedMap[rootID]; !ok { - logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable) - sql.DeleteTree(blocksTable, rootID) - } - - if util.IsExiting { - break - } - } -} - -func reindexTreeByPath(box, p string, i, size int) { - tree, err := LoadTree(box, p) - if nil != err { - return - } - - reindexTree0(tree, i, size) -} - -func reindexTree(rootID string, i, size int) { - root := treenode.GetBlockTree(rootID) - if nil == root { - logging.LogWarnf("root block not found", rootID) - return - } - - tree, err := LoadTree(root.BoxID, root.Path) - if nil != err { - if os.IsNotExist(err) { - // 文件系统上没有找到该 .sy 文件，则订正块树 - treenode.RemoveBlockTreesByRootID(rootID) - } - return - } - - reindexTree0(tree, i, size) -} - -func reindexTree0(tree *parse.Tree, i, size int) { - updated := tree.Root.IALAttr("updated") - if "" == updated { - updated = util.TimeFromID(tree.Root.ID) - tree.Root.SetIALAttr("updated", updated) - indexWriteJSONQueue(tree) - } else { - treenode.IndexBlockTree(tree) - sql.IndexTreeQueue(tree.Box, tree.Path) - } - - if 0 == i%64 { - util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath)))) - } -} From 5aff312a7bdc9cc7cc397ea62e6d12d868b788e5 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 13:47:12 +0800 Subject: [PATCH 04/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/index_fix.go | 100 +++++++++++++++++++++++--------------- kernel/sql/block.go | 14 ------ kernel/sql/block_query.go | 4 +- 3 files changed, 64 insertions(+), 54 deletions(-) diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go index ea8850cb7..97bfae4af 100644 --- a/kernel/model/index_fix.go +++ b/kernel/model/index_fix.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/88250/gulu" "github.com/88250/lute/ast" "github.com/88250/lute/html" "github.com/88250/lute/parse" @@ -48,6 +49,42 @@ var autoFixLock = sync.Mutex{} func autoFixIndex() { defer logging.Recover() + autoFixLock.Lock() + defer autoFixLock.Unlock() + + // 去除重复的数据库块记录 + duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks") + if 1 > len(duplicatedRootIDs) { + duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts") + if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive { + duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive") + } + } + + roots := sql.GetBlocks(duplicatedRootIDs) + rootMap := map[string]*sql.Block{} + for _, root := range roots { + rootMap[root.ID] = root + } + for _, rootID := range duplicatedRootIDs { + root := rootMap[rootID] + if nil == root { + continue + } + + //logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) + sql.RemoveTreeQueue(root.Box, rootID) + + if util.IsExiting { + break + } + } + if 0 < len(duplicatedRootIDs) { + logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", len(duplicatedRootIDs)) + } + + sql.WaitForWritingDatabase() + // 根据文件系统补全块树 boxes := Conf.GetOpenedBoxes() for _, box := range boxes { @@ -88,6 +125,8 @@ func autoFixIndex() { } } + sql.WaitForWritingDatabase() + // 清理已关闭的笔记本块树 boxes = Conf.GetClosedBoxes() for _, box := range boxes { @@ -96,49 +135,17 @@ func autoFixIndex() { // 对比块树和数据库并订正数据库 rootUpdatedMap := treenode.GetRootUpdated() - dbRootUpdatedMap, err := sql.GetRootUpdated("blocks") + dbRootUpdatedMap, err := sql.GetRootUpdated() if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks") - } - dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts") - if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts") - } - if !Conf.Search.CaseSensitive { - dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive") - if nil == err { - reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive") - } + reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap) } - // 去除重复的数据库块记录 - duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks") - if 1 > len(duplicatedRootIDs) { - duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts") - if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive { - duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive") - } - } - size := len(duplicatedRootIDs) - for i, rootID := range duplicatedRootIDs { - root := sql.GetBlock(rootID) - if nil == root { - continue - } - - logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) - sql.RemoveTreeQueue(root.Box, rootID) - reindexTree(rootID, i, size) - - if util.IsExiting { - break - } - } + sql.WaitForWritingDatabase() util.PushStatusBar(Conf.Language(185)) } -func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) { +func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) { i := -1 size := len(rootUpdatedMap) for rootID, updated := range rootUpdatedMap { @@ -174,16 +181,33 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, bl } } + var rootIDs []string for rootID, _ := range dbRootUpdatedMap { if _, ok := rootUpdatedMap[rootID]; !ok { - logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable) - sql.DeleteTree(blocksTable, rootID) + rootIDs = append(rootIDs, rootID) } if util.IsExiting { break } } + rootIDs = gulu.Str.RemoveDuplicatedElem(rootIDs) + roots := map[string]*sql.Block{} + blocks := sql.GetBlocks(rootIDs) + for _, block := range blocks { + roots[block.RootID] = block + } + for id, root := range roots { + if nil == root { + continue + } + + logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", id, root.Box) + sql.RemoveTreeQueue(root.Box, root.ID) + if util.IsExiting { + break + } + } } func reindexTreeByPath(box, p string, i, size int) { diff --git a/kernel/sql/block.go b/kernel/sql/block.go index 41135ba3e..e12b1debd 100644 --- a/kernel/sql/block.go +++ b/kernel/sql/block.go @@ -92,17 +92,3 @@ func UpdateBlockContent(block *Block) { tx.Commit() putBlockCache(block) } - -func DeleteTree(table, rootID string) { - tx, err := beginTx() - if nil != err { - return - } - - stmt := "DELETE FROM `" + table + "` WHERE root_id = ?" - if err = execStmtTx(tx, stmt, rootID); nil != err { - tx.Rollback() - return - } - tx.Commit() -} diff --git a/kernel/sql/block_query.go b/kernel/sql/block_query.go index 8dcbd5584..c8dd627f2 100644 --- a/kernel/sql/block_query.go +++ b/kernel/sql/block_query.go @@ -597,8 +597,8 @@ func GetBlock(id string) (ret *Block) { return } -func GetRootUpdated(blocksTable string) (ret map[string]string, err error) { - rows, err := query("SELECT root_id, updated FROM `" + blocksTable + "` WHERE type = 'd'") +func GetRootUpdated() (ret map[string]string, err error) { + rows, err := query("SELECT root_id, updated FROM `blocks` WHERE type = 'd'") if nil != err { logging.LogErrorf("sql query failed: %s", err) return From e214543c86b827c22fd4c74b249819fc79e40939 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 13:52:22 +0800 Subject: [PATCH 05/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/import.go | 5 +++-- kernel/model/mount.go | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/kernel/model/import.go b/kernel/model/import.go index 941cc1ab1..3dd5dc16d 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -31,7 +31,7 @@ import ( "os" "path" "path/filepath" - "runtime/debug" + "runtime" "sort" "strconv" "strings" @@ -708,8 +708,9 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { } IncSync() } - debug.FreeOSMemory() + IncSync() + runtime.GC() return } diff --git a/kernel/model/mount.go b/kernel/model/mount.go index 1b7c6b74a..06b419bfb 100644 --- a/kernel/model/mount.go +++ b/kernel/model/mount.go @@ -21,7 +21,6 @@ import ( "fmt" "os" "path/filepath" - "runtime/debug" "strings" "time" "unicode/utf8" @@ -132,7 +131,6 @@ func unmount0(boxID string) { boxConf.Closed = true box.SaveConf(boxConf) box.Unindex() - debug.FreeOSMemory() } func Mount(boxID string) (alreadyMount bool, err error) { From 81df07598d6c293498e9b9656a64e59925449e99 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 13:58:58 +0800 Subject: [PATCH 06/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/treenode/blocktree.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index 0a9702089..afbd8e53d 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -19,7 +19,7 @@ package treenode import ( "io" "os" - "runtime/debug" + "runtime" "strings" "sync" "time" @@ -353,7 +353,7 @@ func InitBlockTree(force bool) { return } blockTreesLock.Unlock() - debug.FreeOSMemory() + runtime.GC() if elapsed := time.Since(start).Seconds(); 2 < elapsed { logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed) @@ -385,7 +385,7 @@ func SaveBlockTree(force bool) { os.Exit(util.ExitCodeBlockTreeErr) return } - debug.FreeOSMemory() + runtime.GC() if elapsed := time.Since(start).Seconds(); 2 < elapsed { logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed) From fef57e49caf3a7b339534dab2614197f7aeecc04 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 17:12:38 +0800 Subject: [PATCH 07/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=9D=97?= =?UTF-8?q?=E6=A0=91=E6=95=B0=E6=8D=AE=E5=AD=98=E5=8F=96=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7168?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/treenode/blocktree.go | 756 ++++++++++++++++++++-------------- kernel/util/working.go | 2 +- kernel/util/working_mobile.go | 2 +- 3 files changed, 455 insertions(+), 305 deletions(-) diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index afbd8e53d..e243ed7c2 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -19,6 +19,7 @@ package treenode import ( "io" "os" + "path/filepath" "runtime" "strings" "sync" @@ -28,14 +29,19 @@ import ( "github.com/88250/lute/ast" "github.com/88250/lute/parse" "github.com/dustin/go-humanize" + util2 "github.com/siyuan-note/dejavu/util" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" "github.com/vmihailenco/msgpack/v5" ) -var blockTrees = map[string]*BlockTree{} -var blockTreesLock = sync.Mutex{} -var blockTreesChanged = time.Time{} +var blockTrees = sync.Map{} + +type btSlice struct { + data map[string]*BlockTree + changed time.Time + m *sync.Mutex +} type BlockTree struct { ID string // 块 ID @@ -49,46 +55,463 @@ type BlockTree struct { } func GetRootUpdated() (ret map[string]string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - ret = map[string]string{} - for _, b := range blockTrees { - if b.RootID == b.ID { - ret[b.RootID] = b.Updated + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.RootID == b.ID { + ret[b.RootID] = b.Updated + } } - } + slice.m.Unlock() + return true + }) return } -func GetBlockTreeByPath(path string) *BlockTree { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, b := range blockTrees { - if b.Path == path { - return b +func GetBlockTreeByPath(path string) (ret *BlockTree) { + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.Path == path { + ret = b + break + } } - } - return nil + slice.m.Unlock() + return nil == ret + }) + return } func CountTrees() (ret int) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - roots := map[string]bool{} - for _, b := range blockTrees { - roots[b.RootID] = true - } + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + roots[b.RootID] = true + } + slice.m.Unlock() + return true + }) ret = len(roots) return } func CountBlocks() (ret int) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - return len(blockTrees) + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + ret += len(slice.data) + slice.m.Unlock() + return true + }) + return +} + +func GetRedundantPaths(boxID string, paths []string) (ret []string) { + pathsMap := map[string]bool{} + for _, path := range paths { + pathsMap[path] = true + } + + btPathsMap := map[string]bool{} + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID { + btPathsMap[b.Path] = true + } + } + slice.m.Unlock() + return true + }) + + for p, _ := range btPathsMap { + if !pathsMap[p] { + ret = append(ret, p) + } + } + ret = gulu.Str.RemoveDuplicatedElem(ret) + return +} + +func GetNotExistPaths(boxID string, paths []string) (ret []string) { + pathsMap := map[string]bool{} + for _, path := range paths { + pathsMap[path] = true + } + + btPathsMap := map[string]bool{} + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID { + btPathsMap[b.Path] = true + } + } + slice.m.Unlock() + return true + }) + + for p, _ := range pathsMap { + if !btPathsMap[p] { + ret = append(ret, p) + } + } + ret = gulu.Str.RemoveDuplicatedElem(ret) + return +} + +func GetBlockTreeRootByPath(boxID, path string) (ret *BlockTree) { + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID && b.Path == path && b.RootID == b.ID { + ret = b + break + } + } + slice.m.Unlock() + return nil == ret + }) + return +} + +func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) { + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID { + ret = b + break + } + } + slice.m.Unlock() + return nil == ret + }) + return +} + +func GetBlockTree(id string) (ret *BlockTree) { + if "" == id { + return + } + + hash := btHash(id) + val, ok := blockTrees.Load(hash) + if !ok { + return + } + slice := val.(*btSlice) + slice.m.Lock() + ret = slice.data[id] + slice.m.Unlock() + return +} + +func SetBlockTreePath(tree *parse.Tree) { + hash := btHash(tree.ID) + val, ok := blockTrees.Load(hash) + if !ok { + val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}} + blockTrees.Store(hash, val) + } + + slice := val.(*btSlice) + slice.m.Lock() + slice.data[tree.ID] = &BlockTree{ + ID: tree.ID, + RootID: tree.Root.ID, + BoxID: tree.Box, + Path: tree.Path, + HPath: tree.HPath, + Updated: tree.Root.IALAttr("updated"), + Type: TypeAbbr(ast.NodeDocument.String()), + } + slice.m.Unlock() + slice.changed = time.Now() +} + +func RemoveBlockTreesByRootID(rootID string) { + var ids []string + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.RootID == rootID { + ids = append(ids, b.RootID) + } + } + slice.m.Unlock() + return true + }) + + ids = gulu.Str.RemoveDuplicatedElem(ids) + for _, id := range ids { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + continue + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() + } +} + +func RemoveBlockTreesByPath(path string) { + var ids []string + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.Path == path { + ids = append(ids, b.RootID) + } + } + slice.m.Unlock() + return true + }) + + ids = gulu.Str.RemoveDuplicatedElem(ids) + for _, id := range ids { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + continue + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() + } +} + +func RemoveBlockTreesByPathPrefix(pathPrefix string) { + var ids []string + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if strings.HasPrefix(b.Path, pathPrefix) { + ids = append(ids, b.RootID) + } + } + slice.m.Unlock() + return true + }) + + ids = gulu.Str.RemoveDuplicatedElem(ids) + for _, id := range ids { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + continue + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() + } +} + +func RemoveBlockTreesByBoxID(boxID string) (ids []string) { + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + slice.m.Lock() + for _, b := range slice.data { + if b.BoxID == boxID { + ids = append(ids, b.RootID) + } + } + slice.m.Unlock() + return true + }) + + ids = gulu.Str.RemoveDuplicatedElem(ids) + for _, id := range ids { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + continue + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() + } + return +} + +func RemoveBlockTree(id string) { + val, ok := blockTrees.Load(btHash(id)) + if !ok { + return + } + slice := val.(*btSlice) + slice.m.Lock() + delete(slice.data, id) + slice.m.Unlock() + slice.changed = time.Now() +} + +func IndexBlockTree(tree *parse.Tree) { + ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering || !n.IsBlock() { + return ast.WalkContinue + } + var parentID string + if nil != n.Parent { + parentID = n.Parent.ID + } + if "" == n.ID { + return ast.WalkContinue + } + + hash := btHash(n.ID) + val, ok := blockTrees.Load(hash) + if !ok { + val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}} + blockTrees.Store(hash, val) + } + slice := val.(*btSlice) + slice.m.Lock() + if bt := slice.data[n.ID]; nil != bt { + if bt.Updated != n.IALAttr("updated") { + slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())} + slice.changed = time.Now() + } + } else { + slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())} + slice.changed = time.Now() + } + slice.m.Unlock() + return ast.WalkContinue + }) +} + +func AutoFlushBlockTree() { + for { + SaveBlockTree(false) + time.Sleep(1 * time.Second) + } +} + +func InitBlockTree(force bool) { + start := time.Now() + + if force { + err := os.RemoveAll(util.BlockTreePath) + if nil != err { + logging.LogErrorf("remove blocktree file failed: %s", err) + } + return + } + + entries, err := os.ReadDir(util.BlockTreePath) + if nil != err { + logging.LogErrorf("read block tree dir failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return + } + + size := uint64(0) + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".msgpack") { + continue + } + + p := filepath.Join(util.BlockTreePath, entry.Name()) + var fh *os.File + fh, err = os.OpenFile(p, os.O_RDWR, 0644) + if nil != err { + logging.LogErrorf("open block tree file failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return + } + + var data []byte + data, err = io.ReadAll(fh) + fh.Close() + if nil != err { + logging.LogErrorf("read block tree failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return + } + + sliceData := map[string]*BlockTree{} + if err = msgpack.Unmarshal(data, &sliceData); nil != err { + logging.LogErrorf("unmarshal block tree failed: %s", err) + if err = os.RemoveAll(util.BlockTreePath); nil != err { + logging.LogErrorf("removed corrupted block tree failed: %s", err) + } + os.Exit(util.ExitCodeBlockTreeErr) + return + } + + name := entry.Name()[0:strings.Index(entry.Name(), ".")] + blockTrees.Store(name, &btSlice{data: sliceData, changed: time.Time{}, m: &sync.Mutex{}}) + size += uint64(len(data)) + } + + runtime.GC() + + if elapsed := time.Since(start).Seconds(); 2 < elapsed { + logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes((size)), util.BlockTreePath, elapsed) + } + return +} + +func SaveBlockTree(force bool) { + if force { + + return + } + + start := time.Now() + os.MkdirAll(util.BlockTreePath, 0755) + + size := uint64(0) + blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + if !force && (slice.changed.IsZero() || slice.changed.After(start.Add(-7*time.Second))) { + return true + } + + slice.m.Lock() + data, err := msgpack.Marshal(slice.data) + if nil != err { + logging.LogErrorf("marshal block tree failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return false + } + slice.m.Unlock() + + p := filepath.Join(util.BlockTreePath, key.(string)) + ".msgpack" + if err = gulu.File.WriteFileSafer(p, data, 0644); nil != err { + logging.LogErrorf("write block tree failed: %s", err) + os.Exit(util.ExitCodeBlockTreeErr) + return false + } + slice.changed = time.Time{} + size += uint64(len(data)) + return true + }) + + runtime.GC() + + if elapsed := time.Since(start).Seconds(); 2 < elapsed { + logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(size), util.BlockTreePath, elapsed) + } } func CeilTreeCount(count int) int { @@ -117,279 +540,6 @@ func CeilBlockCount(count int) int { return 10000*100 + 1 } -func GetRedundantPaths(boxID string, paths []string) (ret []string) { - pathsMap := map[string]bool{} - for _, path := range paths { - pathsMap[path] = true - } - - tmp := blockTrees - btPathsMap := map[string]bool{} - for _, blockTree := range tmp { - if blockTree.BoxID != boxID { - continue - } - - btPathsMap[blockTree.Path] = true - } - - for p, _ := range btPathsMap { - if !pathsMap[p] { - ret = append(ret, p) - } - } - ret = gulu.Str.RemoveDuplicatedElem(ret) - return -} - -func GetNotExistPaths(boxID string, paths []string) (ret []string) { - pathsMap := map[string]bool{} - for _, path := range paths { - pathsMap[path] = true - } - - tmp := blockTrees - btPathsMap := map[string]bool{} - for _, blockTree := range tmp { - if blockTree.BoxID != boxID { - continue - } - - btPathsMap[blockTree.Path] = true - } - - for p, _ := range pathsMap { - if !btPathsMap[p] { - ret = append(ret, p) - } - } - ret = gulu.Str.RemoveDuplicatedElem(ret) - return -} - -func GetBlockTreeRootByPath(boxID, path string) *BlockTree { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, blockTree := range blockTrees { - if blockTree.BoxID == boxID && blockTree.Path == path && blockTree.RootID == blockTree.ID { - return blockTree - } - } - return nil -} - -func GetBlockTreeRootByHPath(boxID, hPath string) *BlockTree { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, blockTree := range blockTrees { - if blockTree.BoxID == boxID && blockTree.HPath == hPath && blockTree.RootID == blockTree.ID { - return blockTree - } - } - return nil -} - -func GetBlockTree(id string) *BlockTree { - if "" == id { - return nil - } - - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - return blockTrees[id] -} - -func SetBlockTreePath(tree *parse.Tree) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, b := range blockTrees { - if b.RootID == tree.ID { - b.BoxID, b.Path, b.HPath, b.Updated, b.Type = tree.Box, tree.Path, tree.HPath, tree.Root.IALAttr("updated"), TypeAbbr(ast.NodeDocument.String()) - } - } - blockTreesChanged = time.Now() -} - -func RemoveBlockTreesByRootID(rootID string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - var ids []string - for _, b := range blockTrees { - if b.RootID == rootID { - ids = append(ids, b.RootID) - } - } - ids = gulu.Str.RemoveDuplicatedElem(ids) - for _, id := range ids { - delete(blockTrees, id) - } - blockTreesChanged = time.Now() -} - -func RemoveBlockTreesByPath(path string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - var ids []string - for _, b := range blockTrees { - if b.Path == path { - ids = append(ids, b.ID) - } - } - ids = gulu.Str.RemoveDuplicatedElem(ids) - for _, id := range ids { - delete(blockTrees, id) - } - blockTreesChanged = time.Now() -} - -func RemoveBlockTreesByPathPrefix(pathPrefix string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - var ids []string - for _, b := range blockTrees { - if strings.HasPrefix(b.Path, pathPrefix) { - ids = append(ids, b.ID) - } - } - ids = gulu.Str.RemoveDuplicatedElem(ids) - for _, id := range ids { - delete(blockTrees, id) - } - blockTreesChanged = time.Now() -} - -func RemoveBlockTreesByBoxID(boxID string) (ids []string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - for _, b := range blockTrees { - if b.BoxID == boxID { - ids = append(ids, b.ID) - } - } - ids = gulu.Str.RemoveDuplicatedElem(ids) - for _, id := range ids { - delete(blockTrees, id) - } - blockTreesChanged = time.Now() - return -} - -func RemoveBlockTree(id string) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - delete(blockTrees, id) - blockTreesChanged = time.Now() -} - -func IndexBlockTree(tree *parse.Tree) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering || !n.IsBlock() { - return ast.WalkContinue - } - var parentID string - if nil != n.Parent { - parentID = n.Parent.ID - } - if "" == n.ID { - return ast.WalkContinue - } - blockTrees[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: tree.Root.IALAttr("updated"), Type: TypeAbbr(n.Type.String())} - return ast.WalkContinue - }) - blockTreesChanged = time.Now() -} - -func AutoFlushBlockTree() { - for { - SaveBlockTree(false) - time.Sleep(1 * time.Second) - } -} - -func InitBlockTree(force bool) { - start := time.Now() - - if force { - err := os.RemoveAll(util.BlockTreePath) - if nil != err { - logging.LogErrorf("remove blocktree file failed: %s", err) - } - return - } - - var err error - fh, err := os.OpenFile(util.BlockTreePath, os.O_RDWR, 0644) - if nil != err { - logging.LogErrorf("open block tree file failed: %s", err) - os.Exit(util.ExitCodeBlockTreeErr) - return - } - defer fh.Close() - - data, err := io.ReadAll(fh) - if nil != err { - logging.LogErrorf("read block tree failed: %s", err) - os.Exit(util.ExitCodeBlockTreeErr) - return - } - blockTreesLock.Lock() - if err = msgpack.Unmarshal(data, &blockTrees); nil != err { - logging.LogErrorf("unmarshal block tree failed: %s", err) - if err = os.RemoveAll(util.BlockTreePath); nil != err { - logging.LogErrorf("removed corrupted block tree failed: %s", err) - } - os.Exit(util.ExitCodeBlockTreeErr) - return - } - blockTreesLock.Unlock() - runtime.GC() - - if elapsed := time.Since(start).Seconds(); 2 < elapsed { - logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed) - } - return -} - -func SaveBlockTree(force bool) { - if !force && blockTreesChanged.IsZero() { - return - } - - start := time.Now() - if blockTreesChanged.After(start.Add(-7 * time.Second)) { - return - } - - blockTreesLock.Lock() - data, err := msgpack.Marshal(blockTrees) - if nil != err { - logging.LogErrorf("marshal block tree failed: %s", err) - os.Exit(util.ExitCodeBlockTreeErr) - return - } - blockTreesLock.Unlock() - - if err = gulu.File.WriteFileSafer(util.BlockTreePath, data, 0644); nil != err { - logging.LogErrorf("write block tree failed: %s", err) - os.Exit(util.ExitCodeBlockTreeErr) - return - } - runtime.GC() - - if elapsed := time.Since(start).Seconds(); 2 < elapsed { - logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed) - } - - blockTreesChanged = time.Time{} +func btHash(id string) string { + return util2.Hash([]byte(id))[0:2] } diff --git a/kernel/util/working.go b/kernel/util/working.go index 0aaa8d80a..5b11f8b9f 100644 --- a/kernel/util/working.go +++ b/kernel/util/working.go @@ -253,7 +253,7 @@ func initWorkspaceDir(workspaceArg string) { os.Setenv("TMP", osTmpDir) DBPath = filepath.Join(TempDir, DBName) HistoryDBPath = filepath.Join(TempDir, "history.db") - BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack") + BlockTreePath = filepath.Join(TempDir, "blocktree") SnippetsPath = filepath.Join(DataDir, "snippets") } diff --git a/kernel/util/working_mobile.go b/kernel/util/working_mobile.go index e998996b8..abbcadaff 100644 --- a/kernel/util/working_mobile.go +++ b/kernel/util/working_mobile.go @@ -155,7 +155,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) { os.Setenv("TMP", osTmpDir) DBPath = filepath.Join(TempDir, DBName) HistoryDBPath = filepath.Join(TempDir, "history.db") - BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack") + BlockTreePath = filepath.Join(TempDir, "blocktree") SnippetsPath = filepath.Join(DataDir, "snippets") AppearancePath = filepath.Join(ConfDir, "appearance") From fd5538a7b0f071dafff82078bd5d14f90469ff04 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 17:17:50 +0800 Subject: [PATCH 08/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=9D=97?= =?UTF-8?q?=E6=A0=91=E6=95=B0=E6=8D=AE=E5=AD=98=E5=8F=96=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7168?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/treenode/blocktree.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index e243ed7c2..01db7bad1 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -472,11 +472,6 @@ func InitBlockTree(force bool) { } func SaveBlockTree(force bool) { - if force { - - return - } - start := time.Now() os.MkdirAll(util.BlockTreePath, 0755) From 3c79a226bea8f48a3e89e001ca16bdffabc38a92 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 17:23:45 +0800 Subject: [PATCH 09/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 2 +- app/appearance/langs/es_ES.json | 2 +- app/appearance/langs/fr_FR.json | 2 +- app/appearance/langs/zh_CHT.json | 2 +- app/appearance/langs/zh_CN.json | 2 +- kernel/model/index_fix.go | 11 ++++++++--- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 695142b39..a8b97f415 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -945,7 +945,7 @@ "55": "Indexed references of [%d] documents", "56": "Reindexing, please wait until rebuilding is complete before trying to open", "57": "Failed to create temp key", - "58": "TODO", + "58": "Verifying index...", "59": "Failed to set sync ignore list", "60": "Failed to get the update package: %s", "61": "⬆️ The new version installation package is ready, do you want to install the new version now?", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index ab6f57240..e0ddce8c3 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -945,7 +945,7 @@ "55": "Referencias indexadas de [%d] documentos", "56": "Reindexando, espere hasta que se complete la reconstrucción antes de intentar abrir", "57": "Fallo en la creación de la clave temporal", - "58": "TODO", + "58": "Verificando índice...", "59": " Falló la configuración de sincronización de la lista de ignorados", "60": "Fallo al obtener el paquete de actualización: %s", "61": "⬆️ El paquete de instalación de la nueva versión está listo, ¿quieres instalar la nueva versión ahora?", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index ef641a34f..233156012 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -945,7 +945,7 @@ "55": "Références indexées de [%d] documents", "56": "Réindexation, veuillez attendre que la reconstruction soit terminée avant d'essayer d'ouvrir", "57": "Échec de la création d'une clé temporaire", - "58": "TODO", + "58": "Vérification de l'index...", "59": "Échec de la définition de la liste des ignores de synchronisation", "60": "Échec de la récupération du paquet de mise à jour : %s", "61": "⬆️ Le package d'installation de la nouvelle version est prêt, voulez-vous installer la nouvelle version maintenant ?", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 5bfa6fc5a..de7848c8a 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -945,7 +945,7 @@ "55": "已完成索引 [%d] 篇文檔的引用關係", "56": "正在重建索引，請等重建索引完畢後再嘗試打開", "57": "創建臨時金鑰失敗", - "58": "TODO", + "58": "正在校驗索引...", "59": "設置同步忽略列表失敗", "60": "獲取更新包失敗：%s", "61": "⬆️ 新版本安裝包已經準備就緒，是否現在安裝新版本？", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index a7f4dd4fa..7d225d82a 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -945,7 +945,7 @@ "55": "已完成索引 [%d] 篇文档的引用关系", "56": "正在重建索引，请等重建索引完毕后再尝试打开", "57": "创建临时密钥失败", - "58": "TODO", + "58": "正在校验索引...", "59": "设置同步忽略列表失败", "60": "获取更新包失败：%s", "61": "⬆️ 新版本安装包已经准备就绪，是否现在安装新版本？", diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go index 97bfae4af..6f86ad195 100644 --- a/kernel/model/index_fix.go +++ b/kernel/model/index_fix.go @@ -52,6 +52,8 @@ func autoFixIndex() { autoFixLock.Lock() defer autoFixLock.Unlock() + util.PushStatusBar(Conf.Language(58)) + // 去除重复的数据库块记录 duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks") if 1 > len(duplicatedRootIDs) { @@ -61,6 +63,7 @@ func autoFixIndex() { } } + util.PushStatusBar(Conf.Language(58)) roots := sql.GetBlocks(duplicatedRootIDs) rootMap := map[string]*sql.Block{} for _, root := range roots { @@ -83,8 +86,9 @@ func autoFixIndex() { logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", len(duplicatedRootIDs)) } + util.PushStatusBar(Conf.Language(58)) sql.WaitForWritingDatabase() - + util.PushStatusBar(Conf.Language(58)) // 根据文件系统补全块树 boxes := Conf.GetOpenedBoxes() for _, box := range boxes { @@ -125,8 +129,9 @@ func autoFixIndex() { } } + util.PushStatusBar(Conf.Language(58)) sql.WaitForWritingDatabase() - + util.PushStatusBar(Conf.Language(58)) // 清理已关闭的笔记本块树 boxes = Conf.GetClosedBoxes() for _, box := range boxes { @@ -140,8 +145,8 @@ func autoFixIndex() { reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap) } + util.PushStatusBar(Conf.Language(58)) sql.WaitForWritingDatabase() - util.PushStatusBar(Conf.Language(185)) } From f8ef5639c08066800ba631a8fa60067a0a5486c5 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 17:50:21 +0800 Subject: [PATCH 10/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 2 +- app/appearance/langs/es_ES.json | 2 +- app/appearance/langs/fr_FR.json | 2 +- app/appearance/langs/zh_CHT.json | 2 +- app/appearance/langs/zh_CN.json | 2 +- kernel/go.mod | 2 +- kernel/go.sum | 2 ++ kernel/model/index.go | 12 ++++++++++++ kernel/model/index_fix.go | 7 ++++--- kernel/sql/cache.go | 4 ++-- kernel/sql/database.go | 4 +++- kernel/sql/queue.go | 28 +++++++++++----------------- 12 files changed, 40 insertions(+), 29 deletions(-) diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index a8b97f415..c2acc177b 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -980,7 +980,7 @@ "90": "[%d/%d] Created [%d] of search indexes of block-level elements [%s]", "91": "Reading block tree data...", "92": "Parsing document tree [%s]", - "93": "TODO", + "93": "[%d/%d] Cleaned up the index related to document [%s]", "94": "Upload failed: %s", "95": "Exiting...", "96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index e0ddce8c3..8f6eb88d0 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -980,7 +980,7 @@ "90": "[%d/%d] Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]", "91": "Leyendo datos del árbol de bloques...", "92": "Analizando el árbol del documento [%s]", - "93": "TODO", + "93": "[%d/%d] ha limpiado el índice relacionado con el documento [%s]", "94": "Carga fallida: %s", "95": "Saliendo...", "96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 233156012..36fb2d97d 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -980,7 +980,7 @@ "90": "[%d/%d] Création de [%d] index de recherche d'éléments de niveau bloc [%s]", "91": "Lecture des données de l'arborescence des blocs...", "92": "Analyse de l'arborescence du document [%s]", - "93": "TODO", + "93": "[%d/%d] a nettoyé l'index lié au document [%s]", "94": "Échec du téléchargement : %s", "95": "Quitter le programme...", "96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index de7848c8a..ade184f41 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -980,7 +980,7 @@ "90": "[%d/%d] 已經建立 [%d] 個塊級元素的搜索索引 [%s]", "91": "正在讀取塊樹數據...", "92": "正在解析文檔樹 [%s]", - "93": "TODO", + "93": "[%d/%d] 已經清理文檔 [%s] 相關的索引", "94": "上傳失敗：%s", "95": "正在退出...", "96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 7d225d82a..a4d28afb7 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -980,7 +980,7 @@ "90": "[%d/%d] 已经建立 [%d] 个块级元素的搜索索引 [%s]", "91": "正在读取块树数据...", "92": "正在解析文档树 [%s]", - "93": "TODO", + "93": "[%d/%d] 已经删除文档 [%s] 相关的索引", "94": "上传失败：%s", "95": "正在退出...", "96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致", diff --git a/kernel/go.mod b/kernel/go.mod index 5e614d32d..cc044a6ca 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -41,7 +41,7 @@ require ( github.com/shirou/gopsutil/v3 v3.22.12 github.com/siyuan-note/dejavu v0.0.0-20230117131301-821aa3adc1e7 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 - github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da + github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2 github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e github.com/siyuan-note/httpclient v0.0.0-20230116125720-ee36ddf6f223 github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a diff --git a/kernel/go.sum b/kernel/go.sum index fe258a3db..fc0df239f 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -377,6 +377,8 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da h1:/jNhl7LC+9BhkWvNxuJDdsNfA/2wvfuj9mqWx4CbV90= github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= +github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2 h1:njoj0265FOHRjF/O1aWDzdwEdYdLTZwhbuPd/hoLJT0= +github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e h1:i3RKrdrddr4AuaHJtoWYAEVNuR7Y9wIsEqPmuFFbJC4= github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e/go.mod h1:NmpSIVtIGy8eNWapjDIiiCw5+5r5wxC76k40oG+WRXQ= github.com/siyuan-note/httpclient v0.0.0-20230116125720-ee36ddf6f223 h1:hG+gucj92x4Dl4lIe2G0WkPgBdlEBnnQCmYpghHeW54= diff --git a/kernel/model/index.go b/kernel/model/index.go index d252b9a42..eee8e68af 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -253,4 +253,16 @@ func init() { util.SetBootDetails(msg) util.ContextPushMsg(context, msg) }) + eventbus.Subscribe(eventbus.EvtSQLDeleteBlocks, func(context map[string]interface{}, rootID string) { + if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container { + // Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392 + return + } + + current := context["current"].(int) + 1 + total := context["total"] + msg := fmt.Sprintf(Conf.Language(93), current, total, rootID) + util.SetBootDetails(msg) + util.ContextPushMsg(context, msg) + }) } diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go index 6f86ad195..e5b332886 100644 --- a/kernel/model/index_fix.go +++ b/kernel/model/index_fix.go @@ -69,6 +69,7 @@ func autoFixIndex() { for _, root := range roots { rootMap[root.ID] = root } + var deletes int for _, rootID := range duplicatedRootIDs { root := rootMap[rootID] if nil == root { @@ -77,13 +78,13 @@ func autoFixIndex() { //logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) sql.RemoveTreeQueue(root.Box, rootID) - + deletes++ if util.IsExiting { break } } - if 0 < len(duplicatedRootIDs) { - logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", len(duplicatedRootIDs)) + if 0 < deletes { + logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", deletes) } util.PushStatusBar(Conf.Language(58)) diff --git a/kernel/sql/cache.go b/kernel/sql/cache.go index c0cc49e8f..6b4849d36 100644 --- a/kernel/sql/cache.go +++ b/kernel/sql/cache.go @@ -17,7 +17,7 @@ package sql import ( - "runtime/debug" + "runtime" "time" "github.com/88250/lute/ast" @@ -45,7 +45,7 @@ func DisableCache() { func ClearBlockCache() { memCache.Clear() - debug.FreeOSMemory() + runtime.GC() } func putBlockCache(block *Block) { diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 144b9fb14..1a8984d29 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -34,6 +34,7 @@ import ( "github.com/88250/lute/parse" "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3" + "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" @@ -947,7 +948,7 @@ func deleteFileAnnotationRefsByBoxTx(tx *sql.Tx, box string) (err error) { return } -func deleteByRootID(tx *sql.Tx, rootID string) (err error) { +func deleteByRootID(tx *sql.Tx, rootID string, context map[string]interface{}) (err error) { stmt := "DELETE FROM blocks WHERE root_id = ?" if err = execStmtTx(tx, stmt, rootID); nil != err { return @@ -969,6 +970,7 @@ func deleteByRootID(tx *sql.Tx, rootID string) (err error) { return } ClearBlockCache() + eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, rootID) return } diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index 1b8e5f6db..3ed95779f 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -147,7 +147,7 @@ func execOp(op *dbQueueOperation, tx *sql.Tx, context map[string]interface{}) (e case "delete": err = batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath) case "delete_id": - err = deleteByRootID(tx, op.removeTreeID) + err = deleteByRootID(tx, op.removeTreeID, context) case "rename": err = batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath) if nil != err { @@ -285,16 +285,13 @@ func RemoveTreeQueue(box, rootID string) { dbQueueLock.Lock() defer dbQueueLock.Unlock() - var tmp []*dbQueueOperation - // 将已有的 upsert 操作去重 - for _, op := range operationQueue { - if "upsert" == op.action && op.upsertTree.ID != rootID { - tmp = append(tmp, op) + newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"} + for i, op := range operationQueue { + if "delete_id" == op.action && op.removeTreeIDBox == box && op.removeTreeID == rootID { + operationQueue[i] = newOp + return } } - operationQueue = tmp - - newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"} operationQueue = append(operationQueue, newOp) } @@ -302,15 +299,12 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) { dbQueueLock.Lock() defer dbQueueLock.Unlock() - var tmp []*dbQueueOperation - // 将已有的 upsert 操作去重 - for _, op := range operationQueue { - if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) { - tmp = append(tmp, op) + newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"} + for i, op := range operationQueue { + if "delete" == op.action && (op.removeTreeBox == treeBox && op.removeTreePath == treePathPrefix) { + operationQueue[i] = newOp + return } } - operationQueue = tmp - - newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"} operationQueue = append(operationQueue, newOp) } From b1cccc1639e85380959d2f14350581d4436009bb Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 18:01:01 +0800 Subject: [PATCH 11/12] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/zh_CN.json | 2 +- kernel/sql/database.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index a4d28afb7..93a2c7183 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -980,7 +980,7 @@ "90": "[%d/%d] 已经建立 [%d] 个块级元素的搜索索引 [%s]", "91": "正在读取块树数据...", "92": "正在解析文档树 [%s]", - "93": "[%d/%d] 已经删除文档 [%s] 相关的索引", + "93": "[%d/%d] 已经清理文档 [%s] 相关的索引", "94": "上传失败：%s", "95": "正在退出...", "96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致", diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 1a8984d29..7d0bcb975 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -953,6 +953,14 @@ func deleteByRootID(tx *sql.Tx, rootID string, context map[string]interface{}) ( if err = execStmtTx(tx, stmt, rootID); nil != err { return } + stmt = "DELETE FROM blocks_fts WHERE root_id = ?" + if err = execStmtTx(tx, stmt, rootID); nil != err { + return + } + stmt = "DELETE FROM blocks_fts_case_insensitive WHERE root_id = ?" + if err = execStmtTx(tx, stmt, rootID); nil != err { + return + } stmt = "DELETE FROM spans WHERE root_id = ?" if err = execStmtTx(tx, stmt, rootID); nil != err { return From e2fc3f442cbf1d5b84b9963200ee9ceec6ba043d Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 18:01:59 +0800 Subject: [PATCH 12/12] =?UTF-8?q?:art:=20=E6=95=B0=E6=8D=AE=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E6=97=B6=E5=B1=95=E5=BC=80=E6=96=87=E6=A1=A3=E6=A0=91?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E5=8F=AF=E8=83=BD=E5=AF=BC=E8=87=B4=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=B8=A2=E5=A4=B1=20https://github.com/siyuan-note/si?= =?UTF-8?q?yuan/issues/7129?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/conf.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/kernel/model/conf.go b/kernel/model/conf.go index b27c07d53..a1389eca5 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -696,20 +696,7 @@ func clearCorruptedNotebooks() { boxDirPath := filepath.Join(util.DataDir, dir.Name()) boxConfPath := filepath.Join(boxDirPath, ".siyuan", "conf.json") if !gulu.File.IsExist(boxConfPath) { - if IsUserGuide(dir.Name()) { - filelock.Remove(boxDirPath) - continue - } - to := filepath.Join(util.WorkspaceDir, "corrupted", time.Now().Format("2006-01-02-150405"), dir.Name()) - if copyErr := filelock.Copy(boxDirPath, to); nil != copyErr { - logging.LogErrorf("copy corrupted box [%s] failed: %s", boxDirPath, copyErr) - continue - } - if removeErr := filelock.Remove(boxDirPath); nil != removeErr { - logging.LogErrorf("remove corrupted box [%s] failed: %s", boxDirPath, removeErr) - continue - } - logging.LogWarnf("moved corrupted box [%s] to [%s]", boxDirPath, to) + logging.LogWarnf("found a corrupted box [%s]", boxDirPath) continue } }