From 225930d3cf6a2f1256cae1447100a00f7f63bd90 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Tue, 9 Apr 2024 17:51:39 +0800 Subject: [PATCH 01/10] :art: Improve `Add to Database` search sorting https://github.com/siyuan-note/siyuan/issues/10952 --- kernel/model/attribute_view.go | 34 ++++++++++++++++++++++++++-------- kernel/model/search.go | 17 +++++++++++------ 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/kernel/model/attribute_view.go b/kernel/model/attribute_view.go index 25bcc4a06..934af36ae 100644 --- a/kernel/model/attribute_view.go +++ b/kernel/model/attribute_view.go @@ -36,6 +36,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" + "github.com/xrash/smetrics" ) func SetDatabaseBlockView(blockID, viewID string) (err error) { @@ -194,7 +195,12 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { ret = []*SearchAttributeViewResult{} keyword = strings.TrimSpace(keyword) - avs := map[string]string{} + type result struct { + AvID string + AvName string + Score float64 + } + var avs []*result avDir := filepath.Join(util.DataDir, "storage", "av") const limit = 16 entries, err := os.ReadDir(avDir) @@ -220,17 +226,23 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { } if strings.Contains(strings.ToLower(name), strings.ToLower(keyword)) { - avs[id] = name + score := smetrics.JaroWinkler(name, keyword, 0.7, 4) + avs = append(avs, &result{AvID: id, AvName: name, Score: score}) count++ - if limit <= count { + if "" == keyword && limit <= count { break } } } + sort.Slice(avs, func(i, j int) bool { return avs[i].Score > avs[j].Score }) + if limit < len(avs) { + avs = avs[:limit] + } + var avIDs []string - for avID := range avs { - avIDs = append(avIDs, avID) + for _, av := range avs { + avIDs = append(avIDs, av.AvID) } blockIDs := treenode.BatchGetMirrorAttrViewBlockIDs(avIDs) trees := map[string]*parse.Tree{} @@ -261,8 +273,14 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { } avID := node.AttributeViewID - name := avs[avID] - if "" == name { + var existAv *result + for _, av := range avs { + if av.AvID == avID { + existAv = av + break + } + } + if nil == existAv { continue } @@ -287,7 +305,7 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { if !exist { ret = append(ret, &SearchAttributeViewResult{ AvID: avID, - AvName: name, + AvName: existAv.AvName, BlockID: blockID, HPath: hPath, }) diff --git a/kernel/model/search.go b/kernel/model/search.go index d3e7d5378..ba519f4e5 100644 --- a/kernel/model/search.go +++ b/kernel/model/search.go @@ -384,17 +384,22 @@ func SearchRefBlock(id, rootID, keyword string, beforeLen int, isSquareBrackets } } - if b.ID != id && !hitFirstChildID && b.ID != rootID { + if "NodeAttributeView" == b.Type { + // 数据库块可以添加到自身数据库块中，当前文档也可以添加到自身数据库块中 tmp = append(tmp, b) + } else { + // 排除自身块、父块和根块 + if b.ID != id && !hitFirstChildID && b.ID != rootID { + tmp = append(tmp, b) + } } + } ret = tmp - if "" != keyword { - if block := treenode.GetBlockTree(id); nil != block { - p := path.Join(block.HPath, keyword) - newDoc = nil == treenode.GetBlockTreeRootByHPath(block.BoxID, p) - } + if block := treenode.GetBlockTree(id); nil != block { + p := path.Join(block.HPath, keyword) + newDoc = nil == treenode.GetBlockTreeRootByHPath(block.BoxID, p) } // 在 hPath 中加入笔记本名 Show notebooks in hpath of block ref search list results https://github.com/siyuan-note/siyuan/issues/9378 From c4d2f9e706de6dc34cf0d56a8b1874a43dba686b Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Tue, 9 Apr 2024 21:26:41 +0800 Subject: [PATCH 02/10] :art: Improve `Add to Database` search sorting https://github.com/siyuan-note/siyuan/issues/10952 --- kernel/model/attribute_view.go | 51 ++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/kernel/model/attribute_view.go b/kernel/model/attribute_view.go index 934af36ae..33b65ea81 100644 --- a/kernel/model/attribute_view.go +++ b/kernel/model/attribute_view.go @@ -196,20 +196,19 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { keyword = strings.TrimSpace(keyword) type result struct { - AvID string - AvName string - Score float64 + AvID string + AvName string + AvUpdated int64 + Score float64 } var avs []*result avDir := filepath.Join(util.DataDir, "storage", "av") - const limit = 16 entries, err := os.ReadDir(avDir) if nil != err { logging.LogErrorf("read directory [%s] failed: %s", avDir, err) return } - count := 0 for _, entry := range entries { if entry.IsDir() { continue @@ -221,29 +220,39 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { } name, _ := av.GetAttributeViewNameByPath(filepath.Join(avDir, entry.Name())) - if "" == name { - continue - } - - if strings.Contains(strings.ToLower(name), strings.ToLower(keyword)) { - score := smetrics.JaroWinkler(name, keyword, 0.7, 4) - avs = append(avs, &result{AvID: id, AvName: name, Score: score}) - count++ - if "" == keyword && limit <= count { - break + info, _ := entry.Info() + if "" != keyword { + if strings.Contains(strings.ToLower(name), strings.ToLower(keyword)) { + score := smetrics.JaroWinkler(name, keyword, 0.7, 4) + a := &result{AvID: id, AvName: name, Score: score} + if nil != info && !info.ModTime().IsZero() { + a.AvUpdated = info.ModTime().UnixMilli() + } + avs = append(avs, a) } + } else { + a := &result{AvID: id, AvName: name} + if nil != info && !info.ModTime().IsZero() { + a.AvUpdated = info.ModTime().UnixMilli() + } + avs = append(avs, a) } + } - sort.Slice(avs, func(i, j int) bool { return avs[i].Score > avs[j].Score }) - if limit < len(avs) { - avs = avs[:limit] + if "" == keyword { + sort.Slice(avs, func(i, j int) bool { return avs[i].AvUpdated > avs[j].AvUpdated }) + } else { + sort.Slice(avs, func(i, j int) bool { return avs[i].Score > avs[j].Score }) + } + if 16 < len(avs) { + avs = avs[:16] } - var avIDs []string - for _, av := range avs { - avIDs = append(avIDs, av.AvID) + for _, a := range avs { + avIDs = append(avIDs, a.AvID) } + blockIDs := treenode.BatchGetMirrorAttrViewBlockIDs(avIDs) trees := map[string]*parse.Tree{} for _, blockID := range blockIDs { From 7d680ed39192f0a8379cfd6bee233bed98824a65 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Tue, 9 Apr 2024 22:26:18 +0800 Subject: [PATCH 03/10] :art: Rebuilding database block relations when importing .sy.zip and rebuilding indexes https://github.com/siyuan-note/siyuan/issues/10959 --- kernel/model/import.go | 4 ++++ kernel/model/index.go | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/kernel/model/import.go b/kernel/model/import.go index 641e0cea1..938cf00e9 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -281,6 +281,10 @@ func ImportSY(zipPath, boxID, toPath string) (err error) { } return ast.WalkContinue }) + + // 关联数据库和块 + avNodes := tree.Root.ChildrenByType(ast.NodeAttributeView) + av.BatchUpsertBlockRel(avNodes) } } diff --git a/kernel/model/index.go b/kernel/model/index.go index 710f77330..3a325368a 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -36,6 +36,7 @@ import ( "github.com/siyuan-note/eventbus" "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/av" "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" @@ -143,6 +144,7 @@ func index(boxID string) { poolSize = 4 } waitGroup := &sync.WaitGroup{} + var avNodes []*ast.Node p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) { defer waitGroup.Done() @@ -168,6 +170,10 @@ func index(boxID string) { } } + lock.Lock() + avNodes = append(avNodes, tree.Root.ChildrenByType(ast.NodeAttributeView)...) + lock.Unlock() + cache.PutDocIAL(file.path, docIAL) treenode.IndexBlockTree(tree) sql.IndexTreeQueue(tree) @@ -191,6 +197,9 @@ func index(boxID string) { waitGroup.Wait() p.Release() + // 关联数据库和块 + av.BatchUpsertBlockRel(avNodes) + box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 end := time.Now() elapsed := end.Sub(start).Seconds() From b3a2c49f15e4aaa0099de827f3637f2016cfeab8 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Tue, 9 Apr 2024 22:26:37 +0800 Subject: [PATCH 04/10] :art: Improve `Add to Database` search sorting https://github.com/siyuan-note/siyuan/issues/10952 --- kernel/av/mirror.go | 75 +++++++++++++++++++++++++++++++ kernel/model/attribute_view.go | 26 ++++++++--- kernel/treenode/av.go | 81 +++++++++++++++++++--------------- 3 files changed, 141 insertions(+), 41 deletions(-) diff --git a/kernel/av/mirror.go b/kernel/av/mirror.go index 24f02ef18..8814d8f56 100644 --- a/kernel/av/mirror.go +++ b/kernel/av/mirror.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/88250/gulu" + "github.com/88250/lute/ast" "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" @@ -16,6 +17,30 @@ var ( AttributeViewBlocksLock = sync.Mutex{} ) +func GetBlockRels() (ret map[string][]string) { + AttributeViewBlocksLock.Lock() + defer AttributeViewBlocksLock.Unlock() + + ret = map[string][]string{} + + blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack") + if !filelock.IsExist(blocks) { + return + } + + data, err := filelock.ReadFile(blocks) + if nil != err { + logging.LogErrorf("read attribute view blocks failed: %s", err) + return + } + + if err = msgpack.Unmarshal(data, &ret); nil != err { + logging.LogErrorf("unmarshal attribute view blocks failed: %s", err) + return + } + return +} + func IsMirror(avID string) bool { AttributeViewBlocksLock.Lock() defer AttributeViewBlocksLock.Unlock() @@ -86,6 +111,56 @@ func RemoveBlockRel(avID, blockID string) { } } +func BatchUpsertBlockRel(nodes []*ast.Node) { + AttributeViewBlocksLock.Lock() + defer AttributeViewBlocksLock.Unlock() + + avBlocks := map[string][]string{} + blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack") + if !filelock.IsExist(blocks) { + if err := os.MkdirAll(filepath.Dir(blocks), 0755); nil != err { + logging.LogErrorf("create attribute view dir failed: %s", err) + return + } + } else { + data, err := filelock.ReadFile(blocks) + if nil != err { + logging.LogErrorf("read attribute view blocks failed: %s", err) + return + } + + if err = msgpack.Unmarshal(data, &avBlocks); nil != err { + logging.LogErrorf("unmarshal attribute view blocks failed: %s", err) + return + } + } + + for _, n := range nodes { + if ast.NodeAttributeView != n.Type { + continue + } + + if "" == n.AttributeViewID || "" == n.ID { + continue + } + + blockIDs := avBlocks[n.AttributeViewID] + blockIDs = append(blockIDs, n.ID) + blockIDs = gulu.Str.RemoveDuplicatedElem(blockIDs) + avBlocks[n.AttributeViewID] = blockIDs + } + + data, err := msgpack.Marshal(avBlocks) + if nil != err { + logging.LogErrorf("marshal attribute view blocks failed: %s", err) + return + } + if err = filelock.WriteFile(blocks, data); nil != err { + logging.LogErrorf("write attribute view blocks failed: %s", err) + return + } +} + func UpsertBlockRel(avID, blockID string) { AttributeViewBlocksLock.Lock() defer AttributeViewBlocksLock.Unlock() diff --git a/kernel/model/attribute_view.go b/kernel/model/attribute_view.go index 33b65ea81..39a548896 100644 --- a/kernel/model/attribute_view.go +++ b/kernel/model/attribute_view.go @@ -208,7 +208,7 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { logging.LogErrorf("read directory [%s] failed: %s", avDir, err) return } - + avBlockRels := av.GetBlockRels() for _, entry := range entries { if entry.IsDir() { continue @@ -219,6 +219,10 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { continue } + if nil == avBlockRels[id] { + continue + } + name, _ := av.GetAttributeViewNameByPath(filepath.Join(avDir, entry.Name())) info, _ := entry.Info() if "" != keyword { @@ -237,23 +241,33 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) { } avs = append(avs, a) } - } if "" == keyword { sort.Slice(avs, func(i, j int) bool { return avs[i].AvUpdated > avs[j].AvUpdated }) } else { - sort.Slice(avs, func(i, j int) bool { return avs[i].Score > avs[j].Score }) + sort.SliceStable(avs, func(i, j int) bool { + if avs[i].Score == avs[j].Score { + return avs[i].AvUpdated > avs[j].AvUpdated + } + return avs[i].Score > avs[j].Score + }) } - if 16 < len(avs) { - avs = avs[:16] + if 12 <= len(avs) { + avs = avs[:12] } var avIDs []string for _, a := range avs { avIDs = append(avIDs, a.AvID) } - blockIDs := treenode.BatchGetMirrorAttrViewBlockIDs(avIDs) + avBlocks := treenode.BatchGetMirrorAttrViewBlocks(avIDs) + var blockIDs []string + for _, avBlock := range avBlocks { + blockIDs = append(blockIDs, avBlock.BlockIDs...) + } + blockIDs = gulu.Str.RemoveDuplicatedElem(blockIDs) + trees := map[string]*parse.Tree{} for _, blockID := range blockIDs { bt := treenode.GetBlockTree(blockID) diff --git a/kernel/treenode/av.go b/kernel/treenode/av.go index d2137d4f1..6fd0e1720 100644 --- a/kernel/treenode/av.go +++ b/kernel/treenode/av.go @@ -26,41 +26,6 @@ import ( "github.com/vmihailenco/msgpack/v5" ) -func BatchGetMirrorAttrViewBlockIDs(avIDs []string) (ret map[string]string) { - av.AttributeViewBlocksLock.Lock() - defer av.AttributeViewBlocksLock.Unlock() - - ret = map[string]string{} - - blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack") - if !filelock.IsExist(blocks) { - return - } - - data, err := filelock.ReadFile(blocks) - if nil != err { - logging.LogErrorf("read attribute view blocks failed: %s", err) - return - } - - avBlocks := map[string][]string{} - if err = msgpack.Unmarshal(data, &avBlocks); nil != err { - logging.LogErrorf("unmarshal attribute view blocks failed: %s", err) - return - } - - for _, avID := range avIDs { - blockIDs := avBlocks[avID] - for _, blockID := range blockIDs { - if nil != GetBlockTree(blockID) { - ret[avID] = blockID - break - } - } - } - return -} - func GetMirrorAttrViewBlockIDs(avID string) (ret []string) { av.AttributeViewBlocksLock.Lock() defer av.AttributeViewBlocksLock.Unlock() @@ -91,3 +56,49 @@ func GetMirrorAttrViewBlockIDs(avID string) (ret []string) { } return } + +type AvBlock struct { + AvID string + BlockIDs []string +} + +func BatchGetMirrorAttrViewBlocks(avIDs []string) (ret []*AvBlock) { + av.AttributeViewBlocksLock.Lock() + defer av.AttributeViewBlocksLock.Unlock() + + ret = []*AvBlock{} + + blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack") + if !filelock.IsExist(blocks) { + return + } + + data, err := filelock.ReadFile(blocks) + if nil != err { + logging.LogErrorf("read attribute view blocks failed: %s", err) + return + } + + avBlocks := map[string][]string{} + if err = msgpack.Unmarshal(data, &avBlocks); nil != err { + logging.LogErrorf("unmarshal attribute view blocks failed: %s", err) + return + } + + for _, avID := range avIDs { + var blockIDs []string + for _, blockID := range avBlocks[avID] { + if nil == GetBlockTree(blockID) { + continue + } + + blockIDs = append(blockIDs, blockID) + } + avBlock := &AvBlock{ + AvID: avID, + BlockIDs: blockIDs, + } + ret = append(ret, avBlock) + } + return +} From 5e116a16d277d6370e858ff80041cde3a306c88b Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Tue, 9 Apr 2024 23:12:26 +0800 Subject: [PATCH 05/10] :art: Referenced blocks support rendering virtual references https://github.com/siyuan-note/siyuan/issues/10960 --- kernel/model/virutalref.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 668bb204e..9ff20fbbd 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -144,7 +144,7 @@ func AddVirtualBlockRefExclude(keyword []string) { } func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool { - if !Conf.Editor.VirtualBlockRef { + if !Conf.Editor.VirtualBlockRef || 1 > len(virtualBlockRefKeywords) { return false } @@ -153,12 +153,15 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword } parentBlock := treenode.ParentBlock(n) - if nil == parentBlock || 0 < refCount[parentBlock.ID] { + if nil == parentBlock { return false } - if 1 > len(virtualBlockRefKeywords) { - return false + if 0 < refCount[parentBlock.ID] { + // 如果块被引用过，则将其自身的文本排除在虚拟引用关键字之外 + // Referenced blocks support rendering virtual references https://github.com/siyuan-note/siyuan/issues/10960 + parentText := getNodeRefText(parentBlock) + virtualBlockRefKeywords = gulu.Str.RemoveElem(virtualBlockRefKeywords, parentText) } content := string(n.Tokens) From 59d7b4dfc53164ec35a1060b969b934f7f317889 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Tue, 9 Apr 2024 23:37:00 +0800 Subject: [PATCH 06/10] :art: Improve database null value field sorting Fix https://github.com/siyuan-note/siyuan/issues/10951 --- kernel/av/av.go | 2 +- kernel/av/sort.go | 8 ++++---- kernel/model/attribute_view.go | 2 +- kernel/treenode/node.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/av/av.go b/kernel/av/av.go index 450de9e33..0120effb1 100644 --- a/kernel/av/av.go +++ b/kernel/av/av.go @@ -431,7 +431,7 @@ func SaveAttributeView(av *AttributeView) (err error) { } if 0 == v.UpdatedAt { - v.UpdatedAt = v.CreatedAt + 1000 + v.UpdatedAt = v.CreatedAt } } } diff --git a/kernel/av/sort.go b/kernel/av/sort.go index ad6b6d1a4..381a29479 100644 --- a/kernel/av/sort.go +++ b/kernel/av/sort.go @@ -74,9 +74,9 @@ func (value *Value) Compare(other *Value, attrView *AttributeView) int { return 0 } else { if !other.Number.IsNotEmpty { - return 0 + return 1 } - return 1 + return 0 } } case KeyTypeDate: @@ -94,9 +94,9 @@ func (value *Value) Compare(other *Value, attrView *AttributeView) int { return 0 } else { if !other.Date.IsNotEmpty { - return 0 + return 1 } - return 1 + return 0 } } case KeyTypeCreated: diff --git a/kernel/model/attribute_view.go b/kernel/model/attribute_view.go index 39a548896..0dbe323b4 100644 --- a/kernel/model/attribute_view.go +++ b/kernel/model/attribute_view.go @@ -774,7 +774,7 @@ func renderAttributeView(attrView *av.AttributeView, viewID, query string, page, } if 0 == v.UpdatedAt { - v.UpdatedAt = v.CreatedAt + 1000 + v.UpdatedAt = v.CreatedAt } } } diff --git a/kernel/treenode/node.go b/kernel/treenode/node.go index 295fc3292..2ffed90f8 100644 --- a/kernel/treenode/node.go +++ b/kernel/treenode/node.go @@ -954,7 +954,7 @@ func GetAttributeViewDefaultValue(valueID, keyID, blockID string, typ av.KeyType ret.CreatedAt = time.Now().UnixMilli() } if 0 == ret.UpdatedAt { - ret.UpdatedAt = ret.CreatedAt + 1000 + ret.UpdatedAt = ret.CreatedAt } switch typ { From 06a2f3fb5c7b4136d76cbaf353cec13e0604399a Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Wed, 10 Apr 2024 11:49:35 +0800 Subject: [PATCH 07/10] :zap: Reduce disk reads when editing documents https://github.com/siyuan-note/siyuan/issues/10961 --- kernel/sql/database.go | 70 +++++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 4cb5523ce..dcd0b9bf2 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -130,6 +130,11 @@ func initDBTables() { logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks] failed: %s", err) } + _, err = db.Exec("CREATE INDEX idx_blocks_id ON blocks(id)") + if nil != err { + logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create index [idx_blocks_id] failed: %s", err) + } + _, err = db.Exec("CREATE INDEX idx_blocks_root_id ON blocks(root_id)") if nil != err { logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create index [idx_blocks_root_id] failed: %s", err) @@ -139,7 +144,7 @@ func initDBTables() { if nil != err { logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "drop table [blocks_fts] failed: %s", err) } - _, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan\")") + _, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts USING fts5(id, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan\")") if nil != err { logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks_fts] failed: %s", err) } @@ -148,7 +153,7 @@ func initDBTables() { if nil != err { logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "drop table [blocks_fts_case_insensitive] failed: %s", err) } - _, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts_case_insensitive USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan case_insensitive\")") + _, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts_case_insensitive USING fts5(id, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan case_insensitive\")") if nil != err { logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks_fts_case_insensitive] failed: %s", err) } @@ -954,30 +959,59 @@ func deleteByBoxTx(tx *sql.Tx, box string) (err error) { } func deleteBlocksByIDs(tx *sql.Tx, ids []string) (err error) { - in := bytes.Buffer{} - in.Grow(4096) - in.WriteString("(") + placeholders := strings.Repeat("?,", len(ids)) + placeholders = placeholders[:len(placeholders)-1] + stmt := "DELETE FROM blocks WHERE id IN (" + placeholders + ")" + args := make([]interface{}, len(ids)) for i, id := range ids { - in.WriteString("'") - in.WriteString(id) - in.WriteString("'") - if i < len(ids)-1 { - in.WriteString(",") - } + args[i] = id + } + if err = execStmtTx(tx, stmt, args...); nil != err { + return + } + var ftsIDs []string + for _, id := range ids { removeBlockCache(id) + ftsIDs = append(ftsIDs, "\""+id+"\"") } - in.WriteString(")") - stmt := "DELETE FROM blocks WHERE id IN " + in.String() - if err = execStmtTx(tx, stmt); nil != err { - return - } - stmt = "DELETE FROM blocks_fts WHERE id IN " + in.String() + stmt = "SELECT ROWID FROM blocks_fts WHERE blocks_fts MATCH 'id:(" + strings.Join(ftsIDs, " OR ") + ")'" + rows, err := tx.Query(stmt) + if nil != err { + logging.LogErrorf("query blocks_fts failed: %s", err) + return + } + var rowIDs []string + for rows.Next() { + var rowID int + if err = rows.Scan(&rowID); nil != err { + return + } + rowIDs = append(rowIDs, fmt.Sprintf("%d", rowID)) + } + rows.Close() + stmt = "DELETE FROM blocks_fts WHERE rowid IN (" + strings.Join(rowIDs, ",") + ")" if err = execStmtTx(tx, stmt); nil != err { return } + if !caseSensitive { - stmt = "DELETE FROM blocks_fts_case_insensitive WHERE id IN " + in.String() + stmt = "SELECT ROWID FROM blocks_fts_case_insensitive WHERE blocks_fts_case_insensitive MATCH 'id:(" + strings.Join(ftsIDs, " OR ") + ")'" + rows, err = tx.Query(stmt) + if nil != err { + logging.LogErrorf("query blocks_fts_case_insensitive failed: %s", err) + return + } + rowIDs = nil + for rows.Next() { + var rowID int + if err = rows.Scan(&rowID); nil != err { + return + } + rowIDs = append(rowIDs, fmt.Sprintf("%d", rowID)) + } + rows.Close() + stmt = "DELETE FROM blocks_fts_case_insensitive WHERE rowid IN (" + strings.Join(rowIDs, ",") + ")" if err = execStmtTx(tx, stmt); nil != err { return } From 616821a090e77daa43a01bdf93717c95a90a6193 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Wed, 10 Apr 2024 15:38:40 +0800 Subject: [PATCH 08/10] :zap: Reduce disk reads when editing documents https://github.com/siyuan-note/siyuan/issues/10961 --- kernel/sql/database.go | 48 ++++++------------------------------------ 1 file changed, 7 insertions(+), 41 deletions(-) diff --git a/kernel/sql/database.go b/kernel/sql/database.go index dcd0b9bf2..d709b2964 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -959,59 +959,25 @@ func deleteByBoxTx(tx *sql.Tx, box string) (err error) { } func deleteBlocksByIDs(tx *sql.Tx, ids []string) (err error) { - placeholders := strings.Repeat("?,", len(ids)) - placeholders = placeholders[:len(placeholders)-1] - stmt := "DELETE FROM blocks WHERE id IN (" + placeholders + ")" - args := make([]interface{}, len(ids)) - for i, id := range ids { - args[i] = id - } - if err = execStmtTx(tx, stmt, args...); nil != err { - return - } - var ftsIDs []string for _, id := range ids { removeBlockCache(id) ftsIDs = append(ftsIDs, "\""+id+"\"") } - stmt = "SELECT ROWID FROM blocks_fts WHERE blocks_fts MATCH 'id:(" + strings.Join(ftsIDs, " OR ") + ")'" - rows, err := tx.Query(stmt) - if nil != err { - logging.LogErrorf("query blocks_fts failed: %s", err) + + stmt := "DELETE FROM blocks WHERE id IN (" + strings.Join(ftsIDs, ",") + ")" + if err = execStmtTx(tx, stmt); nil != err { return } - var rowIDs []string - for rows.Next() { - var rowID int - if err = rows.Scan(&rowID); nil != err { - return - } - rowIDs = append(rowIDs, fmt.Sprintf("%d", rowID)) - } - rows.Close() - stmt = "DELETE FROM blocks_fts WHERE rowid IN (" + strings.Join(rowIDs, ",") + ")" + + ftsIDsMatch := strings.Join(ftsIDs, " OR ") + stmt = "DELETE FROM blocks_fts WHERE rowid IN (SELECT ROWID FROM blocks_fts WHERE blocks_fts MATCH 'id:(" + ftsIDsMatch + ")')" if err = execStmtTx(tx, stmt); nil != err { return } if !caseSensitive { - stmt = "SELECT ROWID FROM blocks_fts_case_insensitive WHERE blocks_fts_case_insensitive MATCH 'id:(" + strings.Join(ftsIDs, " OR ") + ")'" - rows, err = tx.Query(stmt) - if nil != err { - logging.LogErrorf("query blocks_fts_case_insensitive failed: %s", err) - return - } - rowIDs = nil - for rows.Next() { - var rowID int - if err = rows.Scan(&rowID); nil != err { - return - } - rowIDs = append(rowIDs, fmt.Sprintf("%d", rowID)) - } - rows.Close() - stmt = "DELETE FROM blocks_fts_case_insensitive WHERE rowid IN (" + strings.Join(rowIDs, ",") + ")" + stmt = "DELETE FROM blocks_fts_case_insensitive WHERE rowid IN (SELECT ROWID FROM blocks_fts_case_insensitive WHERE blocks_fts_case_insensitive MATCH 'id:(" + ftsIDsMatch + ")')" if err = execStmtTx(tx, stmt); nil != err { return } From 5e99a372424a5f92e2810ea59cbdb46783b4ed3e Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Wed, 10 Apr 2024 15:39:00 +0800 Subject: [PATCH 09/10] :zap: Reduce disk reads when editing documents https://github.com/siyuan-note/siyuan/issues/10961 --- kernel/sql/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sql/database.go b/kernel/sql/database.go index d709b2964..e29d2e633 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -971,13 +971,13 @@ func deleteBlocksByIDs(tx *sql.Tx, ids []string) (err error) { } ftsIDsMatch := strings.Join(ftsIDs, " OR ") - stmt = "DELETE FROM blocks_fts WHERE rowid IN (SELECT ROWID FROM blocks_fts WHERE blocks_fts MATCH 'id:(" + ftsIDsMatch + ")')" + stmt = "DELETE FROM blocks_fts WHERE ROWID IN (SELECT ROWID FROM blocks_fts WHERE blocks_fts MATCH 'id:(" + ftsIDsMatch + ")')" if err = execStmtTx(tx, stmt); nil != err { return } if !caseSensitive { - stmt = "DELETE FROM blocks_fts_case_insensitive WHERE rowid IN (SELECT ROWID FROM blocks_fts_case_insensitive WHERE blocks_fts_case_insensitive MATCH 'id:(" + ftsIDsMatch + ")')" + stmt = "DELETE FROM blocks_fts_case_insensitive WHERE ROWID IN (SELECT ROWID FROM blocks_fts_case_insensitive WHERE blocks_fts_case_insensitive MATCH 'id:(" + ftsIDsMatch + ")')" if err = execStmtTx(tx, stmt); nil != err { return } From 3f62aea1e9d4b796fc5744f3f33cbf2fd0dd063d Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Wed, 10 Apr 2024 15:57:18 +0800 Subject: [PATCH 10/10] :art: Import md progress --- kernel/model/import.go | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/model/import.go b/kernel/model/import.go index 938cf00e9..5552f3e9d 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -899,6 +899,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { util.PushEndlessProgress(fmt.Sprintf(Conf.Language(66), fmt.Sprintf("%d/%d ", i, len(importTrees))+tree.HPath)) } } + util.PushClearProgress() importTrees = []*parse.Tree{} searchLinks = map[string]string{}