From 7d05f4e8c7a2e52611d0aa0c7ad287b510370a6d Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 10:41:02 +0800 Subject: [PATCH 1/9] =?UTF-8?q?:recycle:=20=E6=90=9C=E7=B4=A2=E5=91=BD?= =?UTF-8?q?=E4=B8=AD=E9=AB=98=E4=BA=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/search.go | 12 ++++++------ kernel/search/mark.go | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/kernel/model/search.go b/kernel/model/search.go index ed1ff2b7e..8288f0da3 100644 --- a/kernel/model/search.go +++ b/kernel/model/search.go @@ -915,9 +915,9 @@ func stringQuery(query string) string { func markReplaceSpan(n *ast.Node, unlinks *[]*ast.Node, keywords []string, markSpanDataType string, luteEngine *lute.Lute) bool { text := n.Content() if ast.NodeText == n.Type { - text = search.EncloseHighlighting(text, keywords, getMarkSpanStart(markSpanDataType), getMarkSpanEnd(), Conf.Search.CaseSensitive) + text = search.EncloseHighlighting(text, keywords, search.GetMarkSpanStart(markSpanDataType), search.GetMarkSpanEnd(), Conf.Search.CaseSensitive) n.Tokens = gulu.Str.ToBytes(text) - if bytes.Contains(n.Tokens, []byte(searchMarkDataType)) { + if bytes.Contains(n.Tokens, []byte(search.MarkDataType)) { linkTree := parse.Inline("", n.Tokens, luteEngine.ParseOptions) var children []*ast.Node for c := linkTree.Root.FirstChild.FirstChild; nil != c; c = c.Next { @@ -935,10 +935,10 @@ func markReplaceSpan(n *ast.Node, unlinks *[]*ast.Node, keywords []string, markS return false } - startTag := getMarkSpanStart(markSpanDataType) - text = search.EncloseHighlighting(text, keywords, startTag, getMarkSpanEnd(), Conf.Search.CaseSensitive) - if strings.Contains(text, searchMarkDataType) { - dataType := getMarkSpanStart(n.TextMarkType + " " + searchMarkDataType) + startTag := search.GetMarkSpanStart(markSpanDataType) + text = search.EncloseHighlighting(text, keywords, startTag, search.GetMarkSpanEnd(), Conf.Search.CaseSensitive) + if strings.Contains(text, search.MarkDataType) { + dataType := search.GetMarkSpanStart(n.TextMarkType + " " + search.MarkDataType) text = strings.ReplaceAll(text, startTag, dataType) tokens := gulu.Str.ToBytes(text) linkTree := parse.Inline("", tokens, luteEngine.ParseOptions) diff --git a/kernel/search/mark.go b/kernel/search/mark.go index 027d1a36c..b6c35a122 100644 --- a/kernel/search/mark.go +++ b/kernel/search/mark.go @@ -17,6 +17,7 @@ package search import ( + "fmt" "regexp" "strings" "unicode/utf8" @@ -99,3 +100,16 @@ func EncloseHighlighting(text string, keywords []string, openMark, closeMark str } return } + +const ( + MarkDataType = "search-mark" + VirtualBlockRefDataType = "virtual-block-ref" +) + +func GetMarkSpanStart(dataType string) string { + return fmt.Sprintf("", dataType) +} + +func GetMarkSpanEnd() string { + return "" +} From e89f7068642cdc5ffbda4e3deb9f83ab56ee99ef Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 12:33:22 +0800 Subject: [PATCH 2/9] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E6=89=93?= =?UTF-8?q?=E5=BC=80=E8=99=9A=E6=8B=9F=E5=BC=95=E7=94=A8=E5=90=8E=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E6=96=87=E6=A1=A3=E7=9A=84=E6=80=A7=E8=83=BD=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/setting.go | 1 - kernel/model/backlink.go | 4 ++-- kernel/model/history.go | 2 +- kernel/sql/block_ref_query.go | 6 ------ kernel/sql/database.go | 12 ++++++------ 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/kernel/api/setting.go b/kernel/api/setting.go index 45c05fc16..13e870b2e 100644 --- a/kernel/api/setting.go +++ b/kernel/api/setting.go @@ -221,7 +221,6 @@ func setSearch(c *gin.Context) { if s.CaseSensitive != oldCaseSensitive { model.FullReindex() } - sql.ClearVirtualRefKeywords() ret.Data = s } diff --git a/kernel/model/backlink.go b/kernel/model/backlink.go index 9984a2545..f0eee898f 100644 --- a/kernel/model/backlink.go +++ b/kernel/model/backlink.go @@ -167,7 +167,7 @@ func buildBacklink(refID string, refTree *parse.Tree, keywords []string, luteEng return ast.WalkContinue } - markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine) + markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine) return ast.WalkContinue }) @@ -690,7 +690,7 @@ func searchBackmention(mentionKeywords []string, keyword string, excludeBacklink continue } - newText := markReplaceSpanWithSplit(text, mentionKeywords, getMarkSpanStart(searchMarkDataType), getMarkSpanEnd()) + newText := markReplaceSpanWithSplit(text, mentionKeywords, search.GetMarkSpanStart(search.MarkDataType), search.GetMarkSpanEnd()) if text != newText { tmp = append(tmp, b) } diff --git a/kernel/model/history.go b/kernel/model/history.go index b4a7ffcfb..8fefad3e4 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -180,7 +180,7 @@ func GetDocHistoryContent(historyPath, keyword string) (id, rootID, content stri n.RemoveIALAttr("fold") if 0 < len(keywords) { - if markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine) { + if markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine) { return ast.WalkContinue } } diff --git a/kernel/sql/block_ref_query.go b/kernel/sql/block_ref_query.go index b938e2049..93b066e0f 100644 --- a/kernel/sql/block_ref_query.go +++ b/kernel/sql/block_ref_query.go @@ -29,11 +29,6 @@ import ( ) func QueryVirtualRefKeywords(name, alias, anchor, doc bool) (ret []string) { - ret, ok := getVirtualRefKeywordsCache() - if ok { - return ret - } - if name { ret = append(ret, queryNames()...) } @@ -50,7 +45,6 @@ func QueryVirtualRefKeywords(name, alias, anchor, doc bool) (ret []string) { sort.SliceStable(ret, func(i, j int) bool { return len(ret[i]) >= len(ret[j]) }) - setVirtualRefKeywords(ret) return } diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 8ab68b15a..3b8dc0732 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -67,7 +67,7 @@ func InitDatabase(forceRebuild bool) (err error) { initDatabaseLock.Lock() defer initDatabaseLock.Unlock() - ClearBlockCache() + ClearCache() disableCache() defer enableCache() @@ -888,7 +888,7 @@ func deleteBlocksByBoxTx(tx *sql.Tx, box string) (err error) { if err = execStmtTx(tx, stmt, box); nil != err { return } - ClearBlockCache() + ClearCache() return } @@ -1012,7 +1012,7 @@ func deleteByRootID(tx *sql.Tx, rootID string, context map[string]interface{}) ( if err = execStmtTx(tx, stmt, rootID); nil != err { return } - ClearBlockCache() + ClearCache() eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, rootID) return } @@ -1048,7 +1048,7 @@ func batchDeleteByRootIDs(tx *sql.Tx, rootIDs []string, context map[string]inter if err = execStmtTx(tx, stmt); nil != err { return } - ClearBlockCache() + ClearCache() eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, fmt.Sprintf("%d", len(rootIDs))) return } @@ -1082,7 +1082,7 @@ func batchDeleteByPathPrefix(tx *sql.Tx, boxID, pathPrefix string) (err error) { if err = execStmtTx(tx, stmt, boxID, pathPrefix+"%"); nil != err { return } - ClearBlockCache() + ClearCache() return } @@ -1099,7 +1099,7 @@ func batchUpdateHPath(tx *sql.Tx, boxID, rootID, oldHPath, newHPath string) (err if err = execStmtTx(tx, stmt, newHPath, boxID, rootID, oldHPath); nil != err { return } - ClearBlockCache() + ClearCache() return } From 2f15ffffa56441840ff9062db6b43a3d04040359 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 12:39:43 +0800 Subject: [PATCH 3/9] :art: Clean code --- kernel/sql/cache.go | 66 +++++++++++++-------------------------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/kernel/sql/cache.go b/kernel/sql/cache.go index f5688a273..1ff183824 100644 --- a/kernel/sql/cache.go +++ b/kernel/sql/cache.go @@ -28,28 +28,29 @@ import ( "github.com/siyuan-note/logging" ) -var memCache, _ = ristretto.NewCache(&ristretto.Config{ +var cacheDisabled = true + +func enableCache() { + cacheDisabled = false +} + +func disableCache() { + cacheDisabled = true +} + +var blockCache, _ = ristretto.NewCache(&ristretto.Config{ NumCounters: 10240, MaxCost: 1024, BufferItems: 64, }) -var disabled = true -func enableCache() { - disabled = false -} - -func disableCache() { - disabled = true -} - -func ClearBlockCache() { - memCache.Clear() +func ClearCache() { + blockCache.Clear() debug.FreeOSMemory() } func putBlockCache(block *Block) { - if disabled { + if cacheDisabled { return } @@ -58,15 +59,15 @@ func putBlockCache(block *Block) { logging.LogErrorf("clone block failed: %v", err) return } - memCache.Set(cloned.ID, cloned, 1) + blockCache.Set(cloned.ID, cloned, 1) } func getBlockCache(id string) (ret *Block) { - if disabled { + if cacheDisabled { return } - b, _ := memCache.Get(id) + b, _ := blockCache.Get(id) if nil != b { ret = b.(*Block) } @@ -74,41 +75,10 @@ func getBlockCache(id string) (ret *Block) { } func removeBlockCache(id string) { - memCache.Del(id) + blockCache.Del(id) removeRefCacheByDefID(id) } -var virtualRefKeywordsCacheTime = time.Now() - -func getVirtualRefKeywordsCache() ([]string, bool) { - if disabled { - return nil, false - } - - // 虚拟引用关键字缓存调整为 10 分钟 https://github.com/siyuan-note/siyuan/issues/6602 - if 10 < time.Now().Sub(virtualRefKeywordsCacheTime).Minutes() { - ClearVirtualRefKeywords() - return nil, false - } - - if val, ok := memCache.Get("virtual_ref"); ok { - return val.([]string), true - } - return nil, false -} - -func setVirtualRefKeywords(keywords []string) { - if disabled { - return - } - - memCache.Set("virtual_ref", keywords, 1) -} - -func ClearVirtualRefKeywords() { - memCache.Del("virtual_ref") -} - var defIDRefsCache = gcache.New(30*time.Minute, 5*time.Minute) // [defBlockID]map[refBlockID]*Ref func GetRefsCacheByDefID(defID string) (ret []*Ref) { From f07ee114d1a1a93c0241cff40c45a361246045dc Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 13:14:15 +0800 Subject: [PATCH 4/9] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E6=89=93?= =?UTF-8?q?=E5=BC=80=E8=99=9A=E6=8B=9F=E5=BC=95=E7=94=A8=E5=90=8E=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E6=96=87=E6=A1=A3=E7=9A=84=E6=80=A7=E8=83=BD=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/job/cron.go | 1 + kernel/model/file.go | 18 +--- kernel/model/virutalref.go | 181 +++++++++++++++++++++++++++++++++++-- 3 files changed, 177 insertions(+), 23 deletions(-) diff --git a/kernel/job/cron.go b/kernel/job/cron.go index ffb98438d..1a7b6c892 100644 --- a/kernel/job/cron.go +++ b/kernel/job/cron.go @@ -39,6 +39,7 @@ func StartCron() { go every(util.SQLFlushInterval, sql.FlushTxJob) go every(10*time.Minute, model.FixIndexJob) go every(10*time.Minute, model.IndexEmbedBlockJob) + go every(10*time.Minute, model.CacheVirtualBlockRefJob) go every(12*time.Second, model.OCRAssetsJob) go every(12*time.Second, model.FlushAssetsTextsJob) go every(30*time.Second, model.HookDesktopUIProcJob) diff --git a/kernel/model/file.go b/kernel/model/file.go index bb7521dc2..ece4df246 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -440,19 +440,6 @@ func StatTree(id string) (ret *util.BlockStatResult) { } } -const ( - searchMarkDataType = "search-mark" - virtualBlockRefDataType = "virtual-block-ref" -) - -func getMarkSpanStart(dataType string) string { - return fmt.Sprintf("", dataType) -} - -func getMarkSpanEnd() string { - return "" -} - func GetDoc(startID, endID, id string, index int, keyword string, mode int, size int, isBacklink bool) (blockCount, childBlockCount int, dom, parentID, parent2ID, rootID, typ string, eof bool, boxID, docPath string, isBacklinkExpand bool, err error) { //os.MkdirAll("pprof", 0755) //cpuProfile, _ := os.Create("pprof/GetDoc") @@ -618,7 +605,8 @@ func GetDoc(startID, endID, id string, index int, keyword string, mode int, size } refCount := sql.QueryRootChildrenRefCount(rootID) - virtualBlockRefKeywords := getVirtualRefKeywords(tree.Root.IALAttr("title")) + //virtualBlockRefKeywords := getVirtualRefKeywords(tree.Root.IALAttr("title")) + virtualBlockRefKeywords := getBlockVirtualRefKeywords(tree.Root) subTree := &parse.Tree{ID: rootID, Root: &ast.Node{Type: ast.NodeDocument}, Marks: tree.Marks} keyword = strings.Join(strings.Split(keyword, " "), search.TermSep) @@ -659,7 +647,7 @@ func GetDoc(startID, endID, id string, index int, keyword string, mode int, size } } if hitBlock { - if markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine) { + if markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine) { return ast.WalkContinue } } diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 6e3a1e2e5..fe9f97262 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -17,20 +17,171 @@ package model import ( - "regexp" - "sort" - "strings" - + "bytes" "github.com/88250/gulu" "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" + "github.com/dgraph-io/ristretto" + "github.com/panjf2000/ants/v2" + "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/filesys" + "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" + "github.com/siyuan-note/siyuan/kernel/util" + "os" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "sync" ) +// virtualBlockRefCache 用于保存块关联的虚拟引用关键字。 +// 改进打开虚拟引用后加载文档的性能 https://github.com/siyuan-note/siyuan/issues/7378 +var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{ + NumCounters: 1024000, + MaxCost: 102400, + BufferItems: 64, +}) + +func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) { + val, ok := virtualBlockRefCache.Get(root.ID) + if !ok { + treeTitle := root.IALAttr("title") + buf := bytes.Buffer{} + ast.Walk(root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering || !n.IsBlock() { + return ast.WalkContinue + } + + content := treenode.NodeStaticContent(n, nil) + buf.WriteString(content) + return ast.WalkContinue + }) + content := buf.String() + putBlockVirtualRefKeywords(content, root.ID, treeTitle) + val, ok = virtualBlockRefCache.Get(root.ID) + if !ok { + return + } + } + ret = val.([]string) + return +} + +func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) { + keywords := getVirtualRefKeywords(docTitle) + if 1 > len(keywords) { + return + } + + var hitKeywords []string + contentTmp := blockContent + if !Conf.Search.CaseSensitive { + contentTmp = strings.ToLower(blockContent) + } + for _, keyword := range keywords { + keywordTmp := keyword + if !Conf.Search.CaseSensitive { + keywordTmp = strings.ToLower(keyword) + } + + if strings.Contains(contentTmp, keywordTmp) { + hitKeywords = append(hitKeywords, keyword) + } + } + + if 1 > len(hitKeywords) { + return + } + + hitKeywords = gulu.Str.RemoveDuplicatedElem(hitKeywords) + virtualBlockRefCache.Set(blockID, hitKeywords, 1) +} + +func CacheVirtualBlockRefJob() { + virtualBlockRefCache.Del("virtual_ref") + + if !Conf.Editor.VirtualBlockRef { + return + } + + keywords := sql.QueryVirtualRefKeywords(Conf.Search.VirtualRefName, Conf.Search.VirtualRefAlias, Conf.Search.VirtualRefAnchor, Conf.Search.VirtualRefDoc) + virtualBlockRefCache.Set("virtual_ref", keywords, 1) + + boxes := Conf.GetOpenedBoxes() + luteEngine := lute.New() + for _, box := range boxes { + boxPath := filepath.Join(util.DataDir, box.ID) + var paths []string + filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error { + if boxPath == path { + // 跳过根路径（笔记本文件夹） + return nil + } + + if info.IsDir() { + if strings.HasPrefix(info.Name(), ".") { + return filepath.SkipDir + } + return nil + } + + if filepath.Ext(path) != ".sy" || strings.Contains(filepath.ToSlash(path), "/assets/") { + return nil + } + + p := path[len(boxPath):] + p = filepath.ToSlash(p) + paths = append(paths, p) + return nil + }) + + poolSize := runtime.NumCPU() + if 4 < poolSize { + poolSize = 4 + } + i := 0 + waitGroup := &sync.WaitGroup{} + pool, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) { + defer waitGroup.Done() + + p := arg.(string) + tree, loadErr := filesys.LoadTree(box.ID, p, luteEngine) + if nil != loadErr { + return + } + + treeTitle := tree.Root.IALAttr("title") + buf := bytes.Buffer{} + ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering || !n.IsBlock() { + return ast.WalkContinue + } + + content := treenode.NodeStaticContent(n, nil) + buf.WriteString(content) + return ast.WalkContinue + }) + content := buf.String() + putBlockVirtualRefKeywords(content, tree.ID, treeTitle) + i++ + logging.LogInfof("cached virtual block ref for tree [%s, %d/%d]", tree.ID, i, len(paths)) + }) + for _, p := range paths { + waitGroup.Add(1) + pool.Invoke(p) + } + waitGroup.Wait() + pool.Release() + } +} + func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool { - if !Conf.Editor.VirtualBlockRef || 1 > len(virtualBlockRefKeywords) { + if !Conf.Editor.VirtualBlockRef { return false } @@ -43,8 +194,18 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword return false } + if 1 > len(virtualBlockRefKeywords) { + return false + } + content := string(n.Tokens) - newContent := markReplaceSpanWithSplit(content, virtualBlockRefKeywords, getMarkSpanStart(virtualBlockRefDataType), getMarkSpanEnd()) + tmp := gulu.Str.RemoveInvisible(content) + tmp = strings.TrimSpace(tmp) + if "" == tmp { + return false + } + + newContent := markReplaceSpanWithSplit(content, virtualBlockRefKeywords, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd()) if content != newContent { // 虚拟引用排除命中自身块命名和别名的情况 https://github.com/siyuan-note/siyuan/issues/3185 var blockKeys []string @@ -55,7 +216,7 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword blockKeys = append(blockKeys, alias) } if 0 < len(blockKeys) { - keys := gulu.Str.SubstringsBetween(newContent, getMarkSpanStart(virtualBlockRefDataType), getMarkSpanEnd()) + keys := gulu.Str.SubstringsBetween(newContent, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd()) for _, k := range keys { if gulu.Str.Contains(k, blockKeys) { return true @@ -83,7 +244,10 @@ func getVirtualRefKeywords(docName string) (ret []string) { return } - ret = sql.QueryVirtualRefKeywords(Conf.Search.VirtualRefName, Conf.Search.VirtualRefAlias, Conf.Search.VirtualRefAnchor, Conf.Search.VirtualRefDoc) + if val, ok := virtualBlockRefCache.Get("virtual_ref"); ok { + ret = val.([]string) + } + if "" != strings.TrimSpace(Conf.Editor.VirtualBlockRefInclude) { include := strings.ReplaceAll(Conf.Editor.VirtualBlockRefInclude, "\\,", "__comma@sep__") includes := strings.Split(include, ",") @@ -129,6 +293,7 @@ func getVirtualRefKeywords(docName string) (ret []string) { ret = gulu.Str.ExcludeElem(ret, []string{docName}) ret = prepareMarkKeywords(ret) + // 在 设置 - 搜索 中分别增加虚拟引用和反链提及 `关键字数量限制` https://github.com/siyuan-note/siyuan/issues/6603 if Conf.Search.VirtualRefKeywordsLimit < len(ret) { ret = ret[:Conf.Search.VirtualRefKeywordsLimit] } From c1b457eb76a98899698756b2557b2491381b80fe Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 13:34:27 +0800 Subject: [PATCH 5/9] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E6=89=93?= =?UTF-8?q?=E5=BC=80=E8=99=9A=E6=8B=9F=E5=BC=95=E7=94=A8=E5=90=8E=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E6=96=87=E6=A1=A3=E7=9A=84=E6=80=A7=E8=83=BD=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/file.go | 1 - kernel/model/virutalref.go | 22 +++------------------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/kernel/model/file.go b/kernel/model/file.go index ece4df246..7bddac1c5 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -605,7 +605,6 @@ func GetDoc(startID, endID, id string, index int, keyword string, mode int, size } refCount := sql.QueryRootChildrenRefCount(rootID) - //virtualBlockRefKeywords := getVirtualRefKeywords(tree.Root.IALAttr("title")) virtualBlockRefKeywords := getBlockVirtualRefKeywords(tree.Root) subTree := &parse.Tree{ID: rootID, Root: &ast.Node{Type: ast.NodeDocument}, Marks: tree.Marks} diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index fe9f97262..510f2bab8 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -50,23 +50,7 @@ var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{ func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) { val, ok := virtualBlockRefCache.Get(root.ID) if !ok { - treeTitle := root.IALAttr("title") - buf := bytes.Buffer{} - ast.Walk(root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering || !n.IsBlock() { - return ast.WalkContinue - } - - content := treenode.NodeStaticContent(n, nil) - buf.WriteString(content) - return ast.WalkContinue - }) - content := buf.String() - putBlockVirtualRefKeywords(content, root.ID, treeTitle) - val, ok = virtualBlockRefCache.Get(root.ID) - if !ok { - return - } + return } ret = val.([]string) return @@ -141,8 +125,8 @@ func CacheVirtualBlockRefJob() { }) poolSize := runtime.NumCPU() - if 4 < poolSize { - poolSize = 4 + if 8 < poolSize { + poolSize = 8 } i := 0 waitGroup := &sync.WaitGroup{} From ef029d4dea33ae239235f321b4823e812a3110c1 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 13:34:47 +0800 Subject: [PATCH 6/9] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E6=89=93?= =?UTF-8?q?=E5=BC=80=E8=99=9A=E6=8B=9F=E5=BC=95=E7=94=A8=E5=90=8E=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E6=96=87=E6=A1=A3=E7=9A=84=E6=80=A7=E8=83=BD=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/virutalref.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 510f2bab8..9ab265885 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -48,8 +48,8 @@ var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{ }) func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) { - val, ok := virtualBlockRefCache.Get(root.ID) - if !ok { + val, _ := virtualBlockRefCache.Get(root.ID) + if nil == val { return } ret = val.([]string) From 0eabe9bf5f82b3e093b25b981ae798127ad360c8 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 13:41:34 +0800 Subject: [PATCH 7/9] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E6=89=93?= =?UTF-8?q?=E5=BC=80=E8=99=9A=E6=8B=9F=E5=BC=95=E7=94=A8=E5=90=8E=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E6=96=87=E6=A1=A3=E7=9A=84=E6=80=A7=E8=83=BD=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/virutalref.go | 99 +++++++------------------------------- 1 file changed, 18 insertions(+), 81 deletions(-) diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 9ab265885..e51858ce1 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -18,25 +18,18 @@ package model import ( "bytes" + "regexp" + "sort" + "strings" + "github.com/88250/gulu" "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" "github.com/dgraph-io/ristretto" - "github.com/panjf2000/ants/v2" - "github.com/siyuan-note/logging" - "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" - "github.com/siyuan-note/siyuan/kernel/util" - "os" - "path/filepath" - "regexp" - "runtime" - "sort" - "strings" - "sync" ) // virtualBlockRefCache 用于保存块关联的虚拟引用关键字。 @@ -48,8 +41,20 @@ var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{ }) func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) { - val, _ := virtualBlockRefCache.Get(root.ID) - if nil == val { + val, ok := virtualBlockRefCache.Get(root.ID) + if !ok { + buf := bytes.Buffer{} + ast.Walk(root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering || !n.IsBlock() { + return ast.WalkContinue + } + + content := treenode.NodeStaticContent(n, nil) + buf.WriteString(content) + return ast.WalkContinue + }) + content := buf.String() + putBlockVirtualRefKeywords(content, root.ID, root.IALAttr("title")) return } ret = val.([]string) @@ -88,80 +93,12 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) { func CacheVirtualBlockRefJob() { virtualBlockRefCache.Del("virtual_ref") - if !Conf.Editor.VirtualBlockRef { return } keywords := sql.QueryVirtualRefKeywords(Conf.Search.VirtualRefName, Conf.Search.VirtualRefAlias, Conf.Search.VirtualRefAnchor, Conf.Search.VirtualRefDoc) virtualBlockRefCache.Set("virtual_ref", keywords, 1) - - boxes := Conf.GetOpenedBoxes() - luteEngine := lute.New() - for _, box := range boxes { - boxPath := filepath.Join(util.DataDir, box.ID) - var paths []string - filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error { - if boxPath == path { - // 跳过根路径（笔记本文件夹） - return nil - } - - if info.IsDir() { - if strings.HasPrefix(info.Name(), ".") { - return filepath.SkipDir - } - return nil - } - - if filepath.Ext(path) != ".sy" || strings.Contains(filepath.ToSlash(path), "/assets/") { - return nil - } - - p := path[len(boxPath):] - p = filepath.ToSlash(p) - paths = append(paths, p) - return nil - }) - - poolSize := runtime.NumCPU() - if 8 < poolSize { - poolSize = 8 - } - i := 0 - waitGroup := &sync.WaitGroup{} - pool, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) { - defer waitGroup.Done() - - p := arg.(string) - tree, loadErr := filesys.LoadTree(box.ID, p, luteEngine) - if nil != loadErr { - return - } - - treeTitle := tree.Root.IALAttr("title") - buf := bytes.Buffer{} - ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering || !n.IsBlock() { - return ast.WalkContinue - } - - content := treenode.NodeStaticContent(n, nil) - buf.WriteString(content) - return ast.WalkContinue - }) - content := buf.String() - putBlockVirtualRefKeywords(content, tree.ID, treeTitle) - i++ - logging.LogInfof("cached virtual block ref for tree [%s, %d/%d]", tree.ID, i, len(paths)) - }) - for _, p := range paths { - waitGroup.Add(1) - pool.Invoke(p) - } - waitGroup.Wait() - pool.Release() - } } func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool { From b75a46a228c4d7bf73f3c7b867b30d720eb8d9e0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 13:43:53 +0800 Subject: [PATCH 8/9] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E6=89=93?= =?UTF-8?q?=E5=BC=80=E8=99=9A=E6=8B=9F=E5=BC=95=E7=94=A8=E5=90=8E=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E6=96=87=E6=A1=A3=E7=9A=84=E6=80=A7=E8=83=BD=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/virutalref.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index e51858ce1..4e5cb6f40 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -54,20 +54,19 @@ func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) { return ast.WalkContinue }) content := buf.String() - putBlockVirtualRefKeywords(content, root.ID, root.IALAttr("title")) + ret = putBlockVirtualRefKeywords(content, root.ID, root.IALAttr("title")) return } ret = val.([]string) return } -func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) { +func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []string) { keywords := getVirtualRefKeywords(docTitle) if 1 > len(keywords) { return } - var hitKeywords []string contentTmp := blockContent if !Conf.Search.CaseSensitive { contentTmp = strings.ToLower(blockContent) @@ -79,16 +78,17 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) { } if strings.Contains(contentTmp, keywordTmp) { - hitKeywords = append(hitKeywords, keyword) + ret = append(ret, keyword) } } - if 1 > len(hitKeywords) { + if 1 > len(ret) { return } - hitKeywords = gulu.Str.RemoveDuplicatedElem(hitKeywords) - virtualBlockRefCache.Set(blockID, hitKeywords, 1) + ret = gulu.Str.RemoveDuplicatedElem(ret) + virtualBlockRefCache.Set(blockID, ret, 1) + return } func CacheVirtualBlockRefJob() { From 52faf594eb3f0854f46e3d219c364f11ee744009 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 13:44:08 +0800 Subject: [PATCH 9/9] =?UTF-8?q?:zap:=20=E6=94=B9=E8=BF=9B=E6=89=93?= =?UTF-8?q?=E5=BC=80=E8=99=9A=E6=8B=9F=E5=BC=95=E7=94=A8=E5=90=8E=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD=E6=96=87=E6=A1=A3=E7=9A=84=E6=80=A7=E8=83=BD=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/virutalref.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 4e5cb6f40..c70ee8ccb 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -35,8 +35,8 @@ import ( // virtualBlockRefCache 用于保存块关联的虚拟引用关键字。 // 改进打开虚拟引用后加载文档的性能 https://github.com/siyuan-note/siyuan/issues/7378 var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{ - NumCounters: 1024000, - MaxCost: 102400, + NumCounters: 102400, + MaxCost: 10240, BufferItems: 64, })