diff --git a/kernel/api/setting.go b/kernel/api/setting.go index 45c05fc16..13e870b2e 100644 --- a/kernel/api/setting.go +++ b/kernel/api/setting.go @@ -221,7 +221,6 @@ func setSearch(c *gin.Context) { if s.CaseSensitive != oldCaseSensitive { model.FullReindex() } - sql.ClearVirtualRefKeywords() ret.Data = s } diff --git a/kernel/job/cron.go b/kernel/job/cron.go index ffb98438d..1a7b6c892 100644 --- a/kernel/job/cron.go +++ b/kernel/job/cron.go @@ -39,6 +39,7 @@ func StartCron() { go every(util.SQLFlushInterval, sql.FlushTxJob) go every(10*time.Minute, model.FixIndexJob) go every(10*time.Minute, model.IndexEmbedBlockJob) + go every(10*time.Minute, model.CacheVirtualBlockRefJob) go every(12*time.Second, model.OCRAssetsJob) go every(12*time.Second, model.FlushAssetsTextsJob) go every(30*time.Second, model.HookDesktopUIProcJob) diff --git a/kernel/model/backlink.go b/kernel/model/backlink.go index 9984a2545..f0eee898f 100644 --- a/kernel/model/backlink.go +++ b/kernel/model/backlink.go @@ -167,7 +167,7 @@ func buildBacklink(refID string, refTree *parse.Tree, keywords []string, luteEng return ast.WalkContinue } - markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine) + markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine) return ast.WalkContinue }) @@ -690,7 +690,7 @@ func searchBackmention(mentionKeywords []string, keyword string, excludeBacklink continue } - newText := markReplaceSpanWithSplit(text, mentionKeywords, getMarkSpanStart(searchMarkDataType), getMarkSpanEnd()) + newText := markReplaceSpanWithSplit(text, mentionKeywords, search.GetMarkSpanStart(search.MarkDataType), search.GetMarkSpanEnd()) if text != newText { tmp = append(tmp, b) } diff --git a/kernel/model/file.go b/kernel/model/file.go index bb7521dc2..7bddac1c5 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -440,19 +440,6 @@ func StatTree(id string) (ret *util.BlockStatResult) { } } -const ( - searchMarkDataType = "search-mark" - virtualBlockRefDataType = "virtual-block-ref" -) - -func getMarkSpanStart(dataType string) string { - return fmt.Sprintf("", dataType) -} - -func getMarkSpanEnd() string { - return "" -} - func GetDoc(startID, endID, id string, index int, keyword string, mode int, size int, isBacklink bool) (blockCount, childBlockCount int, dom, parentID, parent2ID, rootID, typ string, eof bool, boxID, docPath string, isBacklinkExpand bool, err error) { //os.MkdirAll("pprof", 0755) //cpuProfile, _ := os.Create("pprof/GetDoc") @@ -618,7 +605,7 @@ func GetDoc(startID, endID, id string, index int, keyword string, mode int, size } refCount := sql.QueryRootChildrenRefCount(rootID) - virtualBlockRefKeywords := getVirtualRefKeywords(tree.Root.IALAttr("title")) + virtualBlockRefKeywords := getBlockVirtualRefKeywords(tree.Root) subTree := &parse.Tree{ID: rootID, Root: &ast.Node{Type: ast.NodeDocument}, Marks: tree.Marks} keyword = strings.Join(strings.Split(keyword, " "), search.TermSep) @@ -659,7 +646,7 @@ func GetDoc(startID, endID, id string, index int, keyword string, mode int, size } } if hitBlock { - if markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine) { + if markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine) { return ast.WalkContinue } } diff --git a/kernel/model/history.go b/kernel/model/history.go index b4a7ffcfb..8fefad3e4 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -180,7 +180,7 @@ func GetDocHistoryContent(historyPath, keyword string) (id, rootID, content stri n.RemoveIALAttr("fold") if 0 < len(keywords) { - if markReplaceSpan(n, &unlinks, keywords, searchMarkDataType, luteEngine) { + if markReplaceSpan(n, &unlinks, keywords, search.MarkDataType, luteEngine) { return ast.WalkContinue } } diff --git a/kernel/model/search.go b/kernel/model/search.go index ed1ff2b7e..8288f0da3 100644 --- a/kernel/model/search.go +++ b/kernel/model/search.go @@ -915,9 +915,9 @@ func stringQuery(query string) string { func markReplaceSpan(n *ast.Node, unlinks *[]*ast.Node, keywords []string, markSpanDataType string, luteEngine *lute.Lute) bool { text := n.Content() if ast.NodeText == n.Type { - text = search.EncloseHighlighting(text, keywords, getMarkSpanStart(markSpanDataType), getMarkSpanEnd(), Conf.Search.CaseSensitive) + text = search.EncloseHighlighting(text, keywords, search.GetMarkSpanStart(markSpanDataType), search.GetMarkSpanEnd(), Conf.Search.CaseSensitive) n.Tokens = gulu.Str.ToBytes(text) - if bytes.Contains(n.Tokens, []byte(searchMarkDataType)) { + if bytes.Contains(n.Tokens, []byte(search.MarkDataType)) { linkTree := parse.Inline("", n.Tokens, luteEngine.ParseOptions) var children []*ast.Node for c := linkTree.Root.FirstChild.FirstChild; nil != c; c = c.Next { @@ -935,10 +935,10 @@ func markReplaceSpan(n *ast.Node, unlinks *[]*ast.Node, keywords []string, markS return false } - startTag := getMarkSpanStart(markSpanDataType) - text = search.EncloseHighlighting(text, keywords, startTag, getMarkSpanEnd(), Conf.Search.CaseSensitive) - if strings.Contains(text, searchMarkDataType) { - dataType := getMarkSpanStart(n.TextMarkType + " " + searchMarkDataType) + startTag := search.GetMarkSpanStart(markSpanDataType) + text = search.EncloseHighlighting(text, keywords, startTag, search.GetMarkSpanEnd(), Conf.Search.CaseSensitive) + if strings.Contains(text, search.MarkDataType) { + dataType := search.GetMarkSpanStart(n.TextMarkType + " " + search.MarkDataType) text = strings.ReplaceAll(text, startTag, dataType) tokens := gulu.Str.ToBytes(text) linkTree := parse.Inline("", tokens, luteEngine.ParseOptions) diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 6e3a1e2e5..c70ee8ccb 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -17,6 +17,7 @@ package model import ( + "bytes" "regexp" "sort" "strings" @@ -25,12 +26,83 @@ import ( "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" + "github.com/dgraph-io/ristretto" + "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" ) +// virtualBlockRefCache 用于保存块关联的虚拟引用关键字。 +// 改进打开虚拟引用后加载文档的性能 https://github.com/siyuan-note/siyuan/issues/7378 +var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{ + NumCounters: 102400, + MaxCost: 10240, + BufferItems: 64, +}) + +func getBlockVirtualRefKeywords(root *ast.Node) (ret []string) { + val, ok := virtualBlockRefCache.Get(root.ID) + if !ok { + buf := bytes.Buffer{} + ast.Walk(root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering || !n.IsBlock() { + return ast.WalkContinue + } + + content := treenode.NodeStaticContent(n, nil) + buf.WriteString(content) + return ast.WalkContinue + }) + content := buf.String() + ret = putBlockVirtualRefKeywords(content, root.ID, root.IALAttr("title")) + return + } + ret = val.([]string) + return +} + +func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []string) { + keywords := getVirtualRefKeywords(docTitle) + if 1 > len(keywords) { + return + } + + contentTmp := blockContent + if !Conf.Search.CaseSensitive { + contentTmp = strings.ToLower(blockContent) + } + for _, keyword := range keywords { + keywordTmp := keyword + if !Conf.Search.CaseSensitive { + keywordTmp = strings.ToLower(keyword) + } + + if strings.Contains(contentTmp, keywordTmp) { + ret = append(ret, keyword) + } + } + + if 1 > len(ret) { + return + } + + ret = gulu.Str.RemoveDuplicatedElem(ret) + virtualBlockRefCache.Set(blockID, ret, 1) + return +} + +func CacheVirtualBlockRefJob() { + virtualBlockRefCache.Del("virtual_ref") + if !Conf.Editor.VirtualBlockRef { + return + } + + keywords := sql.QueryVirtualRefKeywords(Conf.Search.VirtualRefName, Conf.Search.VirtualRefAlias, Conf.Search.VirtualRefAnchor, Conf.Search.VirtualRefDoc) + virtualBlockRefCache.Set("virtual_ref", keywords, 1) +} + func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool { - if !Conf.Editor.VirtualBlockRef || 1 > len(virtualBlockRefKeywords) { + if !Conf.Editor.VirtualBlockRef { return false } @@ -43,8 +115,18 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword return false } + if 1 > len(virtualBlockRefKeywords) { + return false + } + content := string(n.Tokens) - newContent := markReplaceSpanWithSplit(content, virtualBlockRefKeywords, getMarkSpanStart(virtualBlockRefDataType), getMarkSpanEnd()) + tmp := gulu.Str.RemoveInvisible(content) + tmp = strings.TrimSpace(tmp) + if "" == tmp { + return false + } + + newContent := markReplaceSpanWithSplit(content, virtualBlockRefKeywords, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd()) if content != newContent { // 虚拟引用排除命中自身块命名和别名的情况 https://github.com/siyuan-note/siyuan/issues/3185 var blockKeys []string @@ -55,7 +137,7 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword blockKeys = append(blockKeys, alias) } if 0 < len(blockKeys) { - keys := gulu.Str.SubstringsBetween(newContent, getMarkSpanStart(virtualBlockRefDataType), getMarkSpanEnd()) + keys := gulu.Str.SubstringsBetween(newContent, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd()) for _, k := range keys { if gulu.Str.Contains(k, blockKeys) { return true @@ -83,7 +165,10 @@ func getVirtualRefKeywords(docName string) (ret []string) { return } - ret = sql.QueryVirtualRefKeywords(Conf.Search.VirtualRefName, Conf.Search.VirtualRefAlias, Conf.Search.VirtualRefAnchor, Conf.Search.VirtualRefDoc) + if val, ok := virtualBlockRefCache.Get("virtual_ref"); ok { + ret = val.([]string) + } + if "" != strings.TrimSpace(Conf.Editor.VirtualBlockRefInclude) { include := strings.ReplaceAll(Conf.Editor.VirtualBlockRefInclude, "\\,", "__comma@sep__") includes := strings.Split(include, ",") @@ -129,6 +214,7 @@ func getVirtualRefKeywords(docName string) (ret []string) { ret = gulu.Str.ExcludeElem(ret, []string{docName}) ret = prepareMarkKeywords(ret) + // 在 设置 - 搜索 中分别增加虚拟引用和反链提及 `关键字数量限制` https://github.com/siyuan-note/siyuan/issues/6603 if Conf.Search.VirtualRefKeywordsLimit < len(ret) { ret = ret[:Conf.Search.VirtualRefKeywordsLimit] } diff --git a/kernel/search/mark.go b/kernel/search/mark.go index 027d1a36c..b6c35a122 100644 --- a/kernel/search/mark.go +++ b/kernel/search/mark.go @@ -17,6 +17,7 @@ package search import ( + "fmt" "regexp" "strings" "unicode/utf8" @@ -99,3 +100,16 @@ func EncloseHighlighting(text string, keywords []string, openMark, closeMark str } return } + +const ( + MarkDataType = "search-mark" + VirtualBlockRefDataType = "virtual-block-ref" +) + +func GetMarkSpanStart(dataType string) string { + return fmt.Sprintf("", dataType) +} + +func GetMarkSpanEnd() string { + return "" +} diff --git a/kernel/sql/block_ref_query.go b/kernel/sql/block_ref_query.go index b938e2049..93b066e0f 100644 --- a/kernel/sql/block_ref_query.go +++ b/kernel/sql/block_ref_query.go @@ -29,11 +29,6 @@ import ( ) func QueryVirtualRefKeywords(name, alias, anchor, doc bool) (ret []string) { - ret, ok := getVirtualRefKeywordsCache() - if ok { - return ret - } - if name { ret = append(ret, queryNames()...) } @@ -50,7 +45,6 @@ func QueryVirtualRefKeywords(name, alias, anchor, doc bool) (ret []string) { sort.SliceStable(ret, func(i, j int) bool { return len(ret[i]) >= len(ret[j]) }) - setVirtualRefKeywords(ret) return } diff --git a/kernel/sql/cache.go b/kernel/sql/cache.go index f5688a273..1ff183824 100644 --- a/kernel/sql/cache.go +++ b/kernel/sql/cache.go @@ -28,28 +28,29 @@ import ( "github.com/siyuan-note/logging" ) -var memCache, _ = ristretto.NewCache(&ristretto.Config{ +var cacheDisabled = true + +func enableCache() { + cacheDisabled = false +} + +func disableCache() { + cacheDisabled = true +} + +var blockCache, _ = ristretto.NewCache(&ristretto.Config{ NumCounters: 10240, MaxCost: 1024, BufferItems: 64, }) -var disabled = true -func enableCache() { - disabled = false -} - -func disableCache() { - disabled = true -} - -func ClearBlockCache() { - memCache.Clear() +func ClearCache() { + blockCache.Clear() debug.FreeOSMemory() } func putBlockCache(block *Block) { - if disabled { + if cacheDisabled { return } @@ -58,15 +59,15 @@ func putBlockCache(block *Block) { logging.LogErrorf("clone block failed: %v", err) return } - memCache.Set(cloned.ID, cloned, 1) + blockCache.Set(cloned.ID, cloned, 1) } func getBlockCache(id string) (ret *Block) { - if disabled { + if cacheDisabled { return } - b, _ := memCache.Get(id) + b, _ := blockCache.Get(id) if nil != b { ret = b.(*Block) } @@ -74,41 +75,10 @@ func getBlockCache(id string) (ret *Block) { } func removeBlockCache(id string) { - memCache.Del(id) + blockCache.Del(id) removeRefCacheByDefID(id) } -var virtualRefKeywordsCacheTime = time.Now() - -func getVirtualRefKeywordsCache() ([]string, bool) { - if disabled { - return nil, false - } - - // 虚拟引用关键字缓存调整为 10 分钟 https://github.com/siyuan-note/siyuan/issues/6602 - if 10 < time.Now().Sub(virtualRefKeywordsCacheTime).Minutes() { - ClearVirtualRefKeywords() - return nil, false - } - - if val, ok := memCache.Get("virtual_ref"); ok { - return val.([]string), true - } - return nil, false -} - -func setVirtualRefKeywords(keywords []string) { - if disabled { - return - } - - memCache.Set("virtual_ref", keywords, 1) -} - -func ClearVirtualRefKeywords() { - memCache.Del("virtual_ref") -} - var defIDRefsCache = gcache.New(30*time.Minute, 5*time.Minute) // [defBlockID]map[refBlockID]*Ref func GetRefsCacheByDefID(defID string) (ret []*Ref) { diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 8ab68b15a..3b8dc0732 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -67,7 +67,7 @@ func InitDatabase(forceRebuild bool) (err error) { initDatabaseLock.Lock() defer initDatabaseLock.Unlock() - ClearBlockCache() + ClearCache() disableCache() defer enableCache() @@ -888,7 +888,7 @@ func deleteBlocksByBoxTx(tx *sql.Tx, box string) (err error) { if err = execStmtTx(tx, stmt, box); nil != err { return } - ClearBlockCache() + ClearCache() return } @@ -1012,7 +1012,7 @@ func deleteByRootID(tx *sql.Tx, rootID string, context map[string]interface{}) ( if err = execStmtTx(tx, stmt, rootID); nil != err { return } - ClearBlockCache() + ClearCache() eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, rootID) return } @@ -1048,7 +1048,7 @@ func batchDeleteByRootIDs(tx *sql.Tx, rootIDs []string, context map[string]inter if err = execStmtTx(tx, stmt); nil != err { return } - ClearBlockCache() + ClearCache() eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, fmt.Sprintf("%d", len(rootIDs))) return } @@ -1082,7 +1082,7 @@ func batchDeleteByPathPrefix(tx *sql.Tx, boxID, pathPrefix string) (err error) { if err = execStmtTx(tx, stmt, boxID, pathPrefix+"%"); nil != err { return } - ClearBlockCache() + ClearCache() return } @@ -1099,7 +1099,7 @@ func batchUpdateHPath(tx *sql.Tx, boxID, rootID, oldHPath, newHPath string) (err if err = execStmtTx(tx, stmt, newHPath, boxID, rootID, oldHPath); nil != err { return } - ClearBlockCache() + ClearCache() return }