Merge remote-tracking branch 'origin/dev' into dev

kernel/av/av.go

@@ -431,7 +431,7 @@ func SaveAttributeView(av *AttributeView) (err error) {
 			}
 
 			if 0 == v.UpdatedAt {
-				v.UpdatedAt = v.CreatedAt + 1000
+				v.UpdatedAt = v.CreatedAt
 			}
 		}
 	}

kernel/av/mirror.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/88250/gulu"
+	"github.com/88250/lute/ast"
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/util"
@@ -16,6 +17,30 @@ var (
 	AttributeViewBlocksLock = sync.Mutex{}
 )
 
+func GetBlockRels() (ret map[string][]string) {
+	AttributeViewBlocksLock.Lock()
+	defer AttributeViewBlocksLock.Unlock()
+
+	ret = map[string][]string{}
+
+	blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack")
+	if !filelock.IsExist(blocks) {
+		return
+	}
+
+	data, err := filelock.ReadFile(blocks)
+	if nil != err {
+		logging.LogErrorf("read attribute view blocks failed: %s", err)
+		return
+	}
+
+	if err = msgpack.Unmarshal(data, &ret); nil != err {
+		logging.LogErrorf("unmarshal attribute view blocks failed: %s", err)
+		return
+	}
+	return
+}
+
 func IsMirror(avID string) bool {
 	AttributeViewBlocksLock.Lock()
 	defer AttributeViewBlocksLock.Unlock()
@@ -86,6 +111,56 @@ func RemoveBlockRel(avID, blockID string) {
 	}
 }
 
+func BatchUpsertBlockRel(nodes []*ast.Node) {
+	AttributeViewBlocksLock.Lock()
+	defer AttributeViewBlocksLock.Unlock()
+
+	avBlocks := map[string][]string{}
+	blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack")
+	if !filelock.IsExist(blocks) {
+		if err := os.MkdirAll(filepath.Dir(blocks), 0755); nil != err {
+			logging.LogErrorf("create attribute view dir failed: %s", err)
+			return
+		}
+	} else {
+		data, err := filelock.ReadFile(blocks)
+		if nil != err {
+			logging.LogErrorf("read attribute view blocks failed: %s", err)
+			return
+		}
+
+		if err = msgpack.Unmarshal(data, &avBlocks); nil != err {
+			logging.LogErrorf("unmarshal attribute view blocks failed: %s", err)
+			return
+		}
+	}
+
+	for _, n := range nodes {
+		if ast.NodeAttributeView != n.Type {
+			continue
+		}
+
+		if "" == n.AttributeViewID || "" == n.ID {
+			continue
+		}
+
+		blockIDs := avBlocks[n.AttributeViewID]
+		blockIDs = append(blockIDs, n.ID)
+		blockIDs = gulu.Str.RemoveDuplicatedElem(blockIDs)
+		avBlocks[n.AttributeViewID] = blockIDs
+	}
+
+	data, err := msgpack.Marshal(avBlocks)
+	if nil != err {
+		logging.LogErrorf("marshal attribute view blocks failed: %s", err)
+		return
+	}
+	if err = filelock.WriteFile(blocks, data); nil != err {
+		logging.LogErrorf("write attribute view blocks failed: %s", err)
+		return
+	}
+}
+
 func UpsertBlockRel(avID, blockID string) {
 	AttributeViewBlocksLock.Lock()
 	defer AttributeViewBlocksLock.Unlock()

kernel/av/sort.go

@@ -74,10 +74,10 @@ func (value *Value) Compare(other *Value, attrView *AttributeView) int {
 				return 0
 			} else {
 				if !other.Number.IsNotEmpty {
-					return 0
-				}
 					return 1
 				}
+				return 0
+			}
 		}
 	case KeyTypeDate:
 		if nil != value.Date && nil != other.Date {
@@ -94,10 +94,10 @@ func (value *Value) Compare(other *Value, attrView *AttributeView) int {
 				return 0
 			} else {
 				if !other.Date.IsNotEmpty {
-					return 0
-				}
 					return 1
 				}
+				return 0
+			}
 		}
 	case KeyTypeCreated:
 		if nil != value.Created && nil != other.Created {

kernel/model/attribute_view.go

@@ -36,6 +36,7 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/cache"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
+	"github.com/xrash/smetrics"
 )
 
 func SetDatabaseBlockView(blockID, viewID string) (err error) {
@@ -194,16 +195,20 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) {
 	ret = []*SearchAttributeViewResult{}
 	keyword = strings.TrimSpace(keyword)
 
-	avs := map[string]string{}
+	type result struct {
+		AvID      string
+		AvName    string
+		AvUpdated int64
+		Score     float64
+	}
+	var avs []*result
 	avDir := filepath.Join(util.DataDir, "storage", "av")
-	const limit = 16
 	entries, err := os.ReadDir(avDir)
 	if nil != err {
 		logging.LogErrorf("read directory [%s] failed: %s", avDir, err)
 		return
 	}
-
+	avBlockRels := av.GetBlockRels()
-	count := 0
 	for _, entry := range entries {
 		if entry.IsDir() {
 			continue
@@ -214,25 +219,55 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) {
 			continue
 		}
 
-		name, _ := av.GetAttributeViewNameByPath(filepath.Join(avDir, entry.Name()))
+		if nil == avBlockRels[id] {
-		if "" == name {
 			continue
 		}
 
+		name, _ := av.GetAttributeViewNameByPath(filepath.Join(avDir, entry.Name()))
+		info, _ := entry.Info()
+		if "" != keyword {
 			if strings.Contains(strings.ToLower(name), strings.ToLower(keyword)) {
-			avs[id] = name
+				score := smetrics.JaroWinkler(name, keyword, 0.7, 4)
-			count++
+				a := &result{AvID: id, AvName: name, Score: score}
-			if limit <= count {
+				if nil != info && !info.ModTime().IsZero() {
-				break
+					a.AvUpdated = info.ModTime().UnixMilli()
 				}
+				avs = append(avs, a)
+			}
+		} else {
+			a := &result{AvID: id, AvName: name}
+			if nil != info && !info.ModTime().IsZero() {
+				a.AvUpdated = info.ModTime().UnixMilli()
+			}
+			avs = append(avs, a)
 		}
 	}
 
-	var avIDs []string
+	if "" == keyword {
-	for avID := range avs {
+		sort.Slice(avs, func(i, j int) bool { return avs[i].AvUpdated > avs[j].AvUpdated })
-		avIDs = append(avIDs, avID)
+	} else {
+		sort.SliceStable(avs, func(i, j int) bool {
+			if avs[i].Score == avs[j].Score {
+				return avs[i].AvUpdated > avs[j].AvUpdated
 			}
-	blockIDs := treenode.BatchGetMirrorAttrViewBlockIDs(avIDs)
+			return avs[i].Score > avs[j].Score
+		})
+	}
+	if 12 <= len(avs) {
+		avs = avs[:12]
+	}
+	var avIDs []string
+	for _, a := range avs {
+		avIDs = append(avIDs, a.AvID)
+	}
+
+	avBlocks := treenode.BatchGetMirrorAttrViewBlocks(avIDs)
+	var blockIDs []string
+	for _, avBlock := range avBlocks {
+		blockIDs = append(blockIDs, avBlock.BlockIDs...)
+	}
+	blockIDs = gulu.Str.RemoveDuplicatedElem(blockIDs)
+
 	trees := map[string]*parse.Tree{}
 	for _, blockID := range blockIDs {
 		bt := treenode.GetBlockTree(blockID)
@@ -261,8 +296,14 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) {
 		}
 
 		avID := node.AttributeViewID
-		name := avs[avID]
+		var existAv *result
-		if "" == name {
+		for _, av := range avs {
+			if av.AvID == avID {
+				existAv = av
+				break
+			}
+		}
+		if nil == existAv {
 			continue
 		}
 
@@ -287,7 +328,7 @@ func SearchAttributeView(keyword string) (ret []*SearchAttributeViewResult) {
 		if !exist {
 			ret = append(ret, &SearchAttributeViewResult{
 				AvID:    avID,
-				AvName:  name,
+				AvName:  existAv.AvName,
 				BlockID: blockID,
 				HPath:   hPath,
 			})
@@ -733,7 +774,7 @@ func renderAttributeView(attrView *av.AttributeView, viewID, query string, page,
 			}
 
 			if 0 == v.UpdatedAt {
-				v.UpdatedAt = v.CreatedAt + 1000
+				v.UpdatedAt = v.CreatedAt
 			}
 		}
 	}

kernel/model/import.go

@@ -281,6 +281,10 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 				}
 				return ast.WalkContinue
 			})
+
+			// 关联数据库和块
+			avNodes := tree.Root.ChildrenByType(ast.NodeAttributeView)
+			av.BatchUpsertBlockRel(avNodes)
 		}
 	}
 
@@ -895,6 +899,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
 				util.PushEndlessProgress(fmt.Sprintf(Conf.Language(66), fmt.Sprintf("%d/%d ", i, len(importTrees))+tree.HPath))
 			}
 		}
+		util.PushClearProgress()
 
 		importTrees = []*parse.Tree{}
 		searchLinks = map[string]string{}

kernel/model/index.go

@@ -36,6 +36,7 @@ import (
 	"github.com/siyuan-note/eventbus"
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
+	"github.com/siyuan-note/siyuan/kernel/av"
 	"github.com/siyuan-note/siyuan/kernel/cache"
 	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/sql"
@@ -143,6 +144,7 @@ func index(boxID string) {
 		poolSize = 4
 	}
 	waitGroup := &sync.WaitGroup{}
+	var avNodes []*ast.Node
 	p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) {
 		defer waitGroup.Done()
 
@@ -168,6 +170,10 @@ func index(boxID string) {
 			}
 		}
 
+		lock.Lock()
+		avNodes = append(avNodes, tree.Root.ChildrenByType(ast.NodeAttributeView)...)
+		lock.Unlock()
+
 		cache.PutDocIAL(file.path, docIAL)
 		treenode.IndexBlockTree(tree)
 		sql.IndexTreeQueue(tree)
@@ -191,6 +197,9 @@ func index(boxID string) {
 	waitGroup.Wait()
 	p.Release()
 
+	// 关联数据库和块
+	av.BatchUpsertBlockRel(avNodes)
+
 	box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
 	end := time.Now()
 	elapsed := end.Sub(start).Seconds()

kernel/model/search.go

@@ -384,18 +384,23 @@ func SearchRefBlock(id, rootID, keyword string, beforeLen int, isSquareBrackets
 			}
 		}
 
+		if "NodeAttributeView" == b.Type {
+			// 数据库块可以添加到自身数据库块中，当前文档也可以添加到自身数据库块中
+			tmp = append(tmp, b)
+		} else {
+			// 排除自身块、父块和根块
 			if b.ID != id && !hitFirstChildID && b.ID != rootID {
 				tmp = append(tmp, b)
 			}
 		}
+
+	}
 	ret = tmp
 
-	if "" != keyword {
 	if block := treenode.GetBlockTree(id); nil != block {
 		p := path.Join(block.HPath, keyword)
 		newDoc = nil == treenode.GetBlockTreeRootByHPath(block.BoxID, p)
 	}
-	}
 
 	// 在 hPath 中加入笔记本名 Show notebooks in hpath of block ref search list results https://github.com/siyuan-note/siyuan/issues/9378
 	prependNotebookNameInHPath(ret)

kernel/model/virutalref.go

@@ -144,7 +144,7 @@ func AddVirtualBlockRefExclude(keyword []string) {
 }
 
 func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool {
-	if !Conf.Editor.VirtualBlockRef {
+	if !Conf.Editor.VirtualBlockRef || 1 > len(virtualBlockRefKeywords) {
 		return false
 	}
 
@@ -153,12 +153,15 @@ func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeyword
 	}
 
 	parentBlock := treenode.ParentBlock(n)
-	if nil == parentBlock || 0 < refCount[parentBlock.ID] {
+	if nil == parentBlock {
 		return false
 	}
 
-	if 1 > len(virtualBlockRefKeywords) {
+	if 0 < refCount[parentBlock.ID] {
-		return false
+		// 如果块被引用过，则将其自身的文本排除在虚拟引用关键字之外
+		// Referenced blocks support rendering virtual references https://github.com/siyuan-note/siyuan/issues/10960
+		parentText := getNodeRefText(parentBlock)
+		virtualBlockRefKeywords = gulu.Str.RemoveElem(virtualBlockRefKeywords, parentText)
 	}
 
 	content := string(n.Tokens)

kernel/sql/database.go

@@ -130,6 +130,11 @@ func initDBTables() {
 		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks] failed: %s", err)
 	}
 
+	_, err = db.Exec("CREATE INDEX idx_blocks_id ON blocks(id)")
+	if nil != err {
+		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create index [idx_blocks_id] failed: %s", err)
+	}
+
 	_, err = db.Exec("CREATE INDEX idx_blocks_root_id ON blocks(root_id)")
 	if nil != err {
 		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create index [idx_blocks_root_id] failed: %s", err)
@@ -139,7 +144,7 @@ func initDBTables() {
 	if nil != err {
 		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "drop table [blocks_fts] failed: %s", err)
 	}
-	_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan\")")
+	_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts USING fts5(id, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan\")")
 	if nil != err {
 		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks_fts] failed: %s", err)
 	}
@@ -148,7 +153,7 @@ func initDBTables() {
 	if nil != err {
 		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "drop table [blocks_fts_case_insensitive] failed: %s", err)
 	}
-	_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts_case_insensitive USING fts5(id UNINDEXED, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan case_insensitive\")")
+	_, err = db.Exec("CREATE VIRTUAL TABLE blocks_fts_case_insensitive USING fts5(id, parent_id UNINDEXED, root_id UNINDEXED, hash UNINDEXED, box UNINDEXED, path UNINDEXED, hpath, name, alias, memo, tag, content, fcontent, markdown UNINDEXED, length UNINDEXED, type UNINDEXED, subtype UNINDEXED, ial, sort UNINDEXED, created UNINDEXED, updated UNINDEXED, tokenize=\"siyuan case_insensitive\")")
 	if nil != err {
 		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [blocks_fts_case_insensitive] failed: %s", err)
 	}
@@ -954,30 +959,25 @@ func deleteByBoxTx(tx *sql.Tx, box string) (err error) {
 }
 
 func deleteBlocksByIDs(tx *sql.Tx, ids []string) (err error) {
-	in := bytes.Buffer{}
+	var ftsIDs []string
-	in.Grow(4096)
+	for _, id := range ids {
-	in.WriteString("(")
+		removeBlockCache(id)
-	for i, id := range ids {
+		ftsIDs = append(ftsIDs, "\""+id+"\"")
-		in.WriteString("'")
-		in.WriteString(id)
-		in.WriteString("'")
-		if i < len(ids)-1 {
-			in.WriteString(",")
 	}
 
-		removeBlockCache(id)
+	stmt := "DELETE FROM blocks WHERE id IN (" + strings.Join(ftsIDs, ",") + ")"
-	}
-	in.WriteString(")")
-	stmt := "DELETE FROM blocks WHERE id IN " + in.String()
 	if err = execStmtTx(tx, stmt); nil != err {
 		return
 	}
-	stmt = "DELETE FROM blocks_fts WHERE id IN " + in.String()
+
+	ftsIDsMatch := strings.Join(ftsIDs, " OR ")
+	stmt = "DELETE FROM blocks_fts WHERE ROWID IN (SELECT ROWID FROM blocks_fts WHERE blocks_fts MATCH 'id:(" + ftsIDsMatch + ")')"
 	if err = execStmtTx(tx, stmt); nil != err {
 		return
 	}
+
 	if !caseSensitive {
-		stmt = "DELETE FROM blocks_fts_case_insensitive WHERE id IN " + in.String()
+		stmt = "DELETE FROM blocks_fts_case_insensitive WHERE ROWID IN (SELECT ROWID FROM blocks_fts_case_insensitive WHERE blocks_fts_case_insensitive MATCH 'id:(" + ftsIDsMatch + ")')"
 		if err = execStmtTx(tx, stmt); nil != err {
 			return
 		}

kernel/treenode/av.go

@@ -26,41 +26,6 @@ import (
 	"github.com/vmihailenco/msgpack/v5"
 )
 
-func BatchGetMirrorAttrViewBlockIDs(avIDs []string) (ret map[string]string) {
-	av.AttributeViewBlocksLock.Lock()
-	defer av.AttributeViewBlocksLock.Unlock()
-
-	ret = map[string]string{}
-
-	blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack")
-	if !filelock.IsExist(blocks) {
-		return
-	}
-
-	data, err := filelock.ReadFile(blocks)
-	if nil != err {
-		logging.LogErrorf("read attribute view blocks failed: %s", err)
-		return
-	}
-
-	avBlocks := map[string][]string{}
-	if err = msgpack.Unmarshal(data, &avBlocks); nil != err {
-		logging.LogErrorf("unmarshal attribute view blocks failed: %s", err)
-		return
-	}
-
-	for _, avID := range avIDs {
-		blockIDs := avBlocks[avID]
-		for _, blockID := range blockIDs {
-			if nil != GetBlockTree(blockID) {
-				ret[avID] = blockID
-				break
-			}
-		}
-	}
-	return
-}
-
 func GetMirrorAttrViewBlockIDs(avID string) (ret []string) {
 	av.AttributeViewBlocksLock.Lock()
 	defer av.AttributeViewBlocksLock.Unlock()
@@ -91,3 +56,49 @@ func GetMirrorAttrViewBlockIDs(avID string) (ret []string) {
 	}
 	return
 }
+
+type AvBlock struct {
+	AvID     string
+	BlockIDs []string
+}
+
+func BatchGetMirrorAttrViewBlocks(avIDs []string) (ret []*AvBlock) {
+	av.AttributeViewBlocksLock.Lock()
+	defer av.AttributeViewBlocksLock.Unlock()
+
+	ret = []*AvBlock{}
+
+	blocks := filepath.Join(util.DataDir, "storage", "av", "blocks.msgpack")
+	if !filelock.IsExist(blocks) {
+		return
+	}
+
+	data, err := filelock.ReadFile(blocks)
+	if nil != err {
+		logging.LogErrorf("read attribute view blocks failed: %s", err)
+		return
+	}
+
+	avBlocks := map[string][]string{}
+	if err = msgpack.Unmarshal(data, &avBlocks); nil != err {
+		logging.LogErrorf("unmarshal attribute view blocks failed: %s", err)
+		return
+	}
+
+	for _, avID := range avIDs {
+		var blockIDs []string
+		for _, blockID := range avBlocks[avID] {
+			if nil == GetBlockTree(blockID) {
+				continue
+			}
+
+			blockIDs = append(blockIDs, blockID)
+		}
+		avBlock := &AvBlock{
+			AvID:     avID,
+			BlockIDs: blockIDs,
+		}
+		ret = append(ret, avBlock)
+	}
+	return
+}

kernel/treenode/node.go

@@ -954,7 +954,7 @@ func GetAttributeViewDefaultValue(valueID, keyID, blockID string, typ av.KeyType
 		ret.CreatedAt = time.Now().UnixMilli()
 	}
 	if 0 == ret.UpdatedAt {
-		ret.UpdatedAt = ret.CreatedAt + 1000
+		ret.UpdatedAt = ret.CreatedAt
 	}
 
 	switch typ {