🎨 Distinguishing between a referenced definition block that does not exist or one that is not indexed https://github.com/siyuan-note/siyuan/issues/16850

Signed-off-by: Daniel <845765@qq.com>

kernel/model/tree.go

@@ -17,12 +17,17 @@
 package model
 
 import (
+	"bufio"
+	"bytes"
 	"errors"
+	"fmt"
 	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/88250/lute"
@@ -179,6 +184,7 @@ var (
 	ErrBlockNotFound = errors.New("block not found")
 	ErrTreeNotFound  = errors.New("tree not found")
 	ErrIndexing      = errors.New("indexing")
+	ErrBoxUnindexed  = errors.New("notebook unindexed")
 )
 
 func LoadTreeByBlockIDWithReindex(id string) (ret *parse.Tree, err error) {
@@ -195,14 +201,13 @@ func LoadTreeByBlockIDWithReindex(id string) (ret *parse.Tree, err error) {
 		}
 
 		// 尝试从文件系统加载并建立索引
-		indexTreeInFilesystem(id)
-
+		err = indexTreeInFilesystem(id)
 		bt = treenode.GetBlockTree(id)
 		if nil == bt {
 			if "dev" == util.Mode {
 				logging.LogWarnf("block tree not found [id=%s], stack: [%s]", id, logging.ShortStack())
 			}
-			return nil, ErrTreeNotFound
+			return
 		}
 	}
 
@@ -246,64 +251,53 @@ func loadTreeByBlockTree(bt *treenode.BlockTree) (ret *parse.Tree, err error) {
 
 var searchTreeLimiter = rate.NewLimiter(rate.Every(3*time.Second), 1)
 
-func indexTreeInFilesystem(rootID string) {
+func indexTreeInFilesystem(blockID string) error {
 	if !searchTreeLimiter.Allow() {
-		return
+		return ErrIndexing
 	}
 
 	msdID := util.PushMsg(Conf.language(45), 7000)
 	defer util.PushClearMsg(msdID)
 
-	logging.LogWarnf("searching tree on filesystem [rootID=%s]", rootID)
-	var treePath string
-	filelock.Walk(util.DataDir, func(path string, d fs.DirEntry, err error) error {
-		if d.IsDir() {
-			if strings.HasPrefix(d.Name(), ".") {
-				return filepath.SkipDir
-			}
-			return nil
-		}
+	logging.LogWarnf("searching tree on filesystem [id=%s]", blockID)
 
-		if !strings.HasSuffix(d.Name(), ".sy") {
-			return nil
-		}
-
-		baseName := filepath.Base(path)
-		if rootID+".sy" != baseName {
-			return nil
-		}
-
-		treePath = path
-		return filepath.SkipAll
-	})
-
-	if "" == treePath {
-		logging.LogErrorf("tree not found on filesystem [rootID=%s]", rootID)
-		return
+	unindexedTreePath := findUnindexedTreePathInAllBoxes(blockID)
+	if "" == unindexedTreePath {
+		logging.LogInfof("tree not found on filesystem [id=%s]", blockID)
+		return ErrTreeNotFound
 	}
 
-	boxID := strings.TrimPrefix(treePath, util.DataDir)
+	boxID := strings.TrimPrefix(unindexedTreePath, util.DataDir)
 	boxID = boxID[1:]
 	boxID = boxID[:strings.Index(boxID, string(os.PathSeparator))]
-	treePath = strings.TrimPrefix(treePath, util.DataDir)
-	treePath = strings.TrimPrefix(treePath, string(os.PathSeparator))
-	treePath = strings.TrimPrefix(treePath, boxID)
-	treePath = filepath.ToSlash(treePath)
+	unindexedTreePath = strings.TrimPrefix(unindexedTreePath, util.DataDir)
+	unindexedTreePath = strings.TrimPrefix(unindexedTreePath, string(os.PathSeparator))
+	unindexedTreePath = strings.TrimPrefix(unindexedTreePath, boxID)
+	unindexedTreePath = filepath.ToSlash(unindexedTreePath)
 	if nil == Conf.Box(boxID) {
+		for _, b := range Conf.GetClosedBoxes() {
+			if b.ID == boxID {
+				logging.LogInfof("box [%s] is closed", boxID)
+				util.PushErrMsg(fmt.Sprintf(Conf.language(197), b.Name), 7000)
+				return ErrBoxUnindexed
+			}
+		}
+
 		logging.LogInfof("box [%s] not found", boxID)
-		// 如果笔记本不存在或者已经关闭，则不处理 https://github.com/siyuan-note/siyuan/issues/11149
-		return
+		// 如果笔记本不存在则不处理 https://github.com/siyuan-note/siyuan/issues/11149
+		return ErrTreeNotFound
 	}
 
-	tree, err := filesys.LoadTree(boxID, treePath, util.NewLute())
+	tree, err := filesys.LoadTree(boxID, unindexedTreePath, util.NewLute())
 	if err != nil {
-		logging.LogErrorf("load tree [%s] failed: %s", treePath, err)
-		return
+		logging.LogErrorf("load tree [%s] failed: %s", unindexedTreePath, err)
+		return err
 	}
 
 	treenode.UpsertBlockTree(tree)
 	sql.IndexTreeQueue(tree)
-	logging.LogInfof("reindexed tree by filesystem [rootID=%s]", rootID)
+	logging.LogInfof("reindexed tree by filesystem [blockID=%s]", blockID)
+	return nil
 }
 
 func loadParentTree(tree *parse.Tree) (ret *parse.Tree) {
@@ -318,3 +312,111 @@ func loadParentTree(tree *parse.Tree) (ret *parse.Tree) {
 	ret, _ = filesys.LoadTree(tree.Box, parentPath, luteEngine)
 	return
 }
+
+func findUnindexedTreePathInAllBoxes(id string) (ret string) {
+	boxes := Conf.GetBoxes()
+	for _, box := range boxes {
+		root := filepath.Join(util.DataDir, box.ID)
+		paths := findAllOccurrences(root, id)
+		var rootIDs []string
+		rootIDPaths := map[string]string{}
+		for _, p := range paths {
+			rootID := util.GetTreeID(p)
+			rootIDs = append(rootIDs, rootID)
+			rootIDPaths[rootID] = p
+		}
+
+		result := treenode.ExistBlockTrees(rootIDs)
+		for rootID, exist := range result {
+			if !exist {
+				return rootIDPaths[rootID]
+			}
+		}
+	}
+	return
+}
+
+func findAllOccurrences(root string, target string) []string {
+	if root == "" || target == "" {
+		return nil
+	}
+
+	searchBytes := []byte(target)
+	jobs := make(chan string, 256)    // 任务通道
+	results := make(chan string, 256) // 结果通道
+
+	// 用于等待所有 Worker 完成
+	var wg sync.WaitGroup
+	// 用于等待结果收集器完成
+	var collectWg sync.WaitGroup
+
+	// 1. 启动结果收集协程
+	var matchedPaths []string
+	collectWg.Add(1)
+	go func() {
+		defer collectWg.Done()
+		for path := range results {
+			matchedPaths = append(matchedPaths, path)
+		}
+	}()
+
+	// 2. 启动并发 Worker Pool (基于 CPU 核心数)
+	numWorkers := runtime.NumCPU()
+	for i := 0; i < numWorkers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for path := range jobs {
+				if containsTarget(path, searchBytes) {
+					results <- path
+				}
+			}
+		}()
+	}
+
+	// 3. 遍历文件夹并分发任务
+	_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
+		if err == nil && d.Type().IsRegular() {
+			jobs <- path
+		}
+		return nil
+	})
+
+	// 4. 关闭通道并等待结束
+	close(jobs)      // 停止分发任务
+	wg.Wait()        // 等待所有 Worker 处理完
+	close(results)   // 停止收集结果
+	collectWg.Wait() // 等待切片组装完成
+
+	return matchedPaths
+}
+
+// containsTarget 针对大文件优化的字节流匹配函数
+func containsTarget(path string, target []byte) bool {
+	f, err := os.Open(path)
+	if err != nil {
+		return false
+	}
+	defer f.Close()
+
+	// 1MB 缓冲区
+	reader := bufio.NewReaderSize(f, 1024*1024)
+	for {
+		// 使用 ReadSlice 实现零拷贝读取
+		line, err := reader.ReadSlice('\n')
+		if len(line) > 0 && bytes.Contains(line, target) {
+			return true
+		}
+		if err != nil {
+			if err == bufio.ErrBufferFull {
+				// 处理超过 1MB 的超长行，直接跳过当前行剩余部分
+				for err == bufio.ErrBufferFull {
+					_, err = reader.ReadSlice('\n')
+				}
+				continue
+			}
+			break // EOF 或其他错误
+		}
+	}
+	return false
+}