⚡ Improve performance for data indexing https://github.com/siyuan-note/siyuan/issues/12777

kernel/model/block.go

@@ -364,7 +364,7 @@ func TransferBlockRef(fromID, toID string, refIDs []string) (err error) {
 		}
 	}
 
-	sql.WaitForWritingDatabase()
+	sql.FlushQueue()
 	return
 }
 

kernel/model/blockial.go

@@ -191,9 +191,7 @@ func setNodeAttrs(node *ast.Node, tree *parse.Tree, nameValues map[string]string
 	pushBroadcastAttrTransactions(oldAttrs, node)
 
 	go func() {
-		if !sql.IsEmptyQueue() {
+		sql.FlushQueue()
-			sql.WaitForWritingDatabase()
-		}
 		refreshDynamicRefText(node, tree)
 	}()
 	return

kernel/model/bookmark.go

@@ -159,9 +159,7 @@ func BookmarkLabels() (ret []string) {
 
 func BuildBookmark() (ret *Bookmarks) {
 	WaitForWritingFiles()
-	if !sql.IsEmptyQueue() {
+	sql.FlushQueue()
-		sql.WaitForWritingDatabase()
-	}
 
 	ret = &Bookmarks{}
 	sqlBlocks := sql.QueryBookmarkBlocks()

kernel/model/box.go

@@ -606,7 +606,7 @@ func FullReindex() {
 	task.AppendTask(task.DatabaseIndexFull, fullReindex)
 	task.AppendTask(task.DatabaseIndexRef, IndexRefs)
 	go func() {
-		sql.WaitForWritingDatabase()
+		sql.FlushQueue()
 		ResetVirtualBlockRefCache()
 	}()
 	task.AppendTaskWithTimeout(task.DatabaseIndexEmbedBlock, 30*time.Second, autoIndexEmbedBlock)

kernel/model/conf.go

@@ -612,7 +612,7 @@ func Close(force, setCurrentWorkspace bool, execInstallPkg int) (exitCode int) {
 	closeUserGuide()
 
 	// Improve indexing completeness when exiting https://github.com/siyuan-note/siyuan/issues/12039
-	sql.WaitForWritingDatabaseIn(200 * time.Millisecond)
+	sql.FlushQueue()
 
 	util.IsExiting.Store(true)
 	waitSecondForExecInstallPkg := false

kernel/model/history.go

@@ -295,7 +295,7 @@ func RollbackDocHistory(boxID, historyPath string) (err error) {
 	}
 
 	go func() {
-		sql.WaitForWritingDatabase()
+		sql.FlushQueue()
 
 		tree, _ = LoadTreeByBlockID(id)
 		if nil == tree {

kernel/model/index.go

@@ -107,7 +107,7 @@ func listSyFiles(dir string) (ret []string) {
 func (box *Box) Unindex() {
 	task.AppendTask(task.DatabaseIndex, unindex, box.ID)
 	go func() {
-		sql.WaitForWritingDatabase()
+		sql.FlushQueue()
 		ResetVirtualBlockRefCache()
 	}()
 }
@@ -123,7 +123,7 @@ func (box *Box) Index() {
 	task.AppendTask(task.DatabaseIndex, index, box.ID)
 	task.AppendTask(task.DatabaseIndexRef, IndexRefs)
 	go func() {
-		sql.WaitForWritingDatabase()
+		sql.FlushQueue()
 		ResetVirtualBlockRefCache()
 	}()
 }

kernel/model/index_fix.go

@@ -50,15 +50,15 @@ func checkIndex() {
 		logging.LogInfof("start checking index...")
 
 		task.AppendTask(task.DatabaseIndexFix, removeDuplicateDatabaseIndex)
-		sql.WaitForWritingDatabase()
+		sql.FlushQueue()
 
 		task.AppendTask(task.DatabaseIndexFix, resetDuplicateBlocksOnFileSys)
 
 		task.AppendTask(task.DatabaseIndexFix, fixBlockTreeByFileSys)
-		sql.WaitForWritingDatabase()
+		sql.FlushQueue()
 
 		task.AppendTask(task.DatabaseIndexFix, fixDatabaseIndexByBlockTree)
-		sql.WaitForWritingDatabase()
+		sql.FlushQueue()
 
 		task.AppendTask(task.DatabaseIndexFix, removeDuplicateDatabaseRefs)
 

kernel/model/push_reload.go

@@ -140,7 +140,7 @@ func refreshProtyle(rootID string) {
 
 // refreshRefCount 用于刷新定义块处的引用计数。
 func refreshRefCount(rootID, blockID string) {
-	sql.WaitForWritingDatabase()
+	sql.FlushQueue()
 
 	bt := treenode.GetBlockTree(blockID)
 	if nil == bt {

kernel/model/repository.go

@@ -660,7 +660,7 @@ func checkoutRepo(id string) {
 	task.AppendTask(task.DatabaseIndexFull, fullReindex)
 	task.AppendTask(task.DatabaseIndexRef, IndexRefs)
 	go func() {
-		sql.WaitForWritingDatabase()
+		sql.FlushQueue()
 		ResetVirtualBlockRefCache()
 	}()
 	task.AppendTask(task.ReloadUI, util.ReloadUIResetScroll)

kernel/model/search.go

@@ -791,14 +791,14 @@ func FindReplace(keyword, replacement string, replaceTypes map[string]bool, ids
 		util.PushEndlessProgress(fmt.Sprintf(Conf.Language(207), i+1, len(renameRoots)))
 	}
 
-	sql.WaitForWritingDatabase()
+	sql.FlushQueue()
 
 	reloadTreeIDs = gulu.Str.RemoveDuplicatedElem(reloadTreeIDs)
 	for _, id := range reloadTreeIDs {
 		refreshProtyle(id)
 	}
 
-	sql.WaitForWritingDatabase()
+	sql.FlushQueue()
 	util.PushClearProgress()
 	return
 }

kernel/model/tag.go

@@ -207,10 +207,7 @@ type Tags []*Tag
 
 func BuildTags() (ret *Tags) {
 	WaitForWritingFiles()
-
+	sql.FlushQueue()
-	if !sql.IsEmptyQueue() {
-		sql.WaitForWritingDatabase()
-	}
 
 	ret = &Tags{}
 	labels := labelTags()

kernel/model/transaction.go

@@ -1532,7 +1532,7 @@ func updateRefTextRenameDoc(renamedTree *parse.Tree) {
 }
 
 func FlushUpdateRefTextRenameDocJob() {
-	sql.WaitForWritingDatabase()
+	sql.FlushQueue()
 	flushUpdateRefTextRenameDoc()
 }
 

kernel/sql/queue.go

@@ -58,44 +58,6 @@ func FlushTxJob() {
 	task.AppendTask(task.DatabaseIndexCommit, FlushQueue)
 }
 
-func WaitForWritingDatabase() {
-	var printLog bool
-	var lastPrintLog bool
-	for i := 0; isWritingDatabase(util.SQLFlushInterval + 50*time.Millisecond); i++ {
-		time.Sleep(50 * time.Millisecond)
-		if 200 < i && !printLog { // 10s 后打日志
-			logging.LogWarnf("database is writing: \n%s", logging.ShortStack())
-			printLog = true
-		}
-		if 1200 < i && !lastPrintLog { // 60s 后打日志
-			logging.LogWarnf("database is still writing")
-			lastPrintLog = true
-		}
-	}
-}
-
-func WaitForWritingDatabaseIn(duration time.Duration) {
-	for i := 0; isWritingDatabase(duration); i++ {
-		time.Sleep(50 * time.Millisecond)
-	}
-}
-
-func isWritingDatabase(d time.Duration) bool {
-	time.Sleep(d)
-	dbQueueLock.Lock()
-	defer dbQueueLock.Unlock()
-	if 0 < len(operationQueue) || isWriting {
-		return true
-	}
-	return false
-}
-
-func IsEmptyQueue() bool {
-	dbQueueLock.Lock()
-	defer dbQueueLock.Unlock()
-	return 1 > len(operationQueue)
-}
-
 func ClearQueue() {
 	dbQueueLock.Lock()
 	defer dbQueueLock.Unlock()