🎨 改进内核任务调度机制提升稳定性 https://github.com/siyuan-note/siyuan/issues/7113

app/appearance/langs/en_US.json

@@ -975,8 +975,8 @@
     "86": "Please configure [Settings - About - Access authorization code]",
     "87": "Cannot move to this location",
     "88": "Finished parsing [%d] data files, remaining to be processed [%d]",
-    "89": "Created [%d] of data indexes of block-level elements [%s]",
+    "89": "[%d/%d] Created [%d] of data indexes of block-level elements [%s]",
-    "90": "Created [%d] of search indexes of block-level elements [%s]",
+    "90": "[%d/%d] Created [%d] of search indexes of block-level elements [%s]",
     "91": "Reading block tree data...",
     "92": "Parsing document tree [%s]",
     "93": "TODO",

app/appearance/langs/es_ES.json

@@ -975,8 +975,8 @@
     "86": "Por favor, configure [Configuración - Acerca de - Código de autorización de acceso]",
     "87": "No se puede mover a esta ubicación",
     "88": "Se ha terminado de analizar [%d] archivos de datos, quedan por procesar [%d]",
-    "89": "Creado [%d] de índices de datos de elementos a nivel de bloque [%s]",
+    "89": "[%d/%d] Creado [%d] de índices de datos de elementos a nivel de bloque [%s]",
-    "90": "Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
+    "90": "[%d/%d] Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
     "91": "Leyendo datos del árbol de bloques...",
     "92": "Analizando el árbol del documento [%s]",
     "93": "TODO",

app/appearance/langs/fr_FR.json

@@ -975,8 +975,8 @@
     "86": "Veuillez configurer [Paramètres - A propos de - Code d'autorisation d'accès]",
     "87": "Impossible de se déplacer vers cet endroit",
     "88": "Fin de l'analyse des fichiers de données [%d], restant à traiter [%d]",
-    "89": "Créé [%d] d'index de données d'éléments de niveau bloc [%s]",
+    "89": "[%d/%d] Créé [%d] d'index de données d'éléments de niveau bloc [%s]",
-    "90": "Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
+    "90": "[%d/%d] Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
     "91": "Lecture des données de l'arborescence des blocs...",
     "92": "Analyse de l'arborescence du document [%s]",
     "93": "TODO",

app/appearance/langs/zh_CHT.json

@@ -975,8 +975,8 @@
     "86": "請先配置 [設置 - 關於 - 存取授權碼]",
     "87": "無法移動到該位置",
     "88": "已完成解析 [%d] 個數據文件，剩餘待處理 [%d]",
-    "89": "已經建立 [%d] 個塊級元素的數據索引 [%s]",
+    "89": "[%d/%d] 已經建立 [%d] 個塊級元素的數據索引 [%s]",
-    "90": "已經建立 [%d] 個塊級元素的搜索索引 [%s]",
+    "90": "[%d/%d] 已經建立 [%d] 個塊級元素的搜索索引 [%s]",
     "91": "正在讀取塊樹數據...",
     "92": "正在解析文檔樹 [%s]",
     "93": "TODO",

app/appearance/langs/zh_CN.json

@@ -975,8 +975,8 @@
     "86": "请先配置 [设置 - 关于 - 访问授权码]",
     "87": "无法移动到该位置",
     "88": "已完成解析 [%d] 个数据文件，剩余待处理 [%d]",
-    "89": "已经建立 [%d] 个块级元素的数据索引 [%s]",
+    "89": "[%d/%d] 已经建立 [%d] 个块级元素的数据索引 [%s]",
-    "90": "已经建立 [%d] 个块级元素的搜索索引 [%s]",
+    "90": "[%d/%d] 已经建立 [%d] 个块级元素的搜索索引 [%s]",
     "91": "正在读取块树数据...",
     "92": "正在解析文档树 [%s]",
     "93": "TODO",

kernel/model/index.go

@@ -187,23 +187,27 @@ func IndexRefs() {
 }
 
 func init() {
-	eventbus.Subscribe(eventbus.EvtSQLInsertBlocks, func(context map[string]interface{}, blockCount int, hash string) {
+	//eventbus.Subscribe(eventbus.EvtSQLInsertBlocks, func(context map[string]interface{}, current, total, blockCount int, hash string) {
+	//	if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
+	//		// Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392
+	//		return
+	//	}
+	//
+	//	msg := fmt.Sprintf(Conf.Language(89), current, total, blockCount, hash)
+	//	util.SetBootDetails(msg)
+	//	util.ContextPushMsg(context, msg)
+	//})
+	eventbus.Subscribe(eventbus.EvtSQLInsertBlocksFTS, func(context map[string]interface{}, current, total, blockCount int, hash string) {
 		if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
 			// Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392
 			return
 		}
 
-		msg := fmt.Sprintf(Conf.Language(89), blockCount, hash)
+		if (1 > current && 1 == total) || current == total-1 {
-		util.SetBootDetails(msg)
+			current = total
-		util.ContextPushMsg(context, msg)
-	})
-	eventbus.Subscribe(eventbus.EvtSQLInsertBlocksFTS, func(context map[string]interface{}, blockCount int, hash string) {
-		if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
-			// Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392
-			return
 		}
 
-		msg := fmt.Sprintf(Conf.Language(90), blockCount, hash)
+		msg := fmt.Sprintf(Conf.Language(90), current, total, blockCount, hash)
 		util.SetBootDetails(msg)
 		util.ContextPushMsg(context, msg)
 	})

kernel/sql/database.go

@@ -180,9 +180,10 @@ func initHistoryDBConnection() {
 
 	dsn := util.HistoryDBPath + "?_journal_mode=OFF" +
 		"&_synchronous=OFF" +
+		"&_mmap_size=2684354560" +
 		"&_secure_delete=OFF" +
 		"&_cache_size=-20480" +
-		"&_page_size=8192" +
+		"&_page_size=32768" +
 		"&_busy_timeout=7000" +
 		"&_ignore_check_constraints=ON" +
 		"&_temp_store=MEMORY" +
@@ -212,9 +213,10 @@ func initDBConnection() {
 	}
 	dsn := util.DBPath + "?_journal_mode=WAL" +
 		"&_synchronous=OFF" +
+		"&_mmap_size=2684354560" +
 		"&_secure_delete=OFF" +
 		"&_cache_size=-20480" +
-		"&_page_size=8192" +
+		"&_page_size=32768" +
 		"&_busy_timeout=7000" +
 		"&_ignore_check_constraints=ON" +
 		"&_temp_store=MEMORY" +

kernel/sql/queue.go

@@ -101,57 +101,42 @@ func FlushQueue() {
 	txLock.Lock()
 	defer txLock.Unlock()
 	start := time.Now()
-	tx, err := beginTx()
-	if nil != err {
-		return
-	}
 
-	var execOps int
 	context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
+	total := len(ops)
 	for i, op := range ops {
 		if util.IsExiting {
 			return
 		}
 
-		err = execOp(op, tx, context)
+		tx, err := beginTx()
-		execOps++
-
 		if nil != err {
+			return
+		}
+
+		if err = execOp(op, tx, i, total, context); nil != err {
 			logging.LogErrorf("queue operation failed: %s", err)
 			return
 		}
 
-		if 0 < i && 0 == execOps%128 {
-			if err = commitTx(tx); nil != err {
-				logging.LogErrorf("commit tx failed: %s", err)
-				return
-			}
-
-			execOps = 0
-			tx, err = beginTx()
-			if nil != err {
-				return
-			}
-		}
-	}
-
-	if 0 < execOps {
 		if err = commitTx(tx); nil != err {
 			logging.LogErrorf("commit tx failed: %s", err)
+			return
 		}
 	}
+
 	elapsed := time.Now().Sub(start).Milliseconds()
 	if 5000 < elapsed {
 		logging.LogInfof("op tx [%dms]", elapsed)
 	}
 }
 
-func execOp(op *dbQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) {
+func execOp(op *dbQueueOperation, tx *sql.Tx, current, total int, context map[string]interface{}) (err error) {
 	switch op.action {
 	case "index":
-		err = indexTree(tx, op.box, op.indexPath, context)
+		err = indexTree(tx, op.box, op.indexPath, current, total, context)
 	case "upsert":
-		err = upsertTree(tx, op.upsertTree, context)
+		err = upsertTree(tx, op.upsertTree, current, total, context)
 	case "delete":
 		err = batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath)
 	case "delete_id":
@@ -171,7 +156,7 @@ func execOp(op *dbQueueOperation, tx *sql.Tx, context map[string]interface{}) (e
 	case "update_refs":
 		err = upsertRefs(tx, op.upsertTree)
 	default:
-		msg := fmt.Sprint("unknown operation [%s]", op.action)
+		msg := fmt.Sprintf("unknown operation [%s]", op.action)
 		logging.LogErrorf(msg)
 		err = errors.New(msg)
 	}

kernel/sql/upsert.go

@@ -51,7 +51,7 @@ const (
 	FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
 )
 
-func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (err error) {
+func insertBlocks(tx *sql.Tx, blocks []*Block, current, total int, context map[string]interface{}) (err error) {
 	if 1 > len(blocks) {
 		return
 	}
@@ -63,20 +63,20 @@ func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (
 			continue
 		}
 
-		if err = insertBlocks0(tx, bulk, context); nil != err {
+		if err = insertBlocks0(tx, bulk, current, total, context); nil != err {
 			return
 		}
 		bulk = []*Block{}
 	}
 	if 0 < len(bulk) {
-		if err = insertBlocks0(tx, bulk, context); nil != err {
+		if err = insertBlocks0(tx, bulk, current, total, context); nil != err {
 			return
 		}
 	}
 	return
 }
 
-func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (err error) {
+func insertBlocks0(tx *sql.Tx, bulk []*Block, current, total int, context map[string]interface{}) (err error) {
 	valueStrings := make([]string, 0, len(bulk))
 	valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?"))
 	hashBuf := bytes.Buffer{}
@@ -114,7 +114,7 @@ func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (e
 	}
 	hashBuf.WriteString("blocks")
 	evtHash := fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
-	eventbus.Publish(eventbus.EvtSQLInsertBlocks, context, len(bulk), evtHash)
+	//eventbus.Publish(eventbus.EvtSQLInsertBlocks, context, current, total, len(bulk), evtHash)
 
 	stmt = fmt.Sprintf(BlocksFTSInsert, strings.Join(valueStrings, ","))
 	if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
@@ -129,7 +129,7 @@ func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (e
 	}
 	hashBuf.WriteString("fts")
 	evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
-	eventbus.Publish(eventbus.EvtSQLInsertBlocksFTS, context, len(bulk), evtHash)
+	eventbus.Publish(eventbus.EvtSQLInsertBlocksFTS, context, current, total, len(bulk), evtHash)
 	return
 }
 
@@ -393,24 +393,24 @@ func insertRefs(tx *sql.Tx, tree *parse.Tree) (err error) {
 	return err
 }
 
-func indexTree(tx *sql.Tx, box, p string, context map[string]interface{}) (err error) {
+func indexTree(tx *sql.Tx, box, p string, current, total int, context map[string]interface{}) (err error) {
 	tree, err := filesys.LoadTree(box, p, luteEngine)
 	if nil != err {
 		return
 	}
 
-	err = insertTree(tx, tree, context)
+	err = insertTree(tx, tree, current, total, context)
 	return
 }
 
-func insertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
+func insertTree(tx *sql.Tx, tree *parse.Tree, current, total int, context map[string]interface{}) (err error) {
 	blocks, spans, assets, attributes := fromTree(tree.Root, tree)
 	refs, fileAnnotationRefs := refsFromTree(tree)
-	err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs)
+	err = insertTree0(tx, tree, current, total, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs)
 	return
 }
 
-func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
+func upsertTree(tx *sql.Tx, tree *parse.Tree, current, total int, context map[string]interface{}) (err error) {
 	oldBlockHashes := queryBlockHashes(tree.ID)
 	blocks, spans, assets, attributes := fromTree(tree.Root, tree)
 	newBlockHashes := map[string]string{}
@@ -460,16 +460,16 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (e
 	}
 
 	refs, fileAnnotationRefs := refsFromTree(tree)
-	if err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); nil != err {
+	if err = insertTree0(tx, tree, current, total, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); nil != err {
 		return
 	}
 	return err
 }
 
-func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{},
+func insertTree0(tx *sql.Tx, tree *parse.Tree, current, total int, context map[string]interface{},
 	blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute,
 	refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) {
-	if err = insertBlocks(tx, blocks, context); nil != err {
+	if err = insertBlocks(tx, blocks, current, total, context); nil != err {
 		return
 	}