🎨 改进内核任务调度机制提升稳定性 https://github.com/siyuan-note/siyuan/issues/7113

This commit is contained in:
Liang Ding 2023-01-26 00:47:14 +08:00
parent c1ff45f4a4
commit 7d1b50116b
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
14 changed files with 51 additions and 52 deletions

View file

@ -873,7 +873,8 @@
"task.database.index.fix": "Execute database index fix", "task.database.index.fix": "Execute database index fix",
"task.ocr.image": "Execute image OCR to extract text", "task.ocr.image": "Execute image OCR to extract text",
"task.history.generateDoc": "Execute GenerateDoc History", "task.history.generateDoc": "Execute GenerateDoc History",
"task.database.index.embedBlock": "Execute database index embed block" "task.database.index.embedBlock": "Execute database index embed block",
"task.reload.ui": "Execute reload UI"
}, },
"_trayMenu": { "_trayMenu": {
"showWindow": "Show Window", "showWindow": "Show Window",
@ -944,7 +945,7 @@
"55": "Indexed references of [%d] documents", "55": "Indexed references of [%d] documents",
"56": "Reindexing, please wait until rebuilding is complete before trying to open", "56": "Reindexing, please wait until rebuilding is complete before trying to open",
"57": "Failed to create temp key", "57": "Failed to create temp key",
"58": "After the index is rebuilt, the interface will be automatically refreshed later...", "58": "TODO",
"59": "Failed to set sync ignore list", "59": "Failed to set sync ignore list",
"60": "Failed to get the update package: %s", "60": "Failed to get the update package: %s",
"61": "⬆️ The new version installation package is ready, do you want to install the new version now?", "61": "⬆️ The new version installation package is ready, do you want to install the new version now?",

View file

@ -873,7 +873,8 @@
"task.database.index.fix": "Ejecutar corrección del índice de la base de datos", "task.database.index.fix": "Ejecutar corrección del índice de la base de datos",
"task.ocr.image": "Ejecutar OCR de imagen para extraer texto", "task.ocr.image": "Ejecutar OCR de imagen para extraer texto",
"task.history.generateDoc": "Ejecutar Historial GenerateDoc", "task.history.generateDoc": "Ejecutar Historial GenerateDoc",
"task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos" "task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos",
"task.reload.ui": "IU de recarga de tareas"
}, },
"_trayMenu": { "_trayMenu": {
"showWindow": "Mostrar ventana", "showWindow": "Mostrar ventana",
@ -944,7 +945,7 @@
"55": "Referencias indexadas de [%d] documentos", "55": "Referencias indexadas de [%d] documentos",
"56": "Reindexando, espere hasta que se complete la reconstrucción antes de intentar abrir", "56": "Reindexando, espere hasta que se complete la reconstrucción antes de intentar abrir",
"57": "Fallo en la creación de la clave temporal", "57": "Fallo en la creación de la clave temporal",
"58": "Después de reconstruir el índice, la interfaz se actualizará automáticamente más tarde...", "58": "TODO",
"59": " Falló la configuración de sincronización de la lista de ignorados", "59": " Falló la configuración de sincronización de la lista de ignorados",
"60": "Fallo al obtener el paquete de actualización: %s", "60": "Fallo al obtener el paquete de actualización: %s",
"61": "⬆️ El paquete de instalación de la nueva versión está listo, ¿quieres instalar la nueva versión ahora?", "61": "⬆️ El paquete de instalación de la nueva versión está listo, ¿quieres instalar la nueva versión ahora?",

View file

@ -873,7 +873,8 @@
"task.database.index.fix": "Effectuer la correction de l'index de la base de données", "task.database.index.fix": "Effectuer la correction de l'index de la base de données",
"task.ocr.image": "Exécute l'OCR d'image pour extraire le texte", "task.ocr.image": "Exécute l'OCR d'image pour extraire le texte",
"task.history.generateDoc": "Exécuter l'historique de GenerateDoc", "task.history.generateDoc": "Exécuter l'historique de GenerateDoc",
"task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données" "task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données",
"task.reload.ui": "Interface utilisateur de rechargement de tâche"
}, },
"_trayMenu": { "_trayMenu": {
"showWindow": "Afficher la fenêtre principale", "showWindow": "Afficher la fenêtre principale",
@ -944,7 +945,7 @@
"55": "Références indexées de [%d] documents", "55": "Références indexées de [%d] documents",
"56": "Réindexation, veuillez attendre que la reconstruction soit terminée avant d'essayer d'ouvrir", "56": "Réindexation, veuillez attendre que la reconstruction soit terminée avant d'essayer d'ouvrir",
"57": "Échec de la création d'une clé temporaire", "57": "Échec de la création d'une clé temporaire",
"58": "Une fois l'index reconstruit, l'interface sera automatiquement rafraîchie ultérieurement...", "58": "TODO",
"59": "Échec de la définition de la liste des ignores de synchronisation", "59": "Échec de la définition de la liste des ignores de synchronisation",
"60": "Échec de la récupération du paquet de mise à jour : %s", "60": "Échec de la récupération du paquet de mise à jour : %s",
"61": "⬆️ Le package d'installation de la nouvelle version est prêt, voulez-vous installer la nouvelle version maintenant ?", "61": "⬆️ Le package d'installation de la nouvelle version est prêt, voulez-vous installer la nouvelle version maintenant ?",

View file

@ -873,7 +873,8 @@
"task.database.index.fix": "執行數據庫索引訂正", "task.database.index.fix": "執行數據庫索引訂正",
"task.ocr.image": "執行圖片 OCR 提取文本", "task.ocr.image": "執行圖片 OCR 提取文本",
"task.history.generateDoc": "執行生成文件歷史", "task.history.generateDoc": "執行生成文件歷史",
"task.database.index.embedBlock": "執行數據庫索引嵌入塊" "task.database.index.embedBlock": "執行數據庫索引嵌入塊",
"task.reload.ui": "執行重載界面"
}, },
"_trayMenu": { "_trayMenu": {
"showWindow": "顯示主窗口", "showWindow": "顯示主窗口",
@ -944,7 +945,7 @@
"55": "已完成索引 [%d] 篇文檔的引用關係", "55": "已完成索引 [%d] 篇文檔的引用關係",
"56": "正在重建索引,請等重建索引完畢後再嘗試打開", "56": "正在重建索引,請等重建索引完畢後再嘗試打開",
"57": "創建臨時金鑰失敗", "57": "創建臨時金鑰失敗",
"58": "重建索引完畢,稍後將自動重新整理介面...", "58": "TODO",
"59": "設置同步忽略列表失敗", "59": "設置同步忽略列表失敗",
"60": "獲取更新包失敗:%s", "60": "獲取更新包失敗:%s",
"61": "⬆️ 新版本安裝包已經準備就緒,是否現在安裝新版本?", "61": "⬆️ 新版本安裝包已經準備就緒,是否現在安裝新版本?",

View file

@ -873,7 +873,8 @@
"task.database.index.fix": "执行数据库索引订正", "task.database.index.fix": "执行数据库索引订正",
"task.ocr.image": "执行图片 OCR 提取文本", "task.ocr.image": "执行图片 OCR 提取文本",
"task.history.generateDoc": "执行生成文件历史", "task.history.generateDoc": "执行生成文件历史",
"task.database.index.embedBlock": "执行数据库索引嵌入块" "task.database.index.embedBlock": "执行数据库索引嵌入块",
"task.reload.ui": "执行重载界面"
}, },
"_trayMenu": { "_trayMenu": {
"showWindow": "显示主窗口", "showWindow": "显示主窗口",
@ -944,7 +945,7 @@
"55": "已完成索引 [%d] 篇文档的引用关系", "55": "已完成索引 [%d] 篇文档的引用关系",
"56": "正在重建索引,请等重建索引完毕后再尝试打开", "56": "正在重建索引,请等重建索引完毕后再尝试打开",
"57": "创建临时密钥失败", "57": "创建临时密钥失败",
"58": "重建索引完毕,稍后将自动刷新界面...", "58": "TODO",
"59": "设置同步忽略列表失败", "59": "设置同步忽略列表失败",
"60": "获取更新包失败:%s", "60": "获取更新包失败:%s",
"61": "⬆️ 新版本安装包已经准备就绪,是否现在安装新版本?", "61": "⬆️ 新版本安装包已经准备就绪,是否现在安装新版本?",

View file

@ -497,6 +497,10 @@ func genTreeID(tree *parse.Tree) {
return return
} }
func ReloadUI() {
task.AppendTask(task.ReloadUI, util.ReloadUI)
}
func FullReindex() { func FullReindex() {
task.PrependTask(task.DatabaseIndexFull, fullReindex) task.PrependTask(task.DatabaseIndexFull, fullReindex)
task.AppendTask(task.DatabaseIndexRef, IndexRefs) task.AppendTask(task.DatabaseIndexRef, IndexRefs)
@ -518,12 +522,6 @@ func fullReindex() {
} }
treenode.SaveBlockTree(true) treenode.SaveBlockTree(true)
LoadFlashcards() LoadFlashcards()
util.PushMsg(Conf.Language(58), 7000)
go func() {
time.Sleep(1 * time.Second)
util.ReloadUI()
}()
} }
func ChangeBoxSort(boxIDs []string) { func ChangeBoxSort(boxIDs []string) {

View file

@ -243,6 +243,7 @@ func RollbackDocHistory(boxID, historyPath string) (err error) {
FullReindex() FullReindex()
IncSync() IncSync()
ReloadUI()
return nil return nil
} }
@ -298,6 +299,7 @@ func RollbackNotebookHistory(historyPath string) (err error) {
FullReindex() FullReindex()
IncSync() IncSync()
ReloadUI()
return nil return nil
} }

View file

@ -441,6 +441,7 @@ func ImportData(zipPath string) (err error) {
IncSync() IncSync()
FullReindex() FullReindex()
ReloadUI()
return return
} }
@ -706,13 +707,6 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
return return
} }
IncSync() IncSync()
sql.WaitForWritingDatabase()
util.PushEndlessProgress(Conf.Language(58))
go func() {
time.Sleep(2 * time.Second)
util.ReloadUI()
}()
} }
debug.FreeOSMemory() debug.FreeOSMemory()
IncSync() IncSync()

View file

@ -197,16 +197,14 @@ func init() {
// util.SetBootDetails(msg) // util.SetBootDetails(msg)
// util.ContextPushMsg(context, msg) // util.ContextPushMsg(context, msg)
//}) //})
eventbus.Subscribe(eventbus.EvtSQLInsertBlocksFTS, func(context map[string]interface{}, current, total, blockCount int, hash string) { eventbus.Subscribe(eventbus.EvtSQLInsertBlocksFTS, func(context map[string]interface{}, blockCount int, hash string) {
if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container { if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
// Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392 // Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392
return return
} }
if (1 > current && 1 == total) || current == total-1 { current := context["current"]
current = total total := context["total"]
}
msg := fmt.Sprintf(Conf.Language(90), current, total, blockCount, hash) msg := fmt.Sprintf(Conf.Language(90), current, total, blockCount, hash)
util.SetBootDetails(msg) util.SetBootDetails(msg)
util.ContextPushMsg(context, msg) util.ContextPushMsg(context, msg)

View file

@ -540,6 +540,7 @@ func checkoutRepo(id string) {
} }
FullReindex() FullReindex()
ReloadUI()
if syncEnabled { if syncEnabled {
func() { func() {
@ -1025,12 +1026,15 @@ func syncRepo(exit, byHand bool) (err error) {
cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939 cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939
if needFullReindex(upsertTrees) { // 改进同步后全量重建索引判断 https://github.com/siyuan-note/siyuan/issues/5764 if needFullReindex(upsertTrees) { // 改进同步后全量重建索引判断 https://github.com/siyuan-note/siyuan/issues/5764
FullReindex() FullReindex()
if !exit {
ReloadUI()
}
return return
} }
incReindex(upserts, removes)
incReindex(upserts, removes)
if !exit { if !exit {
util.ReloadUI() ReloadUI()
} }
elapsed = time.Since(start) elapsed = time.Since(start)

View file

@ -251,11 +251,6 @@ func incReindex(upserts, removes []string) {
treenode.IndexBlockTree(tree) treenode.IndexBlockTree(tree)
sql.UpsertTreeQueue(tree) sql.UpsertTreeQueue(tree)
} }
util.PushStatusBar(Conf.Language(58))
if needPushRemoveProgress || needPushUpsertProgress {
util.PushEndlessProgress(Conf.Language(58))
}
} }
func SetCloudSyncDir(name string) { func SetCloudSyncDir(name string) {

View file

@ -114,7 +114,9 @@ func FlushQueue() {
return return
} }
if err = execOp(op, tx, i, total, context); nil != err { context["current"] = i
context["total"] = total
if err = execOp(op, tx, context); nil != err {
logging.LogErrorf("queue operation failed: %s", err) logging.LogErrorf("queue operation failed: %s", err)
return return
} }
@ -131,12 +133,12 @@ func FlushQueue() {
} }
} }
func execOp(op *dbQueueOperation, tx *sql.Tx, current, total int, context map[string]interface{}) (err error) { func execOp(op *dbQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) {
switch op.action { switch op.action {
case "index": case "index":
err = indexTree(tx, op.box, op.indexPath, current, total, context) err = indexTree(tx, op.box, op.indexPath, context)
case "upsert": case "upsert":
err = upsertTree(tx, op.upsertTree, current, total, context) err = upsertTree(tx, op.upsertTree, context)
case "delete": case "delete":
err = batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath) err = batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath)
case "delete_id": case "delete_id":

View file

@ -51,7 +51,7 @@ const (
FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)" FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
) )
func insertBlocks(tx *sql.Tx, blocks []*Block, current, total int, context map[string]interface{}) (err error) { func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (err error) {
if 1 > len(blocks) { if 1 > len(blocks) {
return return
} }
@ -63,20 +63,20 @@ func insertBlocks(tx *sql.Tx, blocks []*Block, current, total int, context map[s
continue continue
} }
if err = insertBlocks0(tx, bulk, current, total, context); nil != err { if err = insertBlocks0(tx, bulk, context); nil != err {
return return
} }
bulk = []*Block{} bulk = []*Block{}
} }
if 0 < len(bulk) { if 0 < len(bulk) {
if err = insertBlocks0(tx, bulk, current, total, context); nil != err { if err = insertBlocks0(tx, bulk, context); nil != err {
return return
} }
} }
return return
} }
func insertBlocks0(tx *sql.Tx, bulk []*Block, current, total int, context map[string]interface{}) (err error) { func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (err error) {
valueStrings := make([]string, 0, len(bulk)) valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?")) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?"))
hashBuf := bytes.Buffer{} hashBuf := bytes.Buffer{}
@ -129,7 +129,7 @@ func insertBlocks0(tx *sql.Tx, bulk []*Block, current, total int, context map[st
} }
hashBuf.WriteString("fts") hashBuf.WriteString("fts")
evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7] evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
eventbus.Publish(eventbus.EvtSQLInsertBlocksFTS, context, current, total, len(bulk), evtHash) eventbus.Publish(eventbus.EvtSQLInsertBlocksFTS, context, len(bulk), evtHash)
return return
} }
@ -393,24 +393,24 @@ func insertRefs(tx *sql.Tx, tree *parse.Tree) (err error) {
return err return err
} }
func indexTree(tx *sql.Tx, box, p string, current, total int, context map[string]interface{}) (err error) { func indexTree(tx *sql.Tx, box, p string, context map[string]interface{}) (err error) {
tree, err := filesys.LoadTree(box, p, luteEngine) tree, err := filesys.LoadTree(box, p, luteEngine)
if nil != err { if nil != err {
return return
} }
err = insertTree(tx, tree, current, total, context) err = insertTree(tx, tree, context)
return return
} }
func insertTree(tx *sql.Tx, tree *parse.Tree, current, total int, context map[string]interface{}) (err error) { func insertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
blocks, spans, assets, attributes := fromTree(tree.Root, tree) blocks, spans, assets, attributes := fromTree(tree.Root, tree)
refs, fileAnnotationRefs := refsFromTree(tree) refs, fileAnnotationRefs := refsFromTree(tree)
err = insertTree0(tx, tree, current, total, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs) err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs)
return return
} }
func upsertTree(tx *sql.Tx, tree *parse.Tree, current, total int, context map[string]interface{}) (err error) { func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
oldBlockHashes := queryBlockHashes(tree.ID) oldBlockHashes := queryBlockHashes(tree.ID)
blocks, spans, assets, attributes := fromTree(tree.Root, tree) blocks, spans, assets, attributes := fromTree(tree.Root, tree)
newBlockHashes := map[string]string{} newBlockHashes := map[string]string{}
@ -460,16 +460,16 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree, current, total int, context map[st
} }
refs, fileAnnotationRefs := refsFromTree(tree) refs, fileAnnotationRefs := refsFromTree(tree)
if err = insertTree0(tx, tree, current, total, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); nil != err { if err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs); nil != err {
return return
} }
return err return err
} }
func insertTree0(tx *sql.Tx, tree *parse.Tree, current, total int, context map[string]interface{}, func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{},
blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute, blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute,
refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) { refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) {
if err = insertBlocks(tx, blocks, current, total, context); nil != err { if err = insertBlocks(tx, blocks, context); nil != err {
return return
} }

View file

@ -109,6 +109,7 @@ const (
OCRImage = "task.ocr.image" // 图片 OCR 提取文本 OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
ReloadUI = "task.reload.ui" // 重载 UI
) )
func ContainIndexTask() bool { func ContainIndexTask() bool {