diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 7faad9d50..f5389a3a0 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -885,6 +885,7 @@ "task.database.index.fix": "Execute database index fix", "task.ocr.image": "Execute image OCR to extract text", "task.history.generateDoc": "Execute GenerateDoc History", + "task.history.database.index.commit": "Execute history database index commit", "task.database.index.embedBlock": "Execute database index embed block", "task.reload.ui": "Execute reload UI" }, @@ -1089,6 +1090,7 @@ "187": "Unlocking cloud sync directory", "188": "Failed to lock the cloud sync directory, please try again later", "189": "Cloud sync directory is still locked by other devices, please try again later", - "190": "A problem was found while validating the index, which has been automatically fixed" + "190": "A problem was found while validating the index, which has been automatically fixed", + "191": "[%d/%d] Created historical data index" } } diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 29dc2964a..003d733a9 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -885,6 +885,7 @@ "task.database.index.fix": "Ejecutar corrección del índice de la base de datos", "task.ocr.image": "Ejecutar OCR de imagen para extraer texto", "task.history.generateDoc": "Ejecutar Historial GenerateDoc", + "task.history.database.index.commit": "Ejecutar la confirmación del índice de la base de datos del historial", "task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos", "task.reload.ui": "IU de recarga de tareas" }, @@ -1089,6 +1090,7 @@ "187": "Desbloqueando el directorio de sincronización en la nube", "188": "Error al bloquear el directorio de sincronizaci\u00f3n en la nube, int\u00e1ntelo de nuevo m\u00e1s tarde", "189": "El directorio de sincronización en la nube todavía está bloqueado por otros dispositivos, inténtalo de nuevo más tarde", - "190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente" + "190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente", + "191": "[%d/%d] Índice de datos históricos creado" } } diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 304d90ed4..08e5e592f 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -885,6 +885,7 @@ "task.database.index.fix": "Effectuer la correction de l'index de la base de données", "task.ocr.image": "Exécute l'OCR d'image pour extraire le texte", "task.history.generateDoc": "Exécuter l'historique de GenerateDoc", + "task.history.database.index.commit": "Effectuer la validation de l'index de la base de données d'historique", "task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données", "task.reload.ui": "Interface utilisateur de rechargement de tâche" }, @@ -1089,6 +1090,7 @@ "187": "Déverrouillage du répertoire de synchronisation cloud", "188": "Échec du verrouillage du répertoire de synchronisation cloud, veuillez réessayer plus tard", "189": "Le répertoire de synchronisation cloud est toujours verrouillé par d'autres appareils, veuillez réessayer plus tard", - "190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé" + "190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé", + "191": "[%d/%d] Création d'un index de données historiques" } } diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 8e4ff9fef..7bb5976eb 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -885,6 +885,7 @@ "task.database.index.fix": "執行數據庫索引訂正", "task.ocr.image": "執行圖片 OCR 提取文本", "task.history.generateDoc": "執行生成文件歷史", + "task.history.database.index.commit": "執行歷史數據庫索引提交", "task.database.index.embedBlock": "執行數據庫索引嵌入塊", "task.reload.ui": "執行重載界面" }, @@ -1089,6 +1090,7 @@ "187": "正在解鎖雲端同步目錄", "188": "鎖定雲端同步目錄失敗,請稍後再試", "189": "雲端同步目錄還在被其他設備鎖定,請稍後再試", - "190": "校驗索引時發現一個問題,已經自動修復" + "190": "校驗索引時發現一個問題,已經自動修復", + "191": "[%d/%d] 已经建立条历史数据索引" } } diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 470c2ac2e..75411f457 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -885,6 +885,7 @@ "task.database.index.fix": "执行数据库索引订正", "task.ocr.image": "执行图片 OCR 提取文本", "task.history.generateDoc": "执行生成文件历史", + "task.history.database.index.commit": "执行历史数据库索引提交", "task.database.index.embedBlock": "执行数据库索引嵌入块", "task.reload.ui": "执行重载界面" }, @@ -1089,6 +1090,7 @@ "187": "正在解锁云端同步目录", "188": "锁定云端同步目录失败,请稍后再试", "189": "云端同步目录还在被其他设备锁定,请稍后再试", - "190": "校验索引时发现一个问题,已经自动修复" + "190": "校验索引时发现一个问题,已经自动修复", + "191": "[%d/%d] 已经建立条历史数据索引" } } diff --git a/kernel/go.mod b/kernel/go.mod index 5b7add23b..a356f8e2c 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -42,7 +42,7 @@ require ( github.com/shirou/gopsutil/v3 v3.23.1 github.com/siyuan-note/dejavu v0.0.0-20230212031819-32964d704bd2 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 - github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03 + github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a diff --git a/kernel/go.sum b/kernel/go.sum index 5b438ed0a..3e1b8c43f 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -272,6 +272,10 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03 h1:W7nGGluE6sBrFSVkmucGsh2NruleOPsQle7fcAW115o= github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= +github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12 h1:DjZa4jP3J+cZK9BuCXXzY4kr37QXXaCX+IHt8JP8UXQ= +github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= +github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b h1:828lTUW2C0uNiolODqoACu7J8sDUzswD4Xo04mUombg= +github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e h1:i3RKrdrddr4AuaHJtoWYAEVNuR7Y9wIsEqPmuFFbJC4= github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e/go.mod h1:NmpSIVtIGy8eNWapjDIiiCw5+5r5wxC76k40oG+WRXQ= github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea h1:Q6cuN3L4zWR+MwQVdMKeUnfusd+W0LLu1KeqSB3vfdQ= diff --git a/kernel/job/cron.go b/kernel/job/cron.go index 1a7b6c892..0f1b5195d 100644 --- a/kernel/job/cron.go +++ b/kernel/job/cron.go @@ -37,6 +37,7 @@ func StartCron() { go every(3*time.Second, model.FlushUpdateRefTextRenameDocJob) go every(50*time.Millisecond, model.FlushTxJob) go every(util.SQLFlushInterval, sql.FlushTxJob) + go every(util.SQLFlushInterval, sql.FlushHistoryTxJob) go every(10*time.Minute, model.FixIndexJob) go every(10*time.Minute, model.IndexEmbedBlockJob) go every(10*time.Minute, model.CacheVirtualBlockRefJob) diff --git a/kernel/model/history.go b/kernel/model/history.go index f96cdf0e3..074843121 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -70,15 +70,12 @@ func generateDocHistory() { clearOutdatedHistoryDir(historyDir) // 以下部分是老版本的历史数据,不再保留 - for _, box := range Conf.GetBoxes() { historyDir = filepath.Join(util.DataDir, box.ID, ".siyuan", "history") os.RemoveAll(historyDir) } - historyDir = filepath.Join(util.DataDir, "assets", ".siyuan", "history") os.RemoveAll(historyDir) - historyDir = filepath.Join(util.DataDir, ".siyuan", "history") os.RemoveAll(historyDir) } @@ -517,23 +514,7 @@ func clearOutdatedHistoryDir(historyDir string) { //logging.LogInfof("auto removed history dir [%s]", dir) // 清理历史库 - - tx, txErr := sql.BeginHistoryTx() - if nil != txErr { - logging.LogErrorf("begin history tx failed: %s", txErr) - return - } - - p := strings.TrimPrefix(dir, util.HistoryDir) - p = filepath.ToSlash(p[1:]) - if txErr = sql.DeleteHistoriesByPathPrefix(tx, dir); nil != txErr { - logging.LogErrorf("delete history [%s] failed: %s", dir, txErr) - return - } - if txErr = sql.CommitHistoryTx(tx); nil != txErr { - logging.LogErrorf("commit history tx failed: %s", txErr) - return - } + sql.DeleteHistoriesByPathPrefixQueue(dir) } } @@ -678,19 +659,7 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) { }) } - tx, txErr := sql.BeginHistoryTx() - if nil != txErr { - logging.LogErrorf("begin transaction failed: %s", txErr) - return - } - if err := sql.InsertHistories(tx, histories); nil != err { - logging.LogErrorf("insert histories failed: %s", err) - return - } - if err := sql.CommitHistoryTx(tx); nil != err { - logging.LogErrorf("commit transaction failed: %s", err) - return - } + sql.IndexHistoriesQueue(histories) return } diff --git a/kernel/model/index.go b/kernel/model/index.go index f93c07955..ea9ec8282 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -286,4 +286,16 @@ func init() { util.SetBootDetails(msg) util.ContextPushMsg(context, msg) }) + + eventbus.Subscribe(eventbus.EvtSQLInsertHistory, func(context map[string]interface{}) { + if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container { + return + } + + current := context["current"].(int) + 1 + total := context["total"] + msg := fmt.Sprintf(Conf.Language(191), current, total) + util.SetBootDetails(msg) + util.ContextPushMsg(context, msg) + }) } diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 3b8dc0732..07afb726c 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -1142,7 +1142,7 @@ func beginTx() (tx *sql.Tx, err error) { return } -func BeginHistoryTx() (tx *sql.Tx, err error) { +func beginHistoryTx() (tx *sql.Tx, err error) { if tx, err = historyDB.Begin(); nil != err { logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack()) if strings.Contains(err.Error(), "database is locked") { @@ -1152,7 +1152,7 @@ func BeginHistoryTx() (tx *sql.Tx, err error) { return } -func CommitHistoryTx(tx *sql.Tx) (err error) { +func commitHistoryTx(tx *sql.Tx) (err error) { if nil == tx { logging.LogErrorf("tx is nil") return errors.New("tx is nil") diff --git a/kernel/sql/history.go b/kernel/sql/history.go index a939a5f53..f60cc646f 100644 --- a/kernel/sql/history.go +++ b/kernel/sql/history.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" ) @@ -102,7 +103,7 @@ func queryHistory(query string, args ...interface{}) (*sql.Rows, error) { return historyDB.Query(query, args...) } -func DeleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string) (err error) { +func deleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string, context map[string]interface{}) (err error) { stmt := "DELETE FROM histories_fts_case_insensitive WHERE path LIKE ?" if err = execStmtTx(tx, stmt, pathPrefix+"%"); nil != err { return @@ -115,7 +116,7 @@ const ( HistoriesPlaceholder = "(?, ?, ?, ?, ?, ?)" ) -func InsertHistories(tx *sql.Tx, histories []*History) (err error) { +func insertHistories(tx *sql.Tx, histories []*History, context map[string]interface{}) (err error) { if 1 > len(histories) { return } @@ -127,20 +128,20 @@ func InsertHistories(tx *sql.Tx, histories []*History) (err error) { continue } - if err = insertHistories0(tx, bulk); nil != err { + if err = insertHistories0(tx, bulk, context); nil != err { return } bulk = []*History{} } if 0 < len(bulk) { - if err = insertHistories0(tx, bulk); nil != err { + if err = insertHistories0(tx, bulk, context); nil != err { return } } return } -func insertHistories0(tx *sql.Tx, bulk []*History) (err error) { +func insertHistories0(tx *sql.Tx, bulk []*History, context map[string]interface{}) (err error) { valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?")) for _, b := range bulk { @@ -157,5 +158,7 @@ func insertHistories0(tx *sql.Tx, bulk []*History) (err error) { if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err { return } + + eventbus.Publish(eventbus.EvtSQLInsertHistory, context) return } diff --git a/kernel/sql/queue_history.go b/kernel/sql/queue_history.go new file mode 100644 index 000000000..35666f313 --- /dev/null +++ b/kernel/sql/queue_history.go @@ -0,0 +1,139 @@ +// SiYuan - Build Your Eternal Digital Garden +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "database/sql" + "errors" + "fmt" + "runtime/debug" + "sync" + "time" + + "github.com/siyuan-note/eventbus" + "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/task" + "github.com/siyuan-note/siyuan/kernel/util" +) + +var ( + historyOperationQueue []*historyDBQueueOperation + historyDBQueueLock = sync.Mutex{} + + historyTxLock = sync.Mutex{} +) + +type historyDBQueueOperation struct { + inQueueTime time.Time + action string // index/deletePathPrefix + + histories []*History // index + pathPrefix string // deletePathPrefix +} + +func FlushHistoryTxJob() { + task.AppendTask(task.HistoryDatabaseIndexCommit, FlushHistoryQueue) +} + +func FlushHistoryQueue() { + ops := getHistoryOperations() + if 1 > len(ops) { + return + } + + txLock.Lock() + defer txLock.Unlock() + start := time.Now() + + context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} + total := len(ops) + for i, op := range ops { + if util.IsExiting { + return + } + + tx, err := beginHistoryTx() + if nil != err { + return + } + + context["current"] = i + context["total"] = total + if err = execHistoryOp(op, tx, context); nil != err { + tx.Rollback() + logging.LogErrorf("queue operation failed: %s", err) + continue + } + + if err = commitHistoryTx(tx); nil != err { + logging.LogErrorf("commit tx failed: %s", err) + return + } + + if 16 < i && 0 == i%128 { + debug.FreeOSMemory() + } + } + + if 128 < len(ops) { + debug.FreeOSMemory() + } + + elapsed := time.Now().Sub(start).Milliseconds() + if 7000 < elapsed { + logging.LogInfof("database history op tx [%dms]", elapsed) + } +} + +func execHistoryOp(op *historyDBQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) { + switch op.action { + case "index": + err = insertHistories(tx, op.histories, context) + case "deletePathPrefix": + err = deleteHistoriesByPathPrefix(tx, op.pathPrefix, context) + default: + msg := fmt.Sprintf("unknown history operation [%s]", op.action) + logging.LogErrorf(msg) + err = errors.New(msg) + } + return +} + +func DeleteHistoriesByPathPrefixQueue(pathPrefix string) { + historyDBQueueLock.Lock() + defer historyDBQueueLock.Unlock() + + newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "deletePathPrefix", pathPrefix: pathPrefix} + historyOperationQueue = append(historyOperationQueue, newOp) +} + +func IndexHistoriesQueue(histories []*History) { + historyDBQueueLock.Lock() + defer historyDBQueueLock.Unlock() + + newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "index", histories: histories} + historyOperationQueue = append(historyOperationQueue, newOp) +} + +func getHistoryOperations() (ops []*historyDBQueueOperation) { + historyDBQueueLock.Lock() + defer historyDBQueueLock.Unlock() + + ops = historyOperationQueue + historyOperationQueue = nil + return +} diff --git a/kernel/task/queue.go b/kernel/task/queue.go index 86d6b290f..fd657ad10 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -82,16 +82,17 @@ func getCurrentActions() (ret []string) { } const ( - RepoCheckout = "task.repo.checkout" // 从快照中检出 - DatabaseIndexFull = "task.database.index.full" // 重建索引 - DatabaseIndex = "task.database.index" // 数据库索引 - DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交 - DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 - DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 - OCRImage = "task.ocr.image" // 图片 OCR 提取文本 - HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 - DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 - ReloadUI = "task.reload.ui" // 重载 UI + RepoCheckout = "task.repo.checkout" // 从快照中检出 + DatabaseIndexFull = "task.database.index.full" // 重建索引 + DatabaseIndex = "task.database.index" // 数据库索引 + DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交 + DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 + DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 + OCRImage = "task.ocr.image" // 图片 OCR 提取文本 + HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 + HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交 + DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 + ReloadUI = "task.reload.ui" // 重载 UI ) // uniqueActions 描述了唯一的任务,即队列中只能存在一个在执行的任务。 @@ -101,6 +102,7 @@ var uniqueActions = []string{ DatabaseIndexCommit, OCRImage, HistoryGenerateDoc, + HistoryDatabaseIndexCommit, DatabaseIndexEmbedBlock, }