From f581b34ea808651522ec7e6931aec7b96e55ecfa Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 16 Feb 2023 18:42:19 +0800 Subject: [PATCH 01/14] =?UTF-8?q?:art:=20=E5=8E=86=E5=8F=B2=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E7=B4=A2=E5=BC=95=E9=98=9F=E5=88=97=E5=8C=96?= =?UTF-8?q?=20https://github.com/siyuan-note/siyuan/issues/7386?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 4 +- app/appearance/langs/es_ES.json | 4 +- app/appearance/langs/fr_FR.json | 4 +- app/appearance/langs/zh_CHT.json | 4 +- app/appearance/langs/zh_CN.json | 4 +- kernel/go.mod | 2 +- kernel/go.sum | 4 + kernel/job/cron.go | 1 + kernel/model/history.go | 35 +------- kernel/model/index.go | 12 +++ kernel/sql/database.go | 4 +- kernel/sql/history.go | 13 +-- kernel/sql/queue_history.go | 139 +++++++++++++++++++++++++++++++ kernel/task/queue.go | 22 ++--- 14 files changed, 196 insertions(+), 56 deletions(-) create mode 100644 kernel/sql/queue_history.go diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 7faad9d50..f5389a3a0 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -885,6 +885,7 @@ "task.database.index.fix": "Execute database index fix", "task.ocr.image": "Execute image OCR to extract text", "task.history.generateDoc": "Execute GenerateDoc History", + "task.history.database.index.commit": "Execute history database index commit", "task.database.index.embedBlock": "Execute database index embed block", "task.reload.ui": "Execute reload UI" }, @@ -1089,6 +1090,7 @@ "187": "Unlocking cloud sync directory", "188": "Failed to lock the cloud sync directory, please try again later", "189": "Cloud sync directory is still locked by other devices, please try again later", - "190": "A problem was found while validating the index, which has been automatically fixed" + "190": "A problem was found while validating the index, which has been automatically fixed", + "191": "[%d/%d] Created historical data index" } } diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 29dc2964a..003d733a9 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -885,6 +885,7 @@ "task.database.index.fix": "Ejecutar corrección del índice de la base de datos", "task.ocr.image": "Ejecutar OCR de imagen para extraer texto", "task.history.generateDoc": "Ejecutar Historial GenerateDoc", + "task.history.database.index.commit": "Ejecutar la confirmación del índice de la base de datos del historial", "task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos", "task.reload.ui": "IU de recarga de tareas" }, @@ -1089,6 +1090,7 @@ "187": "Desbloqueando el directorio de sincronización en la nube", "188": "Error al bloquear el directorio de sincronizaci\u00f3n en la nube, int\u00e1ntelo de nuevo m\u00e1s tarde", "189": "El directorio de sincronización en la nube todavía está bloqueado por otros dispositivos, inténtalo de nuevo más tarde", - "190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente" + "190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente", + "191": "[%d/%d] Índice de datos históricos creado" } } diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 304d90ed4..08e5e592f 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -885,6 +885,7 @@ "task.database.index.fix": "Effectuer la correction de l'index de la base de données", "task.ocr.image": "Exécute l'OCR d'image pour extraire le texte", "task.history.generateDoc": "Exécuter l'historique de GenerateDoc", + "task.history.database.index.commit": "Effectuer la validation de l'index de la base de données d'historique", "task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données", "task.reload.ui": "Interface utilisateur de rechargement de tâche" }, @@ -1089,6 +1090,7 @@ "187": "Déverrouillage du répertoire de synchronisation cloud", "188": "Échec du verrouillage du répertoire de synchronisation cloud, veuillez réessayer plus tard", "189": "Le répertoire de synchronisation cloud est toujours verrouillé par d'autres appareils, veuillez réessayer plus tard", - "190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé" + "190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé", + "191": "[%d/%d] Création d'un index de données historiques" } } diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 8e4ff9fef..7bb5976eb 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -885,6 +885,7 @@ "task.database.index.fix": "執行數據庫索引訂正", "task.ocr.image": "執行圖片 OCR 提取文本", "task.history.generateDoc": "執行生成文件歷史", + "task.history.database.index.commit": "執行歷史數據庫索引提交", "task.database.index.embedBlock": "執行數據庫索引嵌入塊", "task.reload.ui": "執行重載界面" }, @@ -1089,6 +1090,7 @@ "187": "正在解鎖雲端同步目錄", "188": "鎖定雲端同步目錄失敗,請稍後再試", "189": "雲端同步目錄還在被其他設備鎖定,請稍後再試", - "190": "校驗索引時發現一個問題,已經自動修復" + "190": "校驗索引時發現一個問題,已經自動修復", + "191": "[%d/%d] 已经建立条历史数据索引" } } diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 470c2ac2e..75411f457 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -885,6 +885,7 @@ "task.database.index.fix": "执行数据库索引订正", "task.ocr.image": "执行图片 OCR 提取文本", "task.history.generateDoc": "执行生成文件历史", + "task.history.database.index.commit": "执行历史数据库索引提交", "task.database.index.embedBlock": "执行数据库索引嵌入块", "task.reload.ui": "执行重载界面" }, @@ -1089,6 +1090,7 @@ "187": "正在解锁云端同步目录", "188": "锁定云端同步目录失败,请稍后再试", "189": "云端同步目录还在被其他设备锁定,请稍后再试", - "190": "校验索引时发现一个问题,已经自动修复" + "190": "校验索引时发现一个问题,已经自动修复", + "191": "[%d/%d] 已经建立条历史数据索引" } } diff --git a/kernel/go.mod b/kernel/go.mod index 5b7add23b..a356f8e2c 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -42,7 +42,7 @@ require ( github.com/shirou/gopsutil/v3 v3.23.1 github.com/siyuan-note/dejavu v0.0.0-20230212031819-32964d704bd2 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 - github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03 + github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a diff --git a/kernel/go.sum b/kernel/go.sum index 5b438ed0a..3e1b8c43f 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -272,6 +272,10 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03 h1:W7nGGluE6sBrFSVkmucGsh2NruleOPsQle7fcAW115o= github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= +github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12 h1:DjZa4jP3J+cZK9BuCXXzY4kr37QXXaCX+IHt8JP8UXQ= +github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= +github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b h1:828lTUW2C0uNiolODqoACu7J8sDUzswD4Xo04mUombg= +github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e h1:i3RKrdrddr4AuaHJtoWYAEVNuR7Y9wIsEqPmuFFbJC4= github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e/go.mod h1:NmpSIVtIGy8eNWapjDIiiCw5+5r5wxC76k40oG+WRXQ= github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea h1:Q6cuN3L4zWR+MwQVdMKeUnfusd+W0LLu1KeqSB3vfdQ= diff --git a/kernel/job/cron.go b/kernel/job/cron.go index 1a7b6c892..0f1b5195d 100644 --- a/kernel/job/cron.go +++ b/kernel/job/cron.go @@ -37,6 +37,7 @@ func StartCron() { go every(3*time.Second, model.FlushUpdateRefTextRenameDocJob) go every(50*time.Millisecond, model.FlushTxJob) go every(util.SQLFlushInterval, sql.FlushTxJob) + go every(util.SQLFlushInterval, sql.FlushHistoryTxJob) go every(10*time.Minute, model.FixIndexJob) go every(10*time.Minute, model.IndexEmbedBlockJob) go every(10*time.Minute, model.CacheVirtualBlockRefJob) diff --git a/kernel/model/history.go b/kernel/model/history.go index f96cdf0e3..074843121 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -70,15 +70,12 @@ func generateDocHistory() { clearOutdatedHistoryDir(historyDir) // 以下部分是老版本的历史数据,不再保留 - for _, box := range Conf.GetBoxes() { historyDir = filepath.Join(util.DataDir, box.ID, ".siyuan", "history") os.RemoveAll(historyDir) } - historyDir = filepath.Join(util.DataDir, "assets", ".siyuan", "history") os.RemoveAll(historyDir) - historyDir = filepath.Join(util.DataDir, ".siyuan", "history") os.RemoveAll(historyDir) } @@ -517,23 +514,7 @@ func clearOutdatedHistoryDir(historyDir string) { //logging.LogInfof("auto removed history dir [%s]", dir) // 清理历史库 - - tx, txErr := sql.BeginHistoryTx() - if nil != txErr { - logging.LogErrorf("begin history tx failed: %s", txErr) - return - } - - p := strings.TrimPrefix(dir, util.HistoryDir) - p = filepath.ToSlash(p[1:]) - if txErr = sql.DeleteHistoriesByPathPrefix(tx, dir); nil != txErr { - logging.LogErrorf("delete history [%s] failed: %s", dir, txErr) - return - } - if txErr = sql.CommitHistoryTx(tx); nil != txErr { - logging.LogErrorf("commit history tx failed: %s", txErr) - return - } + sql.DeleteHistoriesByPathPrefixQueue(dir) } } @@ -678,19 +659,7 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) { }) } - tx, txErr := sql.BeginHistoryTx() - if nil != txErr { - logging.LogErrorf("begin transaction failed: %s", txErr) - return - } - if err := sql.InsertHistories(tx, histories); nil != err { - logging.LogErrorf("insert histories failed: %s", err) - return - } - if err := sql.CommitHistoryTx(tx); nil != err { - logging.LogErrorf("commit transaction failed: %s", err) - return - } + sql.IndexHistoriesQueue(histories) return } diff --git a/kernel/model/index.go b/kernel/model/index.go index f93c07955..ea9ec8282 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -286,4 +286,16 @@ func init() { util.SetBootDetails(msg) util.ContextPushMsg(context, msg) }) + + eventbus.Subscribe(eventbus.EvtSQLInsertHistory, func(context map[string]interface{}) { + if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container { + return + } + + current := context["current"].(int) + 1 + total := context["total"] + msg := fmt.Sprintf(Conf.Language(191), current, total) + util.SetBootDetails(msg) + util.ContextPushMsg(context, msg) + }) } diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 3b8dc0732..07afb726c 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -1142,7 +1142,7 @@ func beginTx() (tx *sql.Tx, err error) { return } -func BeginHistoryTx() (tx *sql.Tx, err error) { +func beginHistoryTx() (tx *sql.Tx, err error) { if tx, err = historyDB.Begin(); nil != err { logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack()) if strings.Contains(err.Error(), "database is locked") { @@ -1152,7 +1152,7 @@ func BeginHistoryTx() (tx *sql.Tx, err error) { return } -func CommitHistoryTx(tx *sql.Tx) (err error) { +func commitHistoryTx(tx *sql.Tx) (err error) { if nil == tx { logging.LogErrorf("tx is nil") return errors.New("tx is nil") diff --git a/kernel/sql/history.go b/kernel/sql/history.go index a939a5f53..f60cc646f 100644 --- a/kernel/sql/history.go +++ b/kernel/sql/history.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" ) @@ -102,7 +103,7 @@ func queryHistory(query string, args ...interface{}) (*sql.Rows, error) { return historyDB.Query(query, args...) } -func DeleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string) (err error) { +func deleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string, context map[string]interface{}) (err error) { stmt := "DELETE FROM histories_fts_case_insensitive WHERE path LIKE ?" if err = execStmtTx(tx, stmt, pathPrefix+"%"); nil != err { return @@ -115,7 +116,7 @@ const ( HistoriesPlaceholder = "(?, ?, ?, ?, ?, ?)" ) -func InsertHistories(tx *sql.Tx, histories []*History) (err error) { +func insertHistories(tx *sql.Tx, histories []*History, context map[string]interface{}) (err error) { if 1 > len(histories) { return } @@ -127,20 +128,20 @@ func InsertHistories(tx *sql.Tx, histories []*History) (err error) { continue } - if err = insertHistories0(tx, bulk); nil != err { + if err = insertHistories0(tx, bulk, context); nil != err { return } bulk = []*History{} } if 0 < len(bulk) { - if err = insertHistories0(tx, bulk); nil != err { + if err = insertHistories0(tx, bulk, context); nil != err { return } } return } -func insertHistories0(tx *sql.Tx, bulk []*History) (err error) { +func insertHistories0(tx *sql.Tx, bulk []*History, context map[string]interface{}) (err error) { valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?")) for _, b := range bulk { @@ -157,5 +158,7 @@ func insertHistories0(tx *sql.Tx, bulk []*History) (err error) { if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err { return } + + eventbus.Publish(eventbus.EvtSQLInsertHistory, context) return } diff --git a/kernel/sql/queue_history.go b/kernel/sql/queue_history.go new file mode 100644 index 000000000..35666f313 --- /dev/null +++ b/kernel/sql/queue_history.go @@ -0,0 +1,139 @@ +// SiYuan - Build Your Eternal Digital Garden +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "database/sql" + "errors" + "fmt" + "runtime/debug" + "sync" + "time" + + "github.com/siyuan-note/eventbus" + "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/task" + "github.com/siyuan-note/siyuan/kernel/util" +) + +var ( + historyOperationQueue []*historyDBQueueOperation + historyDBQueueLock = sync.Mutex{} + + historyTxLock = sync.Mutex{} +) + +type historyDBQueueOperation struct { + inQueueTime time.Time + action string // index/deletePathPrefix + + histories []*History // index + pathPrefix string // deletePathPrefix +} + +func FlushHistoryTxJob() { + task.AppendTask(task.HistoryDatabaseIndexCommit, FlushHistoryQueue) +} + +func FlushHistoryQueue() { + ops := getHistoryOperations() + if 1 > len(ops) { + return + } + + txLock.Lock() + defer txLock.Unlock() + start := time.Now() + + context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} + total := len(ops) + for i, op := range ops { + if util.IsExiting { + return + } + + tx, err := beginHistoryTx() + if nil != err { + return + } + + context["current"] = i + context["total"] = total + if err = execHistoryOp(op, tx, context); nil != err { + tx.Rollback() + logging.LogErrorf("queue operation failed: %s", err) + continue + } + + if err = commitHistoryTx(tx); nil != err { + logging.LogErrorf("commit tx failed: %s", err) + return + } + + if 16 < i && 0 == i%128 { + debug.FreeOSMemory() + } + } + + if 128 < len(ops) { + debug.FreeOSMemory() + } + + elapsed := time.Now().Sub(start).Milliseconds() + if 7000 < elapsed { + logging.LogInfof("database history op tx [%dms]", elapsed) + } +} + +func execHistoryOp(op *historyDBQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) { + switch op.action { + case "index": + err = insertHistories(tx, op.histories, context) + case "deletePathPrefix": + err = deleteHistoriesByPathPrefix(tx, op.pathPrefix, context) + default: + msg := fmt.Sprintf("unknown history operation [%s]", op.action) + logging.LogErrorf(msg) + err = errors.New(msg) + } + return +} + +func DeleteHistoriesByPathPrefixQueue(pathPrefix string) { + historyDBQueueLock.Lock() + defer historyDBQueueLock.Unlock() + + newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "deletePathPrefix", pathPrefix: pathPrefix} + historyOperationQueue = append(historyOperationQueue, newOp) +} + +func IndexHistoriesQueue(histories []*History) { + historyDBQueueLock.Lock() + defer historyDBQueueLock.Unlock() + + newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "index", histories: histories} + historyOperationQueue = append(historyOperationQueue, newOp) +} + +func getHistoryOperations() (ops []*historyDBQueueOperation) { + historyDBQueueLock.Lock() + defer historyDBQueueLock.Unlock() + + ops = historyOperationQueue + historyOperationQueue = nil + return +} diff --git a/kernel/task/queue.go b/kernel/task/queue.go index 86d6b290f..fd657ad10 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -82,16 +82,17 @@ func getCurrentActions() (ret []string) { } const ( - RepoCheckout = "task.repo.checkout" // 从快照中检出 - DatabaseIndexFull = "task.database.index.full" // 重建索引 - DatabaseIndex = "task.database.index" // 数据库索引 - DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交 - DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 - DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 - OCRImage = "task.ocr.image" // 图片 OCR 提取文本 - HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 - DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 - ReloadUI = "task.reload.ui" // 重载 UI + RepoCheckout = "task.repo.checkout" // 从快照中检出 + DatabaseIndexFull = "task.database.index.full" // 重建索引 + DatabaseIndex = "task.database.index" // 数据库索引 + DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交 + DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 + DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 + OCRImage = "task.ocr.image" // 图片 OCR 提取文本 + HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 + HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交 + DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 + ReloadUI = "task.reload.ui" // 重载 UI ) // uniqueActions 描述了唯一的任务,即队列中只能存在一个在执行的任务。 @@ -101,6 +102,7 @@ var uniqueActions = []string{ DatabaseIndexCommit, OCRImage, HistoryGenerateDoc, + HistoryDatabaseIndexCommit, DatabaseIndexEmbedBlock, } From 7ee231f134d835a9c36fd479b679e6de453693bb Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 09:35:02 +0800 Subject: [PATCH 02/14] =?UTF-8?q?:art:=20=E8=99=9A=E6=8B=9F=E5=BC=95?= =?UTF-8?q?=E7=94=A8=2010=20=E5=88=86=E9=92=9F=E7=BC=93=E5=AD=98=E8=BF=87?= =?UTF-8?q?=E6=9C=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/virutalref.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index c70ee8ccb..d1060e9e8 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -21,6 +21,7 @@ import ( "regexp" "sort" "strings" + "time" "github.com/88250/gulu" "github.com/88250/lute" @@ -87,7 +88,7 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []s } ret = gulu.Str.RemoveDuplicatedElem(ret) - virtualBlockRefCache.Set(blockID, ret, 1) + virtualBlockRefCache.SetWithTTL(blockID, ret, 1, 10*time.Minute) return } From 8196052c0d3cab2f966032970751c3d5840cc309 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 10:00:03 +0800 Subject: [PATCH 03/14] =?UTF-8?q?:art:=20=E5=BC=80=E5=90=AF=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=90=8C=E6=AD=A5=E6=83=85=E5=86=B5=E4=B8=8B=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E6=97=B6=E5=A6=82=E6=9E=9C=E6=9C=AA=E8=81=94=E7=BD=91?= =?UTF-8?q?=E5=BA=94=E8=AF=A5=E6=8F=90=E7=A4=BA=20Fix=20https://github.com?= =?UTF-8?q?/siyuan-note/siyuan/issues/7389?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/sync.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 6e0d41923..924d69f0a 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -137,6 +137,8 @@ func BootSyncData() { } if !util.IsOnline() { + BootSyncSucc = 1 + util.PushErrMsg(Conf.Language(28), 7000) return } From e08d621da7c91e65184c7af4fe9ef87198530d97 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 10:44:44 +0800 Subject: [PATCH 04/14] =?UTF-8?q?:bug:=20=E7=BC=96=E8=BE=91=E5=99=A8?= =?UTF-8?q?=E4=B8=AD=E8=BE=93=E5=85=A5=20`#`=20=E5=90=8E=E6=A0=87=E7=AD=BE?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E6=8F=90=E7=A4=BA=E4=B8=8D=E5=85=A8=20Fix=20?= =?UTF-8?q?https://github.com/siyuan-note/siyuan/issues/7391?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/graph.go | 2 +- kernel/model/tag.go | 6 +++--- kernel/sql/span.go | 20 +++++++++++++++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/kernel/model/graph.go b/kernel/model/graph.go index 57894978d..7dde5f4a5 100644 --- a/kernel/model/graph.go +++ b/kernel/model/graph.go @@ -205,7 +205,7 @@ func BuildGraph(query string) (boxID string, nodes []*GraphNode, links []*GraphL } func linkTagBlocks(blocks *[]*Block, nodes *[]*GraphNode, links *[]*GraphLink, p string, style map[string]string) { - tagSpans := sql.QueryTagSpans(p, 1024) + tagSpans := sql.QueryTagSpans(p) if 1 > len(tagSpans) { return } diff --git a/kernel/model/tag.go b/kernel/model/tag.go index 6f2590b15..ddd06cf27 100644 --- a/kernel/model/tag.go +++ b/kernel/model/tag.go @@ -41,7 +41,7 @@ func RemoveTag(label string) (err error) { util.PushEndlessProgress(Conf.Language(116)) util.RandomSleep(1000, 2000) - tags := sql.QueryTagSpansByKeyword(label, 102400) + tags := sql.QueryTagSpansByLabel(label) treeBlocks := map[string][]string{} for _, tag := range tags { if blocks, ok := treeBlocks[tag.RootID]; !ok { @@ -126,7 +126,7 @@ func RenameTag(oldLabel, newLabel string) (err error) { util.PushEndlessProgress(Conf.Language(110)) util.RandomSleep(500, 1000) - tags := sql.QueryTagSpansByKeyword(oldLabel, 102400) + tags := sql.QueryTagSpansByLabel(oldLabel) treeBlocks := map[string][]string{} for _, tag := range tags { if blocks, ok := treeBlocks[tag.RootID]; !ok { @@ -310,7 +310,7 @@ func labelBlocksByKeyword(keyword string) (ret map[string]TagBlocks) { func labelTags() (ret map[string]Tags) { ret = map[string]Tags{} - tagSpans := sql.QueryTagSpans("", 10240) + tagSpans := sql.QueryTagSpans("") for _, tagSpan := range tagSpans { label := tagSpan.Content if _, ok := ret[label]; ok { diff --git a/kernel/sql/span.go b/kernel/sql/span.go index d445209d9..94e4623a4 100644 --- a/kernel/sql/span.go +++ b/kernel/sql/span.go @@ -80,8 +80,23 @@ func SelectSpansRawStmt(stmt string, limit int) (ret []*Span) { return } +func QueryTagSpansByLabel(label string) (ret []*Span) { + stmt := "SELECT * FROM spans WHERE type LIKE '%tag%' AND content LIKE '%" + label + "%' GROUP BY block_id" + rows, err := query(stmt) + if nil != err { + logging.LogErrorf("sql query failed: %s", err) + return + } + defer rows.Close() + for rows.Next() { + span := scanSpanRows(rows) + ret = append(ret, span) + } + return +} + func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) { - stmt := "SELECT * FROM spans WHERE type LIKE '%tag%' AND content LIKE '%" + keyword + "%'" + stmt := "SELECT * FROM spans WHERE type LIKE '%tag%' AND content LIKE '%" + keyword + "%' GROUP BY markdown" stmt += " LIMIT " + strconv.Itoa(limit) rows, err := query(stmt) if nil != err { @@ -96,12 +111,11 @@ func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) { return } -func QueryTagSpans(p string, limit int) (ret []*Span) { +func QueryTagSpans(p string) (ret []*Span) { stmt := "SELECT * FROM spans WHERE type LIKE '%tag%'" if "" != p { stmt += " AND path = '" + p + "'" } - stmt += " LIMIT " + strconv.Itoa(limit) rows, err := query(stmt) if nil != err { logging.LogErrorf("sql query failed: %s", err) From 19d6c505350fe033b7452ed6814ce5a5695a85e9 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 10:47:15 +0800 Subject: [PATCH 05/14] =?UTF-8?q?:art:=20=E4=B9=A6=E7=AD=BE=E5=92=8C?= =?UTF-8?q?=E6=A0=87=E7=AD=BE=E9=9D=A2=E6=9D=BF=E5=88=B7=E6=96=B0=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E7=AD=89=E5=BE=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/bookmark.go | 2 -- kernel/model/tag.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/kernel/model/bookmark.go b/kernel/model/bookmark.go index 9352b4e8e..bc4601db5 100644 --- a/kernel/model/bookmark.go +++ b/kernel/model/bookmark.go @@ -158,8 +158,6 @@ func BuildBookmark() (ret *Bookmarks) { WaitForWritingFiles() if !sql.IsEmptyQueue() { sql.WaitForWritingDatabase() - } else { - util.RandomSleep(200, 500) } ret = &Bookmarks{} diff --git a/kernel/model/tag.go b/kernel/model/tag.go index ddd06cf27..172403afe 100644 --- a/kernel/model/tag.go +++ b/kernel/model/tag.go @@ -212,8 +212,6 @@ func BuildTags() (ret *Tags) { WaitForWritingFiles() if !sql.IsEmptyQueue() { sql.WaitForWritingDatabase() - } else { - util.RandomSleep(200, 500) } ret = &Tags{} From fbddab69ebe3754e9472e40fd4c77951e800d35b Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 11:00:04 +0800 Subject: [PATCH 06/14] =?UTF-8?q?:recycle:=20HTTP=20client=20=E5=8D=87?= =?UTF-8?q?=E7=BA=A7=E8=B7=9F=E8=BF=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/assets.go | 2 +- kernel/model/export.go | 4 ++-- kernel/model/liandi.go | 24 ++++++++++++------------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/kernel/model/assets.go b/kernel/model/assets.go index 1591d374c..cc415cc57 100644 --- a/kernel/model/assets.go +++ b/kernel/model/assets.go @@ -356,7 +356,7 @@ func uploadAssets2Cloud(sqlAssets []*sql.Asset, bizType string) (err error) { requestResult := gulu.Ret.NewResult() request := httpclient.NewCloudFileRequest2m() resp, reqErr := request. - SetResult(requestResult). + SetSuccessResult(requestResult). SetFile("file[]", absAsset). SetCookies(&http.Cookie{Name: "symphony", Value: uploadToken}). SetHeader("meta-type", metaType). diff --git a/kernel/model/export.go b/kernel/model/export.go index dfe1ddb36..aa71bf140 100644 --- a/kernel/model/export.go +++ b/kernel/model/export.go @@ -76,7 +76,7 @@ func Export2Liandi(id string) (err error) { result := gulu.Ret.NewResult() request := httpclient.NewCloudRequest30s() resp, getErr := request. - SetResult(result). + SetSuccessResult(result). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Get(util.LiandiServer + "/api/v2/article/update/" + articleId) if nil != getErr { @@ -115,7 +115,7 @@ func Export2Liandi(id string) (err error) { result := gulu.Ret.NewResult() request := httpclient.NewCloudRequest30s() request = request. - SetResult(result). + SetSuccessResult(result). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). SetBody(map[string]interface{}{ "articleTitle": title, diff --git a/kernel/model/liandi.go b/kernel/model/liandi.go index 28b9b1a61..1b471e9b3 100644 --- a/kernel/model/liandi.go +++ b/kernel/model/liandi.go @@ -44,7 +44,7 @@ func StartFreeTrial() (err error) { requestResult := gulu.Ret.NewResult() request := httpclient.NewCloudRequest30s() _, err = request. - SetResult(requestResult). + SetSuccessResult(requestResult). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Post(util.AliyunServer + "/apis/siyuan/user/startFreeTrial") if nil != err { @@ -61,7 +61,7 @@ func DeactivateUser() (err error) { requestResult := gulu.Ret.NewResult() request := httpclient.NewCloudRequest30s() resp, err := request. - SetResult(requestResult). + SetSuccessResult(requestResult). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Post(util.AliyunServer + "/apis/siyuan/user/deactivate") if nil != err { @@ -86,7 +86,7 @@ func SetCloudBlockReminder(id, data string, timed int64) (err error) { payload := map[string]interface{}{"dataId": id, "data": data, "timed": timed} request := httpclient.NewCloudRequest30s() resp, err := request. - SetResult(requestResult). + SetSuccessResult(requestResult). SetBody(payload). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Post(util.AliyunServer + "/apis/siyuan/calendar/setBlockReminder") @@ -119,7 +119,7 @@ func LoadUploadToken() (err error) { requestResult := gulu.Ret.NewResult() request := httpclient.NewCloudRequest30s() resp, err := request. - SetResult(requestResult). + SetSuccessResult(requestResult). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Post(util.AliyunServer + "/apis/siyuan/upload/token") if nil != err { @@ -333,7 +333,7 @@ func RemoveCloudShorthands(ids []string) (err error) { "ids": ids, } resp, err := request. - SetResult(&result). + SetSuccessResult(&result). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). SetBody(body). Post(util.AliyunServer + "/apis/siyuan/inbox/removeCloudShorthands") @@ -361,7 +361,7 @@ func GetCloudShorthand(id string) (ret map[string]interface{}, err error) { result := map[string]interface{}{} request := httpclient.NewCloudRequest30s() resp, err := request. - SetResult(&result). + SetSuccessResult(&result). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Post(util.AliyunServer + "/apis/siyuan/inbox/getCloudShorthand?id=" + id) if nil != err { @@ -392,7 +392,7 @@ func GetCloudShorthands(page int) (result map[string]interface{}, err error) { result = map[string]interface{}{} request := httpclient.NewCloudRequest30s() resp, err := request. - SetResult(&result). + SetSuccessResult(&result). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Post(util.AliyunServer + "/apis/siyuan/inbox/getCloudShorthands?p=" + strconv.Itoa(page)) if nil != err { @@ -429,7 +429,7 @@ func getUser(token string) (*conf.User, error) { result := map[string]interface{}{} request := httpclient.NewCloudRequest30s() _, err := request. - SetResult(&result). + SetSuccessResult(&result). SetBody(map[string]string{"token": token}). Post(util.AliyunServer + "/apis/siyuan/user") if nil != err { @@ -462,7 +462,7 @@ func UseActivationcode(code string) (err error) { requestResult := gulu.Ret.NewResult() request := httpclient.NewCloudRequest30s() _, err = request. - SetResult(requestResult). + SetSuccessResult(requestResult). SetBody(map[string]string{"data": code}). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Post(util.AliyunServer + "/apis/siyuan/useActivationcode") @@ -483,7 +483,7 @@ func CheckActivationcode(code string) (retCode int, msg string) { requestResult := gulu.Ret.NewResult() request := httpclient.NewCloudRequest30s() _, err := request. - SetResult(requestResult). + SetSuccessResult(requestResult). SetBody(map[string]string{"data": code}). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). Post(util.AliyunServer + "/apis/siyuan/checkActivationcode") @@ -503,7 +503,7 @@ func Login(userName, password, captcha string) (ret *gulu.Result, err error) { result := map[string]interface{}{} request := httpclient.NewCloudRequest30s() _, err = request. - SetResult(&result). + SetSuccessResult(&result). SetBody(map[string]string{"userName": userName, "userPassword": password, "captcha": captcha}). Post(util.AliyunServer + "/apis/siyuan/login") if nil != err { @@ -529,7 +529,7 @@ func Login2fa(token, code string) (map[string]interface{}, error) { result := map[string]interface{}{} request := httpclient.NewCloudRequest30s() _, err := request. - SetResult(&result). + SetSuccessResult(&result). SetBody(map[string]string{"twofactorAuthCode": code}). SetHeader("token", token). Post(util.AliyunServer + "/apis/siyuan/login/2fa") From b94c6b48f310858d55a2c1811ed37dab29bf05ee Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 11:20:49 +0800 Subject: [PATCH 07/14] =?UTF-8?q?:art:=20=E7=99=BB=E5=BD=95=E9=93=BE?= =?UTF-8?q?=E6=BB=B4=E8=B4=A6=E5=8F=B7=E5=90=8E=E5=8D=B3=E5=8F=AF=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=20`=E5=88=86=E4=BA=AB=E5=88=B0=E9=93=BE=E6=BB=B4`=20h?= =?UTF-8?q?ttps://github.com/siyuan-note/siyuan/issues/7392?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/src/protyle/breadcrumb/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/protyle/breadcrumb/index.ts b/app/src/protyle/breadcrumb/index.ts index a8daafbda..0c09d37b7 100644 --- a/app/src/protyle/breadcrumb/index.ts +++ b/app/src/protyle/breadcrumb/index.ts @@ -286,7 +286,7 @@ export class Breadcrumb { } } }).element); - if (!needSubscribe("")) { + if (window.siyuan.user) { // 登录链滴账号后即可使用 `分享到链滴` https://github.com/siyuan-note/siyuan/issues/7392 window.siyuan.menus.menu.append(new MenuItem({ label: window.siyuan.languages.share2Liandi, icon: "iconLiandi", From c61e0ef3f2b96fa93fa6c28551e8150a7fb90982 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 11:37:43 +0800 Subject: [PATCH 08/14] =?UTF-8?q?:art:=20=E5=8E=86=E5=8F=B2=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E7=B4=A2=E5=BC=95=E9=98=9F=E5=88=97=E5=8C=96?= =?UTF-8?q?=20https://github.com/siyuan-note/siyuan/issues/7386?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/sql/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 07afb726c..610c13eba 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -230,8 +230,8 @@ func initHistoryDBConnection() { if nil != err { logging.LogFatalf("create database failed: %s", err) } - historyDB.SetMaxIdleConns(1) - historyDB.SetMaxOpenConns(1) + historyDB.SetMaxIdleConns(3) + historyDB.SetMaxOpenConns(3) historyDB.SetConnMaxLifetime(365 * 24 * time.Hour) } From 05c2696e0afe4f2785592858655d25d54d4847c0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 14:23:06 +0800 Subject: [PATCH 09/14] =?UTF-8?q?:zap:=20=E4=BD=BF=E7=94=A8=20AC=20?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E4=BC=98=E5=8C=96=E8=99=9A=E6=8B=9F=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E5=8C=B9=E9=85=8D=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/go.mod | 1 + kernel/go.sum | 2 ++ kernel/model/virutalref.go | 23 ++++++++++++++++------- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index a356f8e2c..5f703f748 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -65,6 +65,7 @@ require ( github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect github.com/aws/aws-sdk-go v1.44.199 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 // indirect github.com/dlclark/regexp2 v1.8.0 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index 3e1b8c43f..df5f3f4d9 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -42,6 +42,8 @@ github.com/aws/aws-sdk-go v1.44.199/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 h1:8yL+85JpbwrIc6m+7N1iYrjn/22z68jwrTIBOJHNe4k= +github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184/go.mod h1:tGWUZLZp9ajsxUOnHmFFLnqnlKXsCn6GReG4jAD59H0= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index d1060e9e8..674d33c72 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -27,6 +27,7 @@ import ( "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" + "github.com/cloudflare/ahocorasick" "github.com/dgraph-io/ristretto" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" @@ -69,17 +70,25 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []s } contentTmp := blockContent + keywordsTmp := keywords if !Conf.Search.CaseSensitive { contentTmp = strings.ToLower(blockContent) - } - for _, keyword := range keywords { - keywordTmp := keyword - if !Conf.Search.CaseSensitive { - keywordTmp = strings.ToLower(keyword) + for i, keyword := range keywordsTmp { + keywordsTmp[i] = strings.ToLower(keyword) } + } - if strings.Contains(contentTmp, keywordTmp) { - ret = append(ret, keyword) + if 1024*1024 < len(contentTmp) { + matcher := ahocorasick.NewStringMatcher(keywords) + hits := matcher.Match([]byte(contentTmp)) + for _, hit := range hits { + ret = append(ret, keywords[hit]) + } + } else { + for _, keyword := range keywordsTmp { + if strings.Contains(contentTmp, keyword) { + ret = append(ret, keyword) + } } } From eabd1b67ed315066991f3c1088c5513a7df8f71d Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 14:43:03 +0800 Subject: [PATCH 10/14] =?UTF-8?q?:zap:=20=E4=BD=BF=E7=94=A8=20AC=20?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E4=BC=98=E5=8C=96=E8=99=9A=E6=8B=9F=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E5=8C=B9=E9=85=8D=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/go.mod | 3 +++ kernel/go.sum | 6 ++++++ kernel/model/virutalref.go | 35 ++++++++++++++++++++++++++--------- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 5f703f748..6f414f46b 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -58,10 +58,13 @@ require ( require ( dmitri.shuralyov.com/font/woff2 v0.0.0-20180220214647-957792cbbdab // indirect + github.com/BobuSumisu/aho-corasick v1.0.3 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/alecthomas/chroma v0.10.0 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0 // indirect + github.com/anknown/darts v0.0.0-20151216065714-83ff685239e6 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect github.com/aws/aws-sdk-go v1.44.199 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index df5f3f4d9..3ff4e5003 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -16,6 +16,8 @@ github.com/88250/pdfcpu v0.3.13 h1:touMWMZkCGalMIbEg9bxYp7rETM+zwb9hXjwhqi4I7Q= github.com/88250/pdfcpu v0.3.13/go.mod h1:S5YT38L/GCjVjmB4PB84PymA1qfopjEhfhTNQilLpv4= github.com/88250/vitess-sqlparser v0.0.0-20210205111146-56a2ded2aba1 h1:48T899JQDwyyRu9yXHePYlPdHtpJfrJEUGBMH3SMBWY= github.com/88250/vitess-sqlparser v0.0.0-20210205111146-56a2ded2aba1/go.mod h1:U3pckKQIgxxkmZjV5yXQjHdGxQK0o/vEZeZ6cQsxfHw= +github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= +github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ConradIrwin/font v0.0.0-20210318200717-ce8d41cc0732 h1:0EDePskeF4vNFCk70ATaFHQzjmwXsk+VImnMJttecNU= github.com/ConradIrwin/font v0.0.0-20210318200717-ce8d41cc0732/go.mod h1:krTLO7JWu6g8RMxG8sl+T1Hf8W93XQacBKJmqFZ2MFY= @@ -33,6 +35,10 @@ github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbf github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0 h1:onfun1RA+KcxaMk1lfrRnwCd1UUuOjJM/lri5eM1qMs= +github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0/go.mod h1:4yg+jNTYlDEzBjhGS96v+zjyA3lfXlFd5CiTLIkPBLI= +github.com/anknown/darts v0.0.0-20151216065714-83ff685239e6 h1:HblK3eJHq54yET63qPCTJnks3loDse5xRmmqHgHzwoI= +github.com/anknown/darts v0.0.0-20151216065714-83ff685239e6/go.mod h1:pbiaLIeYLUbgMY1kwEAdwO6UKD5ZNwdPGQlwokS9fe8= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 674d33c72..2caf4d75a 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -18,6 +18,8 @@ package model import ( "bytes" + goahocorasick "github.com/anknown/ahocorasick" + "github.com/siyuan-note/logging" "regexp" "sort" "strings" @@ -27,7 +29,6 @@ import ( "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" - "github.com/cloudflare/ahocorasick" "github.com/dgraph-io/ristretto" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" @@ -70,22 +71,38 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []s } contentTmp := blockContent - keywordsTmp := keywords + var keywordsTmp [][]rune if !Conf.Search.CaseSensitive { contentTmp = strings.ToLower(blockContent) - for i, keyword := range keywordsTmp { - keywordsTmp[i] = strings.ToLower(keyword) + for _, keyword := range keywords { + keywordsTmp = append(keywordsTmp, []rune(strings.ToLower(keyword))) + } + } else { + for _, keyword := range keywords { + keywordsTmp = append(keywordsTmp, []rune(keyword)) } } if 1024*1024 < len(contentTmp) { - matcher := ahocorasick.NewStringMatcher(keywords) - hits := matcher.Match([]byte(contentTmp)) - for _, hit := range hits { - ret = append(ret, keywords[hit]) + m := goahocorasick.Machine{} + buildErr := m.Build(keywordsTmp) + if nil != buildErr { + logging.LogWarnf("build virtual ref keywords AC matcher failed: %s", buildErr) + for _, keywordRunes := range keywordsTmp { + keyword := string(keywordRunes) + if strings.Contains(contentTmp, keyword) { + ret = append(ret, keyword) + } + } + } else { + hits := m.MultiPatternSearch([]rune(contentTmp), false) + for _, hit := range hits { + ret = append(ret, string(hit.Word)) + } } } else { - for _, keyword := range keywordsTmp { + for _, keywordRunes := range keywordsTmp { + keyword := string(keywordRunes) if strings.Contains(contentTmp, keyword) { ret = append(ret, keyword) } From 4485cc1f1ddc19cc06808b8a7f7c2842ded34357 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 15:01:47 +0800 Subject: [PATCH 11/14] =?UTF-8?q?:art:=20=E8=B0=83=E6=95=B4=E8=99=9A?= =?UTF-8?q?=E6=8B=9F=E5=BC=95=E7=94=A8=E6=90=9C=E7=B4=A2=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E9=A1=B9=E5=90=8E=E7=AB=8B=E5=8D=B3=E9=87=8D=E7=BD=AE=E7=BC=93?= =?UTF-8?q?=E5=AD=98=20https://github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/setting.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kernel/api/setting.go b/kernel/api/setting.go index a69c07528..bc97a587f 100644 --- a/kernel/api/setting.go +++ b/kernel/api/setting.go @@ -214,6 +214,11 @@ func setSearch(c *gin.Context) { } oldCaseSensitive := model.Conf.Search.CaseSensitive + oldVirtualRefName := model.Conf.Search.VirtualRefName + oldVirtualRefAlias := model.Conf.Search.VirtualRefAlias + oldVirtualRefAnchor := model.Conf.Search.VirtualRefAnchor + oldVirtualRefDoc := model.Conf.Search.VirtualRefDoc + oldVirtualRefKeywordsLimit := model.Conf.Search.VirtualRefKeywordsLimit model.Conf.Search = s model.Conf.Save() @@ -221,6 +226,14 @@ func setSearch(c *gin.Context) { if s.CaseSensitive != oldCaseSensitive { model.FullReindex() } + + if oldVirtualRefName != s.VirtualRefName || + oldVirtualRefAlias != s.VirtualRefAlias || + oldVirtualRefAnchor != s.VirtualRefAnchor || + oldVirtualRefDoc != s.VirtualRefDoc || + oldVirtualRefKeywordsLimit != s.VirtualRefKeywordsLimit { + model.CacheVirtualBlockRefJob() + } ret.Data = s } From 5f72a7b2cd7fc07d8707428738efbfd976ca4e72 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 15:17:25 +0800 Subject: [PATCH 12/14] =?UTF-8?q?:art:=20=E8=B0=83=E6=95=B4=E8=99=9A?= =?UTF-8?q?=E6=8B=9F=E5=BC=95=E7=94=A8=E6=90=9C=E7=B4=A2=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E9=A1=B9=E5=90=8E=E7=AB=8B=E5=8D=B3=E9=87=8D=E7=BD=AE=E7=BC=93?= =?UTF-8?q?=E5=AD=98=20https://github.com/siyuan-note/siyuan/issues/7378?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/setting.go | 12 ++++++++++- kernel/model/virutalref.go | 42 ++++++++++++-------------------------- kernel/sql/block_query.go | 4 ++-- 3 files changed, 26 insertions(+), 32 deletions(-) diff --git a/kernel/api/setting.go b/kernel/api/setting.go index bc97a587f..0ff8fb322 100644 --- a/kernel/api/setting.go +++ b/kernel/api/setting.go @@ -91,6 +91,10 @@ func setEditor(c *gin.Context) { editor.KaTexMacros = "{}" } + oldVirtualBlockRef := model.Conf.Editor.VirtualBlockRef + oldVirtualBlockRefInclude := model.Conf.Editor.VirtualBlockRefInclude + oldVirtualBlockRefExclude := model.Conf.Editor.VirtualBlockRefExclude + model.Conf.Editor = editor model.Conf.Save() @@ -98,6 +102,12 @@ func setEditor(c *gin.Context) { model.ChangeHistoryTick(editor.GenerateHistoryInterval) } + if oldVirtualBlockRef != model.Conf.Editor.VirtualBlockRef || + oldVirtualBlockRefInclude != model.Conf.Editor.VirtualBlockRefInclude || + oldVirtualBlockRefExclude != model.Conf.Editor.VirtualBlockRefExclude { + model.ResetVirtualBlockRefCache() + } + ret.Data = model.Conf.Editor } @@ -232,7 +242,7 @@ func setSearch(c *gin.Context) { oldVirtualRefAnchor != s.VirtualRefAnchor || oldVirtualRefDoc != s.VirtualRefDoc || oldVirtualRefKeywordsLimit != s.VirtualRefKeywordsLimit { - model.CacheVirtualBlockRefJob() + model.ResetVirtualBlockRefCache() } ret.Data = s } diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 2caf4d75a..c8401958d 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -18,8 +18,6 @@ package model import ( "bytes" - goahocorasick "github.com/anknown/ahocorasick" - "github.com/siyuan-note/logging" "regexp" "sort" "strings" @@ -29,6 +27,7 @@ import ( "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" + ahocorasick "github.com/BobuSumisu/aho-corasick" "github.com/dgraph-io/ristretto" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" @@ -71,42 +70,22 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []s } contentTmp := blockContent - var keywordsTmp [][]rune + var keywordsTmp []string if !Conf.Search.CaseSensitive { contentTmp = strings.ToLower(blockContent) for _, keyword := range keywords { - keywordsTmp = append(keywordsTmp, []rune(strings.ToLower(keyword))) + keywordsTmp = append(keywordsTmp, strings.ToLower(keyword)) } } else { for _, keyword := range keywords { - keywordsTmp = append(keywordsTmp, []rune(keyword)) + keywordsTmp = append(keywordsTmp, keyword) } } - if 1024*1024 < len(contentTmp) { - m := goahocorasick.Machine{} - buildErr := m.Build(keywordsTmp) - if nil != buildErr { - logging.LogWarnf("build virtual ref keywords AC matcher failed: %s", buildErr) - for _, keywordRunes := range keywordsTmp { - keyword := string(keywordRunes) - if strings.Contains(contentTmp, keyword) { - ret = append(ret, keyword) - } - } - } else { - hits := m.MultiPatternSearch([]rune(contentTmp), false) - for _, hit := range hits { - ret = append(ret, string(hit.Word)) - } - } - } else { - for _, keywordRunes := range keywordsTmp { - keyword := string(keywordRunes) - if strings.Contains(contentTmp, keyword) { - ret = append(ret, keyword) - } - } + trie := ahocorasick.NewTrieBuilder().AddStrings(keywordsTmp).Build() + hits := trie.MatchString(contentTmp) + for _, hit := range hits { + ret = append(ret, hit.MatchString()) } if 1 > len(ret) { @@ -128,6 +107,11 @@ func CacheVirtualBlockRefJob() { virtualBlockRefCache.Set("virtual_ref", keywords, 1) } +func ResetVirtualBlockRefCache() { + virtualBlockRefCache.Clear() + CacheVirtualBlockRefJob() +} + func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool { if !Conf.Editor.VirtualBlockRef { return false diff --git a/kernel/sql/block_query.go b/kernel/sql/block_query.go index a52c5d41f..d32450780 100644 --- a/kernel/sql/block_query.go +++ b/kernel/sql/block_query.go @@ -274,8 +274,8 @@ func queryDocIDsByTitle(title string, excludeIDs []string) (ret []string) { func queryDocTitles() (ret []string) { ret = []string{} - sqlStmt := "SELECT content FROM blocks WHERE type = 'd' LIMIT ?" - rows, err := query(sqlStmt, 10240) + sqlStmt := "SELECT content FROM blocks WHERE type = 'd'" + rows, err := query(sqlStmt) if nil != err { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return From 72631ad0e961a66143d1f6251a8c2d4e481178f6 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 15:25:59 +0800 Subject: [PATCH 13/14] =?UTF-8?q?:zap:=20=E4=BD=BF=E7=94=A8=20AC=20?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E4=BC=98=E5=8C=96=E8=99=9A=E6=8B=9F=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E5=8C=B9=E9=85=8D=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/go.mod | 1 + kernel/go.sum | 2 ++ kernel/model/virutalref.go | 9 +++++---- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 6f414f46b..a82ea5603 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -59,6 +59,7 @@ require ( require ( dmitri.shuralyov.com/font/woff2 v0.0.0-20180220214647-957792cbbdab // indirect github.com/BobuSumisu/aho-corasick v1.0.3 // indirect + github.com/ClarkThan/ahocorasick v0.0.0-20230216061320-bccdb98581a3 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/alecthomas/chroma v0.10.0 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index 3ff4e5003..75efa4f74 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -19,6 +19,8 @@ github.com/88250/vitess-sqlparser v0.0.0-20210205111146-56a2ded2aba1/go.mod h1:U github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ClarkThan/ahocorasick v0.0.0-20230216061320-bccdb98581a3 h1:JsUUbdK1JdhsJt2ebcQFFfeO3jpzh6Vv7pmCph6qeik= +github.com/ClarkThan/ahocorasick v0.0.0-20230216061320-bccdb98581a3/go.mod h1:a3CzWIqeRxiODAscAIfZ4wbFRXxywBrdCwTENVAWB2g= github.com/ConradIrwin/font v0.0.0-20210318200717-ce8d41cc0732 h1:0EDePskeF4vNFCk70ATaFHQzjmwXsk+VImnMJttecNU= github.com/ConradIrwin/font v0.0.0-20210318200717-ce8d41cc0732/go.mod h1:krTLO7JWu6g8RMxG8sl+T1Hf8W93XQacBKJmqFZ2MFY= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index c8401958d..71b0eeec9 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -27,7 +27,7 @@ import ( "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" - ahocorasick "github.com/BobuSumisu/aho-corasick" + ahocorasick "github.com/ClarkThan/ahocorasick" "github.com/dgraph-io/ristretto" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" @@ -82,10 +82,11 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []s } } - trie := ahocorasick.NewTrieBuilder().AddStrings(keywordsTmp).Build() - hits := trie.MatchString(contentTmp) + m := ahocorasick.NewMatcher() + m.BuildWithPatterns(keywordsTmp) + hits := m.Search(contentTmp) for _, hit := range hits { - ret = append(ret, hit.MatchString()) + ret = append(ret, hit) } if 1 > len(ret) { From af2be5a532cced27d8e5d5d79634539f7095c7bf Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 17 Feb 2023 15:26:09 +0800 Subject: [PATCH 14/14] =?UTF-8?q?:zap:=20=E4=BD=BF=E7=94=A8=20AC=20?= =?UTF-8?q?=E7=AE=97=E6=B3=95=E4=BC=98=E5=8C=96=E8=99=9A=E6=8B=9F=E5=BC=95?= =?UTF-8?q?=E7=94=A8=E5=8C=B9=E9=85=8D=E6=80=A7=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/virutalref.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/model/virutalref.go b/kernel/model/virutalref.go index 71b0eeec9..f16defad2 100644 --- a/kernel/model/virutalref.go +++ b/kernel/model/virutalref.go @@ -27,7 +27,7 @@ import ( "github.com/88250/lute" "github.com/88250/lute/ast" "github.com/88250/lute/parse" - ahocorasick "github.com/ClarkThan/ahocorasick" + "github.com/ClarkThan/ahocorasick" "github.com/dgraph-io/ristretto" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql"