From 9a7f61f71585c4f1d83d641d4c8fc39c51301c21 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Fri, 16 Sep 2022 10:47:54 +0800 Subject: [PATCH] =?UTF-8?q?:art:=20=E9=87=8D=E5=BB=BA=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E7=BB=86=E8=8A=82=E8=BF=9B=E5=BA=A6=E6=8E=A8=E9=80=81=E5=B1=95?= =?UTF-8?q?=E7=8E=B0=20https://github.com/siyuan-note/siyuan/issues/5890?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 4 +-- app/appearance/langs/es_ES.json | 4 +-- app/appearance/langs/fr_FR.json | 4 +-- app/appearance/langs/zh_CHT.json | 4 +-- app/appearance/langs/zh_CN.json | 4 +-- kernel/model/index.go | 20 ++++++++++++-- kernel/sql/block.go | 4 +-- kernel/sql/event.go | 46 ++++++++++++++++++++++++++++++++ kernel/sql/queue.go | 3 ++- kernel/sql/upsert.go | 37 ++++++++++++++++--------- 10 files changed, 103 insertions(+), 27 deletions(-) create mode 100644 kernel/sql/event.go diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 49c27fb5d..769cd2fc0 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -864,8 +864,8 @@ "86": "Please configure [Settings - About - Access authorization code]", "87": "Cannot move to this location", "88": "Finished parsing [%d] data files, remaining to be processed [%d]", - "89": "TODO", - "90": "TODO", + "89": "Data index [%s] of [%d] block-level elements has been created", + "90": "Search index [%s] for [%d] block-level elements has been created", "91": "TODO", "92": "TODO", "93": "TODO", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 97393c81f..c705ce403 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -864,8 +864,8 @@ "86": "Por favor, configure [Configuración - Acerca de - Código de autorización de acceso]", "87": "No se puede mover a esta ubicación", "88": "Se ha terminado de analizar [%d] archivos de datos, quedan por procesar [%d]", - "89": "TODO", - "90": "TODO", + "89": "Se ha creado el índice de datos [%s] de [%d] elementos a nivel de bloque", + "90": "Se ha creado el índice de búsqueda [%s] para [%d] elementos a nivel de bloque", "91": "TODO", "92": "TODO", "93": "TODO", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 489e746a9..e6ff84c6d 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -864,8 +864,8 @@ "86": "Veuillez configurer [Paramètres - A propos de - Code d'autorisation d'accès]", "87": "Impossible de se déplacer vers cet endroit", "88": "Fin de l'analyse des fichiers de données [%d], restant à traiter [%d]", - "89": "TODO", - "90": "TODO", + "89": "L'index de données [%s] de [%d] éléments de niveau bloc a été créé", + "90": "L'index de recherche [%s] pour [%d] éléments de niveau bloc a été créé", "91": "TODO", "92": "TODO", "93": "TODO", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index a27735e73..9a13c1833 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -864,8 +864,8 @@ "86": "請先配置 [設置 - 關於 - 存取授權碼]", "87": "無法移動到該位置", "88": "已完成解析 [%d] 個數據文件，剩餘待處理 [%d]", - "89": "TODO", - "90": "TODO", + "89": "已經建立 [%d] 個塊級元素的數據索引 [%s]", + "90": "已經建立 [%d] 個塊級元素的搜索索引 [%s]", "91": "TODO", "92": "TODO", "93": "TODO", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 37004e38d..4bc33c745 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -865,8 +865,8 @@ "86": "请先配置 [设置 - 关于 - 访问授权码]", "87": "无法移动到该位置", "88": "已完成解析 [%d] 个数据文件，剩余待处理 [%d]", - "89": "TODO", - "90": "TODO", + "89": "已经建立 [%d] 个块级元素的数据索引 [%s]", + "90": "已经建立 [%d] 个块级元素的搜索索引 [%s]", "91": "TODO", "92": "TODO", "93": "TODO", diff --git a/kernel/model/index.go b/kernel/model/index.go index b78da46a3..fb22284e5 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -31,6 +31,7 @@ import ( "github.com/88250/lute/parse" "github.com/dustin/go-humanize" "github.com/emirpasic/gods/sets/hashset" + "github.com/siyuan-note/eventbus" "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/cache" @@ -145,6 +146,7 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) { bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount) + context := map[string]interface{}{sql.CtxPushMsg: sql.CtxPushMsgToStatusBarAndProgress} i = 0 // 块级行级入库，缓存块 // 这里不能并行插入，因为 SQLite 不支持 @@ -164,7 +166,7 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) { if nil != err { continue } - if err = sql.InsertBlocksSpans(tx, tree); nil != err { + if err = sql.InsertBlocksSpans(tx, tree, context); nil != err { sql.RollbackTx(tx) continue } @@ -194,6 +196,7 @@ func IndexRefs() { util.SetBootDetails("Resolving refs...") util.PushEndlessProgress(Conf.Language(54)) + context := map[string]interface{}{sql.CtxPushMsg: sql.CtxPushMsgToStatusBarAndProgress} // 解析并更新引用块 util.SetBootDetails("Resolving ref block content...") refUnresolvedBlocks := sql.GetRefUnresolvedBlocks() // TODO: v2.2.0 以后移除 @@ -234,7 +237,7 @@ func IndexRefs() { if nil != err { return } - sql.InsertBlock(tx, refBlock) + sql.InsertBlock(tx, refBlock, context) sql.CommitTx(tx) if 1 < i && 0 == i%64 { util.PushEndlessProgress(fmt.Sprintf(Conf.Language(53), i, len(refUnresolvedBlocks)-i)) @@ -352,3 +355,16 @@ func legacyDynamicRefTreeToStatic(tree *parse.Tree) { func isLegacyDynamicBlockRef(blockRef *ast.Node) bool { return nil == blockRef.ChildByType(ast.NodeBlockRefText) && nil == blockRef.ChildByType(ast.NodeBlockRefDynamicText) } + +func init() { + eventbus.Subscribe(sql.EvtSQLInsertBlocks, func(context map[string]interface{}, blockCount int, hash string) { + msg := fmt.Sprintf(Conf.Language(89), blockCount, hash) + util.SetBootDetails(msg) + contextPushMsg(context, msg) + }) + eventbus.Subscribe(sql.EvtSQLInsertBlocksFTS, func(context map[string]interface{}, blockCount int, hash string) { + msg := fmt.Sprintf(Conf.Language(90), blockCount, hash) + util.SetBootDetails(msg) + contextPushMsg(context, msg) + }) +} diff --git a/kernel/sql/block.go b/kernel/sql/block.go index 7da59c2bb..6a39bd264 100644 --- a/kernel/sql/block.go +++ b/kernel/sql/block.go @@ -63,6 +63,6 @@ func updateRootContent(tx *sql.Tx, content, updated, id string) { cache.RemoveBlockIAL(id) } -func InsertBlock(tx *sql.Tx, block *Block) (err error) { - return insertBlocks(tx, []*Block{block}) +func InsertBlock(tx *sql.Tx, block *Block, context map[string]interface{}) (err error) { + return insertBlocks(tx, []*Block{block}, context) } diff --git a/kernel/sql/event.go b/kernel/sql/event.go new file mode 100644 index 000000000..72820b3cd --- /dev/null +++ b/kernel/sql/event.go @@ -0,0 +1,46 @@ +// SiYuan - Build Your Eternal Digital Garden +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "github.com/siyuan-note/siyuan/kernel/util" +) + +const ( + EvtSQLInsertBlocks = "sql.insert.blocks" + EvtSQLInsertBlocksFTS = "sql.insert.blocks_fts" +) + +const ( + CtxPushMsg = "pushMsg" + + CtxPushMsgToProgress = iota + CtxPushMsgToStatusBar + CtxPushMsgToStatusBarAndProgress +) + +func contextPushMsg(context map[string]interface{}, msg string) { + switch context[CtxPushMsg].(int) { + case CtxPushMsgToProgress: + util.PushEndlessProgress(msg) + case CtxPushMsgToStatusBar: + util.PushStatusBar(msg) + case CtxPushMsgToStatusBarAndProgress: + util.PushStatusBar(msg) + util.PushEndlessProgress(msg) + } +} diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index cacc2024b..eed5efeb3 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -98,12 +98,13 @@ func flushTreeQueue() { return } + context := map[string]interface{}{CtxPushMsg: CtxPushMsgToStatusBar} boxes := hashset.New() for _, op := range ops { switch op.action { case "upsert": tree := op.upsertTree - if err = upsertTree(tx, tree); nil != err { + if err = upsertTree(tx, tree, context); nil != err { logging.LogErrorf("upsert tree [%s] into database failed: %s", tree.Box+tree.Path, err) } boxes.Add(op.upsertTree.Box) diff --git a/kernel/sql/upsert.go b/kernel/sql/upsert.go index 97489c71c..e5f854f15 100644 --- a/kernel/sql/upsert.go +++ b/kernel/sql/upsert.go @@ -17,12 +17,15 @@ package sql import ( + "bytes" + "crypto/sha256" "database/sql" "fmt" "strings" "github.com/88250/lute/parse" "github.com/emirpasic/gods/sets/hashset" + "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" @@ -34,8 +37,8 @@ func init() { luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial } -func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree) (err error) { - if err = insertBlocksSpans(tx, tree); nil != err { +func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { + if err = insertBlocksSpans(tx, tree, context); nil != err { logging.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err) } return @@ -62,7 +65,7 @@ const ( FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)" ) -func insertBlocks(tx *sql.Tx, blocks []*Block) (err error) { +func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (err error) { if 1 > len(blocks) { return } @@ -74,22 +77,23 @@ func insertBlocks(tx *sql.Tx, blocks []*Block) (err error) { continue } - if err = insertBlocks0(tx, bulk); nil != err { + if err = insertBlocks0(tx, bulk, context); nil != err { return } bulk = []*Block{} } if 0 < len(bulk) { - if err = insertBlocks0(tx, bulk); nil != err { + if err = insertBlocks0(tx, bulk, context); nil != err { return } } return } -func insertBlocks0(tx *sql.Tx, bulk []*Block) (err error) { +func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (err error) { valueStrings := make([]string, 0, len(bulk)) valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?")) + hashBuf := bytes.Buffer{} for _, b := range bulk { valueStrings = append(valueStrings, BlocksPlaceholder) valueArgs = append(valueArgs, b.ID) @@ -113,13 +117,19 @@ func insertBlocks0(tx *sql.Tx, bulk []*Block) (err error) { valueArgs = append(valueArgs, b.Sort) valueArgs = append(valueArgs, b.Created) valueArgs = append(valueArgs, b.Updated) - putBlockCache(b) + + hashBuf.WriteString(b.Hash) } + stmt := fmt.Sprintf(BlocksInsert, strings.Join(valueStrings, ",")) if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err { return } + hashBuf.WriteString("blocks") + evtHash := fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7] + eventbus.Publish(EvtSQLInsertBlocks, context, len(bulk), evtHash) + stmt = fmt.Sprintf(BlocksFTSInsert, strings.Join(valueStrings, ",")) if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err { return @@ -131,6 +141,9 @@ func insertBlocks0(tx *sql.Tx, bulk []*Block) (err error) { return } } + hashBuf.WriteString("fts") + evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7] + eventbus.Publish(EvtSQLInsertBlocksFTS, context, len(bulk), evtHash) return } @@ -383,9 +396,9 @@ func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error return } -func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree) (err error) { +func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { blocks, spans, assets, attributes := fromTree(tree.Root, tree) - if err = insertBlocks(tx, blocks); nil != err { + if err = insertBlocks(tx, blocks, context); nil != err { return } if err = insertSpans(tx, spans); nil != err { @@ -411,7 +424,7 @@ func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) { return err } -func upsertTree(tx *sql.Tx, tree *parse.Tree) (err error) { +func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { oldBlockHashes := queryBlockHashes(tree.ID) blocks, spans, assets, attributes := fromTree(tree.Root, tree) newBlockHashes := map[string]string{} @@ -457,7 +470,7 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree) (err error) { return } - if err = insertBlocks(tx, blocks); nil != err { + if err = insertBlocks(tx, blocks, context); nil != err { return } anchors := map[string]string{} @@ -483,7 +496,7 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree) (err error) { refBlock.Content = blockContent } deleteBlocksByIDs(tx, refIDs) - insertBlocks(tx, refBlocks) + insertBlocks(tx, refBlocks, context) refs, fileAnnotationRefs := refsFromTree(tree) if err = insertRefs(tx, refs); nil != err {