Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2025-12-16 22:50:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

🎨 重建索引细节进度推送展现 https://github.com/siyuan-note/siyuan/issues/5890

Browse source
This commit is contained in:
Liang Ding 2022-09-16 10:47:54 +08:00
parent fbe44a0aa0
commit 9a7f61f715
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
10 changed files with 103 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/appearance/langs/en_US.json
View file

@ -864,8 +864,8 @@
"86": "Please configure [Settings - About - Access authorization code]",
"87": "Cannot move to this location",
"88": "Finished parsing [%d] data files, remaining to be processed [%d]",
"89": "TODO",
"90": "TODO",
"89": "Data index [%s] of [%d] block-level elements has been created",
"90": "Search index [%s] for [%d] block-level elements has been created",
"91": "TODO",
"92": "TODO",
"93": "TODO",

4
app/appearance/langs/es_ES.json
View file

@ -864,8 +864,8 @@
"86": "Por favor, configure [Configuración - Acerca de - Código de autorización de acceso]",
"87": "No se puede mover a esta ubicación",
"88": "Se ha terminado de analizar [%d] archivos de datos, quedan por procesar [%d]",
"89": "TODO",
"90": "TODO",
"89": "Se ha creado el índice de datos [%s] de [%d] elementos a nivel de bloque",
"90": "Se ha creado el índice de búsqueda [%s] para [%d] elementos a nivel de bloque",
"91": "TODO",
"92": "TODO",
"93": "TODO",

4
app/appearance/langs/fr_FR.json
View file

@ -864,8 +864,8 @@
"86": "Veuillez configurer [Paramètres - A propos de - Code d'autorisation d'accès]",
"87": "Impossible de se déplacer vers cet endroit",
"88": "Fin de l'analyse des fichiers de données [%d], restant à traiter [%d]",
"89": "TODO",
"90": "TODO",
"89": "L'index de données [%s] de [%d] éléments de niveau bloc a été créé",
"90": "L'index de recherche [%s] pour [%d] éléments de niveau bloc a été créé",
"91": "TODO",
"92": "TODO",
"93": "TODO",

4
app/appearance/langs/zh_CHT.json
View file

@ -864,8 +864,8 @@
"86": "請先配置 [設置 - 關於 - 存取授權碼]",
"87": "無法移動到該位置",
"88": "已完成解析 [%d] 個數據文件，剩餘待處理 [%d]",
"89": "TODO",
"90": "TODO",
"89": "已經建立 [%d] 個塊級元素的數據索引 [%s]",
"90": "已經建立 [%d] 個塊級元素的搜索索引 [%s]",
"91": "TODO",
"92": "TODO",
"93": "TODO",

4
app/appearance/langs/zh_CN.json
View file

@ -865,8 +865,8 @@
"86": "请先配置 [设置 - 关于 - 访问授权码]",
"87": "无法移动到该位置",
"88": "已完成解析 [%d] 个数据文件，剩余待处理 [%d]",
"89": "TODO",
"90": "TODO",
"89": "已经建立 [%d] 个块级元素的数据索引 [%s]",
"90": "已经建立 [%d] 个块级元素的搜索索引 [%s]",
"91": "TODO",
"92": "TODO",
"93": "TODO",

20
kernel/model/index.go
View file

@ -31,6 +31,7 @@ import (
"github.com/88250/lute/parse"
"github.com/dustin/go-humanize"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
@ -145,6 +146,7 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount)
context := map[string]interface{}{sql.CtxPushMsg: sql.CtxPushMsgToStatusBarAndProgress}
i = 0
// 块级行级入库，缓存块
// 这里不能并行插入，因为 SQLite 不支持
@ -164,7 +166,7 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
if nil != err {
continue
}
if err = sql.InsertBlocksSpans(tx, tree); nil != err {
if err = sql.InsertBlocksSpans(tx, tree, context); nil != err {
sql.RollbackTx(tx)
continue
}
@ -194,6 +196,7 @@ func IndexRefs() {
util.SetBootDetails("Resolving refs...")
util.PushEndlessProgress(Conf.Language(54))
context := map[string]interface{}{sql.CtxPushMsg: sql.CtxPushMsgToStatusBarAndProgress}
// 解析并更新引用块
util.SetBootDetails("Resolving ref block content...")
refUnresolvedBlocks := sql.GetRefUnresolvedBlocks() // TODO: v2.2.0 以后移除
@ -234,7 +237,7 @@ func IndexRefs() {
if nil != err {
return
}
sql.InsertBlock(tx, refBlock)
sql.InsertBlock(tx, refBlock, context)
sql.CommitTx(tx)
if 1 < i && 0 == i%64 {
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(53), i, len(refUnresolvedBlocks)-i))
@ -352,3 +355,16 @@ func legacyDynamicRefTreeToStatic(tree *parse.Tree) {
func isLegacyDynamicBlockRef(blockRef *ast.Node) bool {
return nil == blockRef.ChildByType(ast.NodeBlockRefText) && nil == blockRef.ChildByType(ast.NodeBlockRefDynamicText)
}
func init() {
eventbus.Subscribe(sql.EvtSQLInsertBlocks, func(context map[string]interface{}, blockCount int, hash string) {
msg := fmt.Sprintf(Conf.Language(89), blockCount, hash)
util.SetBootDetails(msg)
contextPushMsg(context, msg)
})
eventbus.Subscribe(sql.EvtSQLInsertBlocksFTS, func(context map[string]interface{}, blockCount int, hash string) {
msg := fmt.Sprintf(Conf.Language(90), blockCount, hash)
util.SetBootDetails(msg)
contextPushMsg(context, msg)
})
}

4
kernel/sql/block.go
View file

@ -63,6 +63,6 @@ func updateRootContent(tx *sql.Tx, content, updated, id string) {
cache.RemoveBlockIAL(id)
}
func InsertBlock(tx *sql.Tx, block *Block) (err error) {
return insertBlocks(tx, []*Block{block})
func InsertBlock(tx *sql.Tx, block *Block, context map[string]interface{}) (err error) {
return insertBlocks(tx, []*Block{block}, context)
}

46
kernel/sql/event.go Normal file
View file

@ -0,0 +1,46 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"github.com/siyuan-note/siyuan/kernel/util"
)
const (
EvtSQLInsertBlocks = "sql.insert.blocks"
EvtSQLInsertBlocksFTS = "sql.insert.blocks_fts"
)
const (
CtxPushMsg = "pushMsg"
CtxPushMsgToProgress = iota
CtxPushMsgToStatusBar
CtxPushMsgToStatusBarAndProgress
)
func contextPushMsg(context map[string]interface{}, msg string) {
switch context[CtxPushMsg].(int) {
case CtxPushMsgToProgress:
util.PushEndlessProgress(msg)
case CtxPushMsgToStatusBar:
util.PushStatusBar(msg)
case CtxPushMsgToStatusBarAndProgress:
util.PushStatusBar(msg)
util.PushEndlessProgress(msg)
}
}

3
kernel/sql/queue.go
View file

@ -98,12 +98,13 @@ func flushTreeQueue() {
return
}
context := map[string]interface{}{CtxPushMsg: CtxPushMsgToStatusBar}
boxes := hashset.New()
for _, op := range ops {
switch op.action {
case "upsert":
tree := op.upsertTree
if err = upsertTree(tx, tree); nil != err {
if err = upsertTree(tx, tree, context); nil != err {
logging.LogErrorf("upsert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
}
boxes.Add(op.upsertTree.Box)

37
kernel/sql/upsert.go
View file

@ -17,12 +17,15 @@
package sql
import (
"bytes"
"crypto/sha256"
"database/sql"
"fmt"
"strings"
"github.com/88250/lute/parse"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -34,8 +37,8 @@ func init() {
luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial
}
func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree) (err error) {
if err = insertBlocksSpans(tx, tree); nil != err {
func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
if err = insertBlocksSpans(tx, tree, context); nil != err {
logging.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
}
return
@ -62,7 +65,7 @@ const (
FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
)
func insertBlocks(tx *sql.Tx, blocks []*Block) (err error) {
func insertBlocks(tx *sql.Tx, blocks []*Block, context map[string]interface{}) (err error) {
if 1 > len(blocks) {
return
}
@ -74,22 +77,23 @@ func insertBlocks(tx *sql.Tx, blocks []*Block) (err error) {
continue
}
if err = insertBlocks0(tx, bulk); nil != err {
if err = insertBlocks0(tx, bulk, context); nil != err {
return
}
bulk = []*Block{}
}
if 0 < len(bulk) {
if err = insertBlocks0(tx, bulk); nil != err {
if err = insertBlocks0(tx, bulk, context); nil != err {
return
}
}
return
}
func insertBlocks0(tx *sql.Tx, bulk []*Block) (err error) {
func insertBlocks0(tx *sql.Tx, bulk []*Block, context map[string]interface{}) (err error) {
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?"))
hashBuf := bytes.Buffer{}
for _, b := range bulk {
valueStrings = append(valueStrings, BlocksPlaceholder)
valueArgs = append(valueArgs, b.ID)
@ -113,13 +117,19 @@ func insertBlocks0(tx *sql.Tx, bulk []*Block) (err error) {
valueArgs = append(valueArgs, b.Sort)
valueArgs = append(valueArgs, b.Created)
valueArgs = append(valueArgs, b.Updated)
putBlockCache(b)
hashBuf.WriteString(b.Hash)
}
stmt := fmt.Sprintf(BlocksInsert, strings.Join(valueStrings, ","))
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
}
hashBuf.WriteString("blocks")
evtHash := fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
eventbus.Publish(EvtSQLInsertBlocks, context, len(bulk), evtHash)
stmt = fmt.Sprintf(BlocksFTSInsert, strings.Join(valueStrings, ","))
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
@ -131,6 +141,9 @@ func insertBlocks0(tx *sql.Tx, bulk []*Block) (err error) {
return
}
}
hashBuf.WriteString("fts")
evtHash = fmt.Sprintf("%x", sha256.Sum256(hashBuf.Bytes()))[:7]
eventbus.Publish(EvtSQLInsertBlocksFTS, context, len(bulk), evtHash)
return
}
@ -383,9 +396,9 @@ func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error
return
}
func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree) (err error) {
func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
blocks, spans, assets, attributes := fromTree(tree.Root, tree)
if err = insertBlocks(tx, blocks); nil != err {
if err = insertBlocks(tx, blocks, context); nil != err {
return
}
if err = insertSpans(tx, spans); nil != err {
@ -411,7 +424,7 @@ func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) {
return err
}
func upsertTree(tx *sql.Tx, tree *parse.Tree) (err error) {
func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) {
oldBlockHashes := queryBlockHashes(tree.ID)
blocks, spans, assets, attributes := fromTree(tree.Root, tree)
newBlockHashes := map[string]string{}
@ -457,7 +470,7 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree) (err error) {
return
}
if err = insertBlocks(tx, blocks); nil != err {
if err = insertBlocks(tx, blocks, context); nil != err {
return
}
anchors := map[string]string{}
@ -483,7 +496,7 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree) (err error) {
refBlock.Content = blockContent
}
deleteBlocksByIDs(tx, refIDs)
insertBlocks(tx, refBlocks)
insertBlocks(tx, refBlocks, context)
refs, fileAnnotationRefs := refsFromTree(tree)
if err = insertRefs(tx, refs); nil != err {