From 6141be700a048ec50581036b3fb2039d5cde35cc Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 23 Jan 2023 10:44:22 +0800 Subject: [PATCH 1/8] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/box.go | 18 +----------------- kernel/model/conf.go | 2 +- kernel/model/index.go | 31 +++++++++++++++++++++++++++++-- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/kernel/model/box.go b/kernel/model/box.go index 64caaf5c9..da1b57307 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -339,22 +339,6 @@ func (box *Box) Remove(path string) error { return nil } -func (box *Box) Unindex() { - task.PrependTask(task.DatabaseIndex, unindex, box.ID) -} - -func unindex(boxID string) { - tx, err := sql.BeginTx() - if nil != err { - return - } - sql.RemoveBoxHash(tx, boxID) - sql.DeleteByBoxTx(tx, boxID) - sql.CommitTx(tx) - ids := treenode.RemoveBlockTreesByBoxID(boxID) - RemoveRecentDoc(ids) -} - func (box *Box) ListFiles(path string) (ret []*FileInfo) { fis, _, err := box.Ls(path) if nil != err { @@ -512,7 +496,7 @@ func fullReindex() { openedBoxes := Conf.GetOpenedBoxes() for _, openedBox := range openedBoxes { - openedBox.Index(true) + index(openedBox.ID, true) } IndexRefs() treenode.SaveBlockTree(true) diff --git a/kernel/model/conf.go b/kernel/model/conf.go index 364821337..2af166b92 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -599,7 +599,7 @@ func InitBoxes() { box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 if !initialized { - box.Index(true) + index(box.ID, true) } } diff --git a/kernel/model/index.go b/kernel/model/index.go index 55b7995ba..cb7cb5387 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/sha256" "fmt" + "github.com/siyuan-note/siyuan/kernel/task" "runtime/debug" "sort" "strings" @@ -37,7 +38,32 @@ import ( "github.com/siyuan-note/siyuan/kernel/util" ) -func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) { +func (box *Box) Unindex() { + task.PrependTask(task.DatabaseIndex, unindex, box.ID) +} + +func unindex(boxID string) { + tx, err := sql.BeginTx() + if nil != err { + return + } + sql.RemoveBoxHash(tx, boxID) + sql.DeleteByBoxTx(tx, boxID) + sql.CommitTx(tx) + ids := treenode.RemoveBlockTreesByBoxID(boxID) + RemoveRecentDoc(ids) +} + +func (box *Box) Index(fullRebuildIndex bool) { + task.PrependTask(task.DatabaseIndex, index, box.ID, fullRebuildIndex) +} + +func index(boxID string, fullRebuildIndex bool) { + box := Conf.Box(boxID) + if nil == box { + return + } + defer debug.FreeOSMemory() sql.IndexMode() @@ -59,7 +85,8 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) { luteEngine := NewLute() idTitleMap := map[string]string{} idHashMap := map[string]string{} - + var treeCount int + var treeSize int64 util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files))) i := 0 From 6ed1c5dc1c2bf48f5d5d04916cf461027c0055c9 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 23 Jan 2023 11:20:05 +0800 Subject: [PATCH 2/8] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 2 +- app/appearance/langs/es_ES.json | 2 +- app/appearance/langs/fr_FR.json | 2 +- app/appearance/langs/zh_CHT.json | 2 +- app/appearance/langs/zh_CN.json | 2 +- kernel/api/transaction.go | 13 ------- kernel/model/file.go | 9 ----- kernel/model/index.go | 67 +++----------------------------- kernel/sql/block.go | 4 -- kernel/sql/queue.go | 47 ---------------------- kernel/sql/stat.go | 21 ---------- kernel/sql/upsert.go | 24 ------------ 12 files changed, 11 insertions(+), 184 deletions(-) diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 891fffbb8..03aaf167c 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -983,7 +983,7 @@ "105": "Corrupted data repo have been automatically reset", "106": "Maximum length is limited to 512 characters", "107": "Moving document [%s]", - "108": "Cleaning obsolete indexes...", + "108": "TODO", "109": "Remove reminder completed [%s]", "110": "Renaming...", "111": "Saving document [%s]...", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index cb1ff166a..ee55b5907 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -983,7 +983,7 @@ "105": "El repositorio de datos corruptos se ha restablecido automáticamente", "106": "La longitud máxima está limitada a 512 caracteres", "107": "Moviendo documento [%s]", - "108": "Limpiando índices obsoletos...", + "108": "TODO", "109": "Eliminación de recordatorios completada [%s]", "110": "Renombrar...", "111": "Guardando documento [%s]...", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 982c8bbae..6243e7af2 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -983,7 +983,7 @@ "105": "Le référentiel de données corrompu a été automatiquement réinitialisé", "106": "La longueur maximale est limitée à 512 caractères", "107": "Déplacement du document [%s]", - "108": "Nettoyage des index obsolètes...", + "108": "TODO", "109": "Supprimer le rappel terminé [%s]", "110": "Renommer...", "111": "Enregistrement du document [%s]...", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index b152fdc43..a9862e96b 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -983,7 +983,7 @@ "105": "已經自動重置損壞的數據倉庫", "106": "最大長度限制為 512 字元", "107": "正在移動文檔 [%s]", - "108": "正在清理已過時的索引...", + "108": "TODO", "109": "移除提醒完畢 [%s]", "110": "正在重命名...", "111": "正在保存文檔 [%s]...", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 2434bb55c..00923c276 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -983,7 +983,7 @@ "105": "已经自动重置损坏的数据仓库", "106": "最大长度限制为 512 字符", "107": "正在移动文档 [%s]", - "108": "正在清理已过时的索引...", + "108": "TODO", "109": "移除提醒完毕 [%s]", "110": "正在重命名...", "111": "正在保存文档 [%s]...", diff --git a/kernel/api/transaction.go b/kernel/api/transaction.go index d50a73fdd..a7f3c21d0 100644 --- a/kernel/api/transaction.go +++ b/kernel/api/transaction.go @@ -25,9 +25,7 @@ import ( "github.com/88250/gulu" "github.com/gin-gonic/gin" "github.com/siyuan-note/filelock" - "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/model" - "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -67,17 +65,6 @@ func performTransactions(c *gin.Context) { ret.Code = 1 return } - if nil != err { - tx, txErr := sql.BeginTx() - if nil != txErr { - logging.LogFatalf("transaction failed: %s", txErr) - return - } - sql.ClearBoxHash(tx) - sql.CommitTx(tx) - logging.LogFatalf("transaction failed: %s", err) - return - } ret.Data = transactions diff --git a/kernel/model/file.go b/kernel/model/file.go index b00162de3..2ca36ee88 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -912,15 +912,7 @@ func createTreeTx(tree *parse.Tree) { transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}} err := PerformTransactions(&[]*Transaction{transaction}) if nil != err { - tx, txErr := sql.BeginTx() - if nil != txErr { - logging.LogFatalf("transaction failed: %s", txErr) - return - } - sql.ClearBoxHash(tx) - sql.CommitTx(tx) logging.LogFatalf("transaction failed: %s", err) - return } } @@ -1472,7 +1464,6 @@ func createDoc(boxID, p, title, dom string) (err error) { logging.LogFatalf("transaction failed: %s", txErr) return } - sql.ClearBoxHash(tx) sql.CommitTx(tx) logging.LogFatalf("transaction failed: %s", err) return diff --git a/kernel/model/index.go b/kernel/model/index.go index cb7cb5387..cc9810a36 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -17,12 +17,8 @@ package model import ( - "bytes" - "crypto/sha256" "fmt" - "github.com/siyuan-note/siyuan/kernel/task" "runtime/debug" - "sort" "strings" "time" @@ -34,6 +30,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -47,7 +44,6 @@ func unindex(boxID string) { if nil != err { return } - sql.RemoveBoxHash(tx, boxID) sql.DeleteByBoxTx(tx, boxID) sql.CommitTx(tx) ids := treenode.RemoveBlockTreesByBoxID(boxID) @@ -83,8 +79,6 @@ func index(boxID string, fullRebuildIndex bool) { bootProgressPart := 10.0 / float64(boxLen) / float64(len(files)) luteEngine := NewLute() - idTitleMap := map[string]string{} - idHashMap := map[string]string{} var treeCount int var treeSize int64 util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files))) @@ -105,7 +99,7 @@ func index(boxID string, fullRebuildIndex bool) { } docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL) - if "" == docIAL["updated"] { + if "" == docIAL["updated"] { // 早期的数据可能没有 updated 属性，这里进行订正 updated := util.TimeFromID(tree.Root.ID) tree.Root.SetIALAttr("updated", updated) docIAL["updated"] = updated @@ -117,12 +111,7 @@ func index(boxID string, fullRebuildIndex bool) { util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) treeSize += file.size treeCount++ - // 缓存文档标题，后面做 Path -> HPath 路径映射时需要 - idTitleMap[tree.ID] = tree.Root.IALAttr("title") - // 缓存块树 treenode.IndexBlockTree(tree) - // 缓存 ID-Hash，后面需要用于判断是否要重建库 - idHashMap[tree.ID] = tree.Hash if 1 < i && 0 == i%64 { util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i)) } @@ -131,25 +120,7 @@ func index(boxID string, fullRebuildIndex bool) { box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 - // 检查是否需要重新建库 - util.SetBootDetails("Checking data hashes...") - var ids []string - for id := range idTitleMap { - ids = append(ids, id) - } - sort.Slice(ids, func(i, j int) bool { return ids[i] >= ids[j] }) - buf := bytes.Buffer{} - for _, id := range ids { - hash, _ := idHashMap[id] - buf.WriteString(hash) - util.SetBootDetails("Checking hash " + hash) - } - boxHash := fmt.Sprintf("%x", sha256.Sum256(buf.Bytes())) - - dbBoxHash := sql.GetBoxHash(box.ID) - if boxHash == dbBoxHash { - //logging.LogInfof("use existing database for box [%s]", box.ID) - util.SetBootDetails("Use existing database for notebook " + box.ID) + if !fullRebuildIndex { return } @@ -159,26 +130,8 @@ func index(boxID string, fullRebuildIndex bool) { defer sql.EnableCache() start := time.Now() - if !fullRebuildIndex { - tx, err := sql.BeginTx() - if nil != err { - return - } - sql.PutBoxHash(tx, box.ID, boxHash) - util.SetBootDetails("Cleaning obsolete indexes...") - util.PushEndlessProgress(Conf.Language(108)) - sql.DeleteByBoxTx(tx, box.ID) - if err = sql.CommitTx(tx); nil != err { - return - } - } - bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount) - - context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress} i = 0 - // 块级行级入库，缓存块 - // 这里不能并行插入，因为 SQLite 不支持 for _, file := range files { if file.isdir || !strings.HasSuffix(file.name, ".sy") { continue @@ -191,23 +144,15 @@ func index(boxID string, fullRebuildIndex bool) { } util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(93), util.ShortPathForBootingDisplay(tree.Path))) - tx, err := sql.BeginTx() - if nil != err { - continue - } - if err = sql.InsertBlocksSpans(tx, tree, context); nil != err { - sql.RollbackTx(tx) - continue - } - if err = sql.CommitTx(tx); nil != err { - continue - } + sql.UpsertTreeQueue(tree) if 1 < i && 0 == i%64 { util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(53), i, treeCount-i)) } i++ } + sql.WaitForWritingDatabase() + end := time.Now() elapsed := end.Sub(start).Seconds() logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize))) diff --git a/kernel/sql/block.go b/kernel/sql/block.go index eb0d11db3..fbd7392b6 100644 --- a/kernel/sql/block.go +++ b/kernel/sql/block.go @@ -65,10 +65,6 @@ func updateRootContent(tx *sql.Tx, content, updated, id string) { cache.RemoveBlockIAL(id) } -func InsertBlock(tx *sql.Tx, block *Block, context map[string]interface{}) (err error) { - return insertBlocks(tx, []*Block{block}, context) -} - func UpdateBlockContent(block *Block) { tx, err := BeginTx() if nil != err { diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index 2dd7349c8..30e672505 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -17,15 +17,10 @@ package sql import ( - "bytes" - "crypto/sha256" - "database/sql" - "fmt" "path" "sync" "time" - "github.com/88250/lute/ast" "github.com/88250/lute/parse" "github.com/emirpasic/gods/sets/hashset" "github.com/siyuan-note/eventbus" @@ -136,23 +131,6 @@ func FlushQueue() { if 5000 < elapsed { logging.LogInfof("op tx [%dms]", elapsed) } - - start = time.Now() - tx, err = BeginTx() - if nil != err { - return - } - for _, box := range boxes.Values() { - if !ast.IsNodeIDPattern(box.(string)) { - continue - } - updateBoxHash(tx, box.(string)) - } - CommitTx(tx) - elapsed = time.Now().Sub(start).Milliseconds() - if 1000 < elapsed { - logging.LogInfof("hash tx [%dms]", elapsed) - } } func mergeUpsertTrees() (ops []*treeQueueOperation) { @@ -229,28 +207,3 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) { newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"} operationQueue = append(operationQueue, newOp) } - -func updateBoxHash(tx *sql.Tx, boxID string) { - sum := boxChecksum(boxID) - PutBoxHash(tx, boxID, sum) -} - -func boxChecksum(box string) (ret string) { - rows, err := query("SELECT hash FROM blocks WHERE type = 'd' AND box = ? ORDER BY id DESC", box) - if nil != err { - logging.LogErrorf("sql query failed: %s", err) - return - } - defer rows.Close() - buf := bytes.Buffer{} - for rows.Next() { - var hash string - if err = rows.Scan(&hash); nil != err { - logging.LogErrorf("query scan field failed: %s", err) - return - } - buf.WriteString(hash) - } - ret = fmt.Sprintf("%x", sha256.Sum256(buf.Bytes())) - return -} diff --git a/kernel/sql/stat.go b/kernel/sql/stat.go index e8222d8d5..1b1baa5ec 100644 --- a/kernel/sql/stat.go +++ b/kernel/sql/stat.go @@ -54,27 +54,6 @@ func setDatabaseVer() { CommitTx(tx) } -func ClearBoxHash(tx *sql.Tx) { - stmt := "DELETE FROM stat WHERE `key` LIKE '%_hash'" - execStmtTx(tx, stmt) -} - -func RemoveBoxHash(tx *sql.Tx, box string) { - key := box + "_hash" - stmt := "DELETE FROM stat WHERE `key` = '" + key + "'" - execStmtTx(tx, stmt) -} - -func PutBoxHash(tx *sql.Tx, box, hash string) { - key := box + "_hash" - putStat(tx, key, hash) -} - -func GetBoxHash(box string) string { - key := box + "_hash" - return getStat(key) -} - func putStat(tx *sql.Tx, key, value string) (err error) { stmt := "DELETE FROM stat WHERE `key` = '" + key + "'" if err = execStmtTx(tx, stmt); nil != err { diff --git a/kernel/sql/upsert.go b/kernel/sql/upsert.go index 24d2c7049..380a52475 100644 --- a/kernel/sql/upsert.go +++ b/kernel/sql/upsert.go @@ -36,13 +36,6 @@ func init() { luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial } -func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { - if err = insertBlocksSpans(tx, tree, context); nil != err { - logging.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err) - } - return -} - func InsertRefs(tx *sql.Tx, tree *parse.Tree) { if err := insertRef(tx, tree); nil != err { logging.LogErrorf("insert refs tree [%s] into database failed: %s", tree.Box+tree.Path, err) @@ -395,23 +388,6 @@ func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error return } -func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { - blocks, spans, assets, attributes := fromTree(tree.Root, tree) - if err = insertBlocks(tx, blocks, context); nil != err { - return - } - if err = insertSpans(tx, spans); nil != err { - return - } - if err = insertAssets(tx, assets); nil != err { - return - } - if err = insertAttributes(tx, attributes); nil != err { - return - } - return -} - func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) { refs, fileAnnotationRefs := refsFromTree(tree) if err = insertRefs(tx, refs); nil != err { From a83a08bdb5817f935d5c2b51e4557fe4ae398f3b Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 23 Jan 2023 11:48:35 +0800 Subject: [PATCH 3/8] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 6 +-- app/appearance/langs/es_ES.json | 6 +-- app/appearance/langs/fr_FR.json | 6 +-- app/appearance/langs/zh_CHT.json | 6 +-- app/appearance/langs/zh_CN.json | 6 +-- kernel/model/box.go | 5 ++- kernel/model/conf.go | 2 +- kernel/model/index.go | 70 +++++--------------------------- kernel/model/mount.go | 2 +- 9 files changed, 32 insertions(+), 77 deletions(-) diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 03aaf167c..862bbd8de 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -928,10 +928,10 @@ "50": "Resolving reference [%s]", "51": "Cache content block [%d]", "52": "Every time you open the user guide notebook data will be reset, so please do not save any data in it", - "53": "The index of [%d] documents has been completed, and [%d] remain to be processed", + "53": "TODO", "54": "Indexing references...", "55": "Indexed references of [%d] documents", - "56": "Indexed [%d] documents", + "56": "TODO", "57": "Failed to create temp key", "58": "After the index is rebuilt, the interface will be automatically refreshed later...", "59": "Failed to set sync ignore list", @@ -968,7 +968,7 @@ "90": "Created [%d] of search indexes of block-level elements [%s]", "91": "Reading block tree data...", "92": "Parsing document tree [%s]", - "93": "Indexing document tree [%s]", + "93": "TODO", "94": "Upload failed: %s", "95": "Exiting...", "96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index ee55b5907..fe3347860 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -928,10 +928,10 @@ "50": "Resolviendo referencia [%s]", "51": "Bloque de contenido de la caché [%d]", "52": "Cada vez que abra la guía del usuario se restablecerán los datos del cuaderno, así que no guarde ningún dato en ella", - "53": "Se ha completado el índice de [%d] documentos y quedan [%d] por procesar", + "53": "TODO", "54": "Indexando referencias...", "55": "Referencias indexadas de [%d] documentos", - "56": "Documentos indexados [%d]", + "56": "TODO", "57": "Fallo en la creación de la clave temporal", "58": "Después de reconstruir el índice, la interfaz se actualizará automáticamente más tarde...", "59": " Falló la configuración de sincronización de la lista de ignorados", @@ -968,7 +968,7 @@ "90": "Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]", "91": "Leyendo datos del árbol de bloques...", "92": "Analizando el árbol del documento [%s]", - "93": "Árbol de documentos de indexación [%s]", + "93": "TODO", "94": "Carga fallida: %s", "95": "Saliendo...", "96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 6243e7af2..b8ed8b90b 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -928,10 +928,10 @@ "50": "Résolution des référence [%s]", "51": "Blocage du contenu du cache [%d]", "52": "Chaque fois que vous ouvrez le guide de l'utilisateur, les données du notebook seront réinitialisées, veuillez donc ne pas y enregistrer de données.", - "53": "L'indexation de [%d] documents est terminée, et [%d] restent à traiter.", + "53": "TODO", "54": "Indexation des références...", "55": "Références indexées de [%d] documents", - "56": "Indexé [%d] documents", + "56": "TODO", "57": "Échec de la création d'une clé temporaire", "58": "Une fois l'index reconstruit, l'interface sera automatiquement rafraîchie ultérieurement...", "59": "Échec de la définition de la liste des ignores de synchronisation", @@ -968,7 +968,7 @@ "90": "Création de [%d] index de recherche d'éléments de niveau bloc [%s]", "91": "Lecture des données de l'arborescence des blocs...", "92": "Analyse de l'arborescence du document [%s]", - "93": "Indexation de l'arborescence du document [%s]", + "93": "TODO", "94": "Échec du téléchargement : %s", "95": "Quitter le programme...", "96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index a9862e96b..33cc0fc18 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -928,10 +928,10 @@ "50": "正在解析引用 [%s]", "51": "正在快取內容塊 [%d]", "52": "每次打開使用者指南筆記本資料都會被重置，所以請勿在其中保存任何資料", - "53": "已完成索引 [%d] 篇文檔，剩餘待處理 [%d]", + "53": "TODO", "54": "正在索引引用關係...", "55": "已完成索引 [%d] 篇文檔的引用關係", - "56": "已完成索引 [%d] 篇文檔", + "56": "TODO", "57": "創建臨時金鑰失敗", "58": "重建索引完畢，稍後將自動重新整理介面...", "59": "設置同步忽略列表失敗", @@ -968,7 +968,7 @@ "90": "已經建立 [%d] 個塊級元素的搜索索引 [%s]", "91": "正在讀取塊樹數據...", "92": "正在解析文檔樹 [%s]", - "93": "正在索引文檔樹 [%s]", + "93": "TODO", "94": "上傳失敗：%s", "95": "正在退出...", "96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 00923c276..6b497d064 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -928,10 +928,10 @@ "50": "正在解析引用 [%s]", "51": "正在缓存内容块 [%d]", "52": "每次打开用户指南笔记本数据都会被重置，所以请勿在其中保存任何数据", - "53": "已完成索引 [%d] 篇文档，剩余待处理 [%d]", + "53": "TODO", "54": "正在索引引用关系...", "55": "已完成索引 [%d] 篇文档的引用关系", - "56": "已完成索引 [%d] 篇文档", + "56": "TODO", "57": "创建临时密钥失败", "58": "重建索引完毕，稍后将自动刷新界面...", "59": "设置同步忽略列表失败", @@ -968,7 +968,7 @@ "90": "已经建立 [%d] 个块级元素的搜索索引 [%s]", "91": "正在读取块树数据...", "92": "正在解析文档树 [%s]", - "93": "正在索引文档树 [%s]", + "93": "TODO", "94": "上传失败：%s", "95": "正在退出...", "96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致", diff --git a/kernel/model/box.go b/kernel/model/box.go index da1b57307..d34867bed 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -494,9 +494,12 @@ func fullReindex() { } treenode.InitBlockTree(true) + sql.IndexMode() + defer sql.NormalMode() + openedBoxes := Conf.GetOpenedBoxes() for _, openedBox := range openedBoxes { - index(openedBox.ID, true) + index(openedBox.ID) } IndexRefs() treenode.SaveBlockTree(true) diff --git a/kernel/model/conf.go b/kernel/model/conf.go index 2af166b92..f1631195c 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -599,7 +599,7 @@ func InitBoxes() { box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 if !initialized { - index(box.ID, true) + index(box.ID) } } diff --git a/kernel/model/index.go b/kernel/model/index.go index cc9810a36..277f3ae85 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -18,7 +18,6 @@ package model import ( "fmt" - "runtime/debug" "strings" "time" @@ -50,51 +49,39 @@ func unindex(boxID string) { RemoveRecentDoc(ids) } -func (box *Box) Index(fullRebuildIndex bool) { - task.PrependTask(task.DatabaseIndex, index, box.ID, fullRebuildIndex) +func (box *Box) Index() { + task.PrependTask(task.DatabaseIndex, index, box.ID) } -func index(boxID string, fullRebuildIndex bool) { +func index(boxID string) { box := Conf.Box(boxID) if nil == box { return } - defer debug.FreeOSMemory() - - sql.IndexMode() - defer sql.NormalMode() - - //os.MkdirAll("pprof", 0755) - //cpuProfile, _ := os.Create("pprof/cpu_profile_index") - //pprof.StartCPUProfile(cpuProfile) - //defer pprof.StopCPUProfile() - util.SetBootDetails("Listing files...") files := box.ListFiles("/") boxLen := len(Conf.GetOpenedBoxes()) if 1 > boxLen { boxLen = 1 } - bootProgressPart := 10.0 / float64(boxLen) / float64(len(files)) + bootProgressPart := 30.0 / float64(boxLen) / float64(len(files)) + start := time.Now() luteEngine := NewLute() var treeCount int var treeSize int64 - util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files))) - i := 0 - // 读取并缓存路径映射 + + util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files))) for _, file := range files { if file.isdir || !strings.HasSuffix(file.name, ".sy") { continue } - p := file.path - - tree, err := filesys.LoadTree(box.ID, p, luteEngine) + tree, err := filesys.LoadTree(box.ID, file.path, luteEngine) if nil != err { - logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, p, err) + logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, file.path, err) continue } @@ -106,7 +93,8 @@ func index(boxID string, fullRebuildIndex bool) { writeJSONQueue(tree) } - cache.PutDocIAL(p, docIAL) + cache.PutDocIAL(file.path, docIAL) + sql.UpsertTreeQueue(tree) util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) treeSize += file.size @@ -119,45 +107,9 @@ func index(boxID string, fullRebuildIndex bool) { } box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 - - if !fullRebuildIndex { - return - } - - // 开始重建库 - - sql.DisableCache() - defer sql.EnableCache() - - start := time.Now() - bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount) - i = 0 - for _, file := range files { - if file.isdir || !strings.HasSuffix(file.name, ".sy") { - continue - } - - tree, err := filesys.LoadTree(box.ID, file.path, luteEngine) - if nil != err { - logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, file.path, err) - continue - } - - util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(93), util.ShortPathForBootingDisplay(tree.Path))) - sql.UpsertTreeQueue(tree) - if 1 < i && 0 == i%64 { - util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(53), i, treeCount-i)) - } - i++ - } - - sql.WaitForWritingDatabase() - end := time.Now() elapsed := end.Sub(start).Seconds() logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize))) - - util.PushEndlessProgress(fmt.Sprintf(Conf.Language(56), treeCount)) return } diff --git a/kernel/model/mount.go b/kernel/model/mount.go index e3ac12ef8..fb4e2a4b2 100644 --- a/kernel/model/mount.go +++ b/kernel/model/mount.go @@ -195,7 +195,7 @@ func Mount(boxID string) (alreadyMount bool, err error) { boxConf.Closed = false box.SaveConf(boxConf) - box.Index(false) + box.Index() IndexRefs() // 缓存根一级的文档树展开 ListDocTree(box.ID, "/", Conf.FileTree.Sort) From 72093f7e2e36169794e75632be9e99886549f732 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 23 Jan 2023 18:30:52 +0800 Subject: [PATCH 4/8] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/backlink.go | 15 ++--- kernel/model/box.go | 2 +- kernel/model/file.go | 6 -- kernel/model/index.go | 27 ++------- kernel/model/mount.go | 1 - kernel/sql/queue.go | 121 +++++++++++++++++++++++++++++---------- kernel/task/queue.go | 1 + 7 files changed, 103 insertions(+), 70 deletions(-) diff --git a/kernel/model/backlink.go b/kernel/model/backlink.go index 5272bf8e9..5cb07493a 100644 --- a/kernel/model/backlink.go +++ b/kernel/model/backlink.go @@ -40,24 +40,19 @@ import ( func RefreshBacklink(id string) { WaitForWritingFiles() - tx, err := sql.BeginTx() - if nil != err { - return - } - defer sql.CommitTx(tx) - refs := sql.QueryRefsByDefID(id, false) trees := map[string]*parse.Tree{} for _, ref := range refs { tree := trees[ref.RootID] if nil == tree { - tree, err = loadTreeByBlockID(ref.RootID) - if nil != err { - logging.LogErrorf("refresh tree refs failed: %s", err) + var loadErr error + tree, loadErr = loadTreeByBlockID(ref.RootID) + if nil != loadErr { + logging.LogErrorf("refresh tree refs failed: %s", loadErr) continue } trees[ref.RootID] = tree - sql.UpsertRefs(tx, tree) + sql.UpdateRefsTreeQueue(tree) } } } diff --git a/kernel/model/box.go b/kernel/model/box.go index d34867bed..56f7f3d4e 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -482,6 +482,7 @@ func genTreeID(tree *parse.Tree) { func FullReindex() { task.PrependTask(task.DatabaseIndexFull, fullReindex) + task.AppendTask(task.DatabaseIndexRef, IndexRefs) } func fullReindex() { @@ -501,7 +502,6 @@ func fullReindex() { for _, openedBox := range openedBoxes { index(openedBox.ID) } - IndexRefs() treenode.SaveBlockTree(true) LoadFlashcards() diff --git a/kernel/model/file.go b/kernel/model/file.go index 2ca36ee88..b878e8646 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -1459,12 +1459,6 @@ func createDoc(boxID, p, title, dom string) (err error) { transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}} err = PerformTransactions(&[]*Transaction{transaction}) if nil != err { - tx, txErr := sql.BeginTx() - if nil != txErr { - logging.LogFatalf("transaction failed: %s", txErr) - return - } - sql.CommitTx(tx) logging.LogFatalf("transaction failed: %s", err) return } diff --git a/kernel/model/index.go b/kernel/model/index.go index 277f3ae85..1d8a14bdd 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -39,18 +39,14 @@ func (box *Box) Unindex() { } func unindex(boxID string) { - tx, err := sql.BeginTx() - if nil != err { - return - } - sql.DeleteByBoxTx(tx, boxID) - sql.CommitTx(tx) ids := treenode.RemoveBlockTreesByBoxID(boxID) RemoveRecentDoc(ids) + sql.DeleteBoxQueue(boxID) } func (box *Box) Index() { task.PrependTask(task.DatabaseIndex, index, box.ID) + task.AppendTask(task.DatabaseIndexRef, IndexRefs) } func index(boxID string) { @@ -94,12 +90,12 @@ func index(boxID string) { } cache.PutDocIAL(file.path, docIAL) + treenode.IndexBlockTree(tree) sql.UpsertTreeQueue(tree) util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) treeSize += file.size treeCount++ - treenode.IndexBlockTree(tree) if 1 < i && 0 == i%64 { util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i)) } @@ -128,16 +124,12 @@ func IndexRefs() { for _, refBlock := range refBlocks { refTreeIDs.Add(refBlock.RootID) } + if 0 < refTreeIDs.Size() { luteEngine := NewLute() bootProgressPart := 10.0 / float64(refTreeIDs.Size()) for _, box := range Conf.GetOpenedBoxes() { - tx, err := sql.BeginTx() - if nil != err { - return - } - sql.DeleteRefsByBoxTx(tx, box.ID) - sql.CommitTx(tx) + sql.DeleteBoxRefsQueue(box.ID) files := box.ListFiles("/") i := 0 @@ -163,14 +155,7 @@ func IndexRefs() { continue } - tx, err = sql.BeginTx() - if nil != err { - continue - } - sql.InsertRefs(tx, tree) - if err = sql.CommitTx(tx); nil != err { - continue - } + sql.InsertRefsTreeQueue(tree) if 1 < i && 0 == i%64 { util.PushEndlessProgress(fmt.Sprintf(Conf.Language(55), i)) } diff --git a/kernel/model/mount.go b/kernel/model/mount.go index fb4e2a4b2..4dbaf0344 100644 --- a/kernel/model/mount.go +++ b/kernel/model/mount.go @@ -196,7 +196,6 @@ func Mount(boxID string) (alreadyMount bool, err error) { box.SaveConf(boxConf) box.Index() - IndexRefs() // 缓存根一级的文档树展开 ListDocTree(box.ID, "/", Conf.FileTree.Sort) treenode.SaveBlockTree(false) diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index 30e672505..b048633ca 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -22,7 +22,6 @@ import ( "time" "github.com/88250/lute/parse" - "github.com/emirpasic/gods/sets/hashset" "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/task" @@ -30,19 +29,20 @@ import ( ) var ( - operationQueue []*treeQueueOperation - upsertTreeQueueLock = sync.Mutex{} + operationQueue []*dbQueueOperation + dbQueueLock = sync.Mutex{} txLock = sync.Mutex{} ) -type treeQueueOperation struct { +type dbQueueOperation struct { inQueueTime time.Time - action string // upsert/delete/delete_id/rename + action string // upsert/delete/delete_id/rename/delete_box/delete_box_refs/insert_refs - upsertTree *parse.Tree // upsert + upsertTree *parse.Tree // upsert/insert_refs removeTreeBox, removeTreePath string // delete removeTreeIDBox, removeTreeID string // delete_id + box string // delete_box/delete_box_refs renameTree *parse.Tree // rename renameTreeOldHPath string // rename } @@ -83,8 +83,8 @@ func IsEmptyQueue() bool { } func ClearQueue() { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() + dbQueueLock.Lock() + defer dbQueueLock.Unlock() operationQueue = nil } @@ -103,7 +103,6 @@ func FlushQueue() { } context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} - boxes := hashset.New() for _, op := range ops { switch op.action { case "upsert": @@ -111,17 +110,21 @@ func FlushQueue() { if err = upsertTree(tx, tree, context); nil != err { logging.LogErrorf("upsert tree [%s] into database failed: %s", tree.Box+tree.Path, err) } - boxes.Add(op.upsertTree.Box) case "delete": batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath) - boxes.Add(op.removeTreeBox) case "delete_id": DeleteByRootID(tx, op.removeTreeID) - boxes.Add(op.removeTreeIDBox) case "rename": batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath) updateRootContent(tx, path.Base(op.renameTree.HPath), op.renameTree.Root.IALAttr("updated"), op.renameTree.ID) - boxes.Add(op.renameTree.Box) + case "delete_box": + DeleteByBoxTx(tx, op.box) + case "delete_box_refs": + DeleteRefsByBoxTx(tx, op.box) + case "insert_refs": + InsertRefs(tx, op.upsertTree) + case "update_refs": + UpsertRefs(tx, op.upsertTree) default: logging.LogErrorf("unknown operation [%s]", op.action) } @@ -133,20 +136,76 @@ func FlushQueue() { } } -func mergeUpsertTrees() (ops []*treeQueueOperation) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() +func mergeUpsertTrees() (ops []*dbQueueOperation) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() ops = operationQueue operationQueue = nil return } -func UpsertTreeQueue(tree *parse.Tree) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() +func UpdateRefsTreeQueue(tree *parse.Tree) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() - newOp := &treeQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"} + newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "update_refs"} + for i, op := range operationQueue { + if "update_refs" == op.action && op.upsertTree.ID == tree.ID { + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + +func InsertRefsTreeQueue(tree *parse.Tree) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "insert_refs"} + for i, op := range operationQueue { + if "insert_refs" == op.action && op.upsertTree.ID == tree.ID { + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + +func DeleteBoxRefsQueue(boxID string) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box_refs"} + for i, op := range operationQueue { + if "delete_box_refs" == op.action && op.box == boxID { + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + +func DeleteBoxQueue(boxID string) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box"} + for i, op := range operationQueue { + if "delete_box" == op.action && op.box == boxID { + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + +func UpsertTreeQueue(tree *parse.Tree) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"} for i, op := range operationQueue { if "upsert" == op.action && op.upsertTree.ID == tree.ID { // 相同树则覆盖 operationQueue[i] = newOp @@ -157,10 +216,10 @@ func UpsertTreeQueue(tree *parse.Tree) { } func RenameTreeQueue(tree *parse.Tree, oldHPath string) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() + dbQueueLock.Lock() + defer dbQueueLock.Unlock() - newOp := &treeQueueOperation{ + newOp := &dbQueueOperation{ renameTree: tree, renameTreeOldHPath: oldHPath, inQueueTime: time.Now(), @@ -175,10 +234,10 @@ func RenameTreeQueue(tree *parse.Tree, oldHPath string) { } func RemoveTreeQueue(box, rootID string) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() + dbQueueLock.Lock() + defer dbQueueLock.Unlock() - var tmp []*treeQueueOperation + var tmp []*dbQueueOperation // 将已有的 upsert 操作去重 for _, op := range operationQueue { if "upsert" == op.action && op.upsertTree.ID != rootID { @@ -187,15 +246,15 @@ func RemoveTreeQueue(box, rootID string) { } operationQueue = tmp - newOp := &treeQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"} + newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"} operationQueue = append(operationQueue, newOp) } func RemoveTreePathQueue(treeBox, treePathPrefix string) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() + dbQueueLock.Lock() + defer dbQueueLock.Unlock() - var tmp []*treeQueueOperation + var tmp []*dbQueueOperation // 将已有的 upsert 操作去重 for _, op := range operationQueue { if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) { @@ -204,6 +263,6 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) { } operationQueue = tmp - newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"} + newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"} operationQueue = append(operationQueue, newOp) } diff --git a/kernel/task/queue.go b/kernel/task/queue.go index a95155d59..9609408a1 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -102,6 +102,7 @@ const ( RepoCheckout = "task.repo.checkout" // 从快照中检出 DatabaseIndexFull = "task.database.index.full" // 重建索引 DatabaseIndex = "task.database.index" // 数据库索引队列 + DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 OCRImage = "task.ocr.image" // 图片 OCR 提取文本 HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 From a452015dedcd5283304c5bfc1239ef43e5d98ab2 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 23 Jan 2023 18:44:19 +0800 Subject: [PATCH 5/8] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/import.go | 2 +- kernel/model/index.go | 6 ++++-- kernel/treenode/blocktree.go | 22 ---------------------- 3 files changed, 5 insertions(+), 25 deletions(-) diff --git a/kernel/model/import.go b/kernel/model/import.go index 2edcd3be4..ddafbb75f 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -393,7 +393,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) { continue } - treenode.IndexBlockTree(tree) + treenode.ReindexBlockTree(tree) sql.UpsertTreeQueue(tree) } diff --git a/kernel/model/index.go b/kernel/model/index.go index 1d8a14bdd..db13ac337 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -86,11 +86,13 @@ func index(boxID string) { updated := util.TimeFromID(tree.Root.ID) tree.Root.SetIALAttr("updated", updated) docIAL["updated"] = updated - writeJSONQueue(tree) + if writeErr := filesys.WriteTree(tree); nil != writeErr { + logging.LogErrorf("write tree [%s] failed: %s", tree.Path, writeErr) + } } cache.PutDocIAL(file.path, docIAL) - treenode.IndexBlockTree(tree) + treenode.ReindexBlockTree(tree) sql.UpsertTreeQueue(tree) util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index 0f0f69787..67df96687 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -320,28 +320,6 @@ func ReindexBlockTree(tree *parse.Tree) { blockTreesChanged = true } -func IndexBlockTree(tree *parse.Tree) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering || !n.IsBlock() { - return ast.WalkContinue - } - var parentID string - if nil != n.Parent { - parentID = n.Parent.ID - } - if "" == n.ID { - return ast.WalkContinue - } - blockTrees[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: tree.Root.IALAttr("updated")} - return ast.WalkContinue - }) - - // 新建索引不变更持久化文件，调用处会负责调用 SaveBlockTree() -} - func AutoFlushBlockTree() { for { SaveBlockTree(false) From 76bed6ce6d570388968d549f33a6575b8a1dffc5 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 23 Jan 2023 18:47:23 +0800 Subject: [PATCH 6/8] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/index.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/model/index.go b/kernel/model/index.go index db13ac337..3184461ae 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -117,7 +117,7 @@ func IndexRefs() { start := time.Now() util.SetBootDetails("Resolving refs...") - util.PushEndlessProgress(Conf.Language(54)) + util.PushStatusBar(Conf.Language(54)) // 引用入库 util.SetBootDetails("Indexing refs...") @@ -159,7 +159,7 @@ func IndexRefs() { sql.InsertRefsTreeQueue(tree) if 1 < i && 0 == i%64 { - util.PushEndlessProgress(fmt.Sprintf(Conf.Language(55), i)) + util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) } i++ } From d32dedf00c39ee1e128e064233b4d4c274e75a3d Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 23 Jan 2023 18:53:08 +0800 Subject: [PATCH 7/8] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/index.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/model/index.go b/kernel/model/index.go index 3184461ae..f5b7ab140 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -70,6 +70,8 @@ func index(boxID string) { i := 0 util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files))) + defer util.PushClearProgress() + for _, file := range files { if file.isdir || !strings.HasSuffix(file.name, ".sy") { continue @@ -127,6 +129,7 @@ func IndexRefs() { refTreeIDs.Add(refBlock.RootID) } + i := 0 if 0 < refTreeIDs.Size() { luteEngine := NewLute() bootProgressPart := 10.0 / float64(refTreeIDs.Size()) @@ -134,7 +137,6 @@ func IndexRefs() { sql.DeleteBoxRefsQueue(box.ID) files := box.ListFiles("/") - i := 0 for _, file := range files { if file.isdir || !strings.HasSuffix(file.name, ".sy") { continue @@ -166,6 +168,7 @@ func IndexRefs() { } } logging.LogInfof("resolved refs [%d] in [%dms]", len(refBlocks), time.Now().Sub(start).Milliseconds()) + util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) } func init() { From dcbe1b766da88b9e24ff3d4d0d850fe2bd44adc8 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 24 Jan 2023 11:08:56 +0800 Subject: [PATCH 8/8] =?UTF-8?q?:art:=20=E4=B8=BA=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E6=B7=BB=E5=8A=A0=E5=AF=B9=E5=BA=94=E7=9A=84?= =?UTF-8?q?=E5=9B=BE=E6=A0=87=20https://github.com/siyuan-note/siyuan/issu?= =?UTF-8?q?es/7041?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/sync.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 2702580f1..5834ed9a7 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -150,7 +150,11 @@ func syncData(boot, exit, byHand bool) { msg := fmt.Sprintf(Conf.Language(82), synced) Conf.Sync.Stat = msg Conf.Save() - util.BroadcastByType("main", "syncing", 1, msg, nil) + code := 1 + if nil != err { + code = 2 + } + util.BroadcastByType("main", "syncing", code, msg, nil) return }