diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 891fffbb8..862bbd8de 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -928,10 +928,10 @@ "50": "Resolving reference [%s]", "51": "Cache content block [%d]", "52": "Every time you open the user guide notebook data will be reset, so please do not save any data in it", - "53": "The index of [%d] documents has been completed, and [%d] remain to be processed", + "53": "TODO", "54": "Indexing references...", "55": "Indexed references of [%d] documents", - "56": "Indexed [%d] documents", + "56": "TODO", "57": "Failed to create temp key", "58": "After the index is rebuilt, the interface will be automatically refreshed later...", "59": "Failed to set sync ignore list", @@ -968,7 +968,7 @@ "90": "Created [%d] of search indexes of block-level elements [%s]", "91": "Reading block tree data...", "92": "Parsing document tree [%s]", - "93": "Indexing document tree [%s]", + "93": "TODO", "94": "Upload failed: %s", "95": "Exiting...", "96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data", @@ -983,7 +983,7 @@ "105": "Corrupted data repo have been automatically reset", "106": "Maximum length is limited to 512 characters", "107": "Moving document [%s]", - "108": "Cleaning obsolete indexes...", + "108": "TODO", "109": "Remove reminder completed [%s]", "110": "Renaming...", "111": "Saving document [%s]...", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index cb1ff166a..fe3347860 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -928,10 +928,10 @@ "50": "Resolviendo referencia [%s]", "51": "Bloque de contenido de la caché [%d]", "52": "Cada vez que abra la guía del usuario se restablecerán los datos del cuaderno, así que no guarde ningún dato en ella", - "53": "Se ha completado el índice de [%d] documentos y quedan [%d] por procesar", + "53": "TODO", "54": "Indexando referencias...", "55": "Referencias indexadas de [%d] documentos", - "56": "Documentos indexados [%d]", + "56": "TODO", "57": "Fallo en la creación de la clave temporal", "58": "Después de reconstruir el índice, la interfaz se actualizará automáticamente más tarde...", "59": " Falló la configuración de sincronización de la lista de ignorados", @@ -968,7 +968,7 @@ "90": "Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]", "91": "Leyendo datos del árbol de bloques...", "92": "Analizando el árbol del documento [%s]", - "93": "Árbol de documentos de indexación [%s]", + "93": "TODO", "94": "Carga fallida: %s", "95": "Saliendo...", "96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube", @@ -983,7 +983,7 @@ "105": "El repositorio de datos corruptos se ha restablecido automáticamente", "106": "La longitud máxima está limitada a 512 caracteres", "107": "Moviendo documento [%s]", - "108": "Limpiando índices obsoletos...", + "108": "TODO", "109": "Eliminación de recordatorios completada [%s]", "110": "Renombrar...", "111": "Guardando documento [%s]...", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 982c8bbae..b8ed8b90b 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -928,10 +928,10 @@ "50": "Résolution des référence [%s]", "51": "Blocage du contenu du cache [%d]", "52": "Chaque fois que vous ouvrez le guide de l'utilisateur, les données du notebook seront réinitialisées, veuillez donc ne pas y enregistrer de données.", - "53": "L'indexation de [%d] documents est terminée, et [%d] restent à traiter.", + "53": "TODO", "54": "Indexation des références...", "55": "Références indexées de [%d] documents", - "56": "Indexé [%d] documents", + "56": "TODO", "57": "Échec de la création d'une clé temporaire", "58": "Une fois l'index reconstruit, l'interface sera automatiquement rafraîchie ultérieurement...", "59": "Échec de la définition de la liste des ignores de synchronisation", @@ -968,7 +968,7 @@ "90": "Création de [%d] index de recherche d'éléments de niveau bloc [%s]", "91": "Lecture des données de l'arborescence des blocs...", "92": "Analyse de l'arborescence du document [%s]", - "93": "Indexation de l'arborescence du document [%s]", + "93": "TODO", "94": "Échec du téléchargement : %s", "95": "Quitter le programme...", "96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud", @@ -983,7 +983,7 @@ "105": "Le référentiel de données corrompu a été automatiquement réinitialisé", "106": "La longueur maximale est limitée à 512 caractères", "107": "Déplacement du document [%s]", - "108": "Nettoyage des index obsolètes...", + "108": "TODO", "109": "Supprimer le rappel terminé [%s]", "110": "Renommer...", "111": "Enregistrement du document [%s]...", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index b152fdc43..33cc0fc18 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -928,10 +928,10 @@ "50": "正在解析引用 [%s]", "51": "正在快取內容塊 [%d]", "52": "每次打開使用者指南筆記本資料都會被重置，所以請勿在其中保存任何資料", - "53": "已完成索引 [%d] 篇文檔，剩餘待處理 [%d]", + "53": "TODO", "54": "正在索引引用關係...", "55": "已完成索引 [%d] 篇文檔的引用關係", - "56": "已完成索引 [%d] 篇文檔", + "56": "TODO", "57": "創建臨時金鑰失敗", "58": "重建索引完畢，稍後將自動重新整理介面...", "59": "設置同步忽略列表失敗", @@ -968,7 +968,7 @@ "90": "已經建立 [%d] 個塊級元素的搜索索引 [%s]", "91": "正在讀取塊樹數據...", "92": "正在解析文檔樹 [%s]", - "93": "正在索引文檔樹 [%s]", + "93": "TODO", "94": "上傳失敗：%s", "95": "正在退出...", "96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致", @@ -983,7 +983,7 @@ "105": "已經自動重置損壞的數據倉庫", "106": "最大長度限制為 512 字元", "107": "正在移動文檔 [%s]", - "108": "正在清理已過時的索引...", + "108": "TODO", "109": "移除提醒完畢 [%s]", "110": "正在重命名...", "111": "正在保存文檔 [%s]...", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 2434bb55c..6b497d064 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -928,10 +928,10 @@ "50": "正在解析引用 [%s]", "51": "正在缓存内容块 [%d]", "52": "每次打开用户指南笔记本数据都会被重置，所以请勿在其中保存任何数据", - "53": "已完成索引 [%d] 篇文档，剩余待处理 [%d]", + "53": "TODO", "54": "正在索引引用关系...", "55": "已完成索引 [%d] 篇文档的引用关系", - "56": "已完成索引 [%d] 篇文档", + "56": "TODO", "57": "创建临时密钥失败", "58": "重建索引完毕，稍后将自动刷新界面...", "59": "设置同步忽略列表失败", @@ -968,7 +968,7 @@ "90": "已经建立 [%d] 个块级元素的搜索索引 [%s]", "91": "正在读取块树数据...", "92": "正在解析文档树 [%s]", - "93": "正在索引文档树 [%s]", + "93": "TODO", "94": "上传失败：%s", "95": "正在退出...", "96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致", @@ -983,7 +983,7 @@ "105": "已经自动重置损坏的数据仓库", "106": "最大长度限制为 512 字符", "107": "正在移动文档 [%s]", - "108": "正在清理已过时的索引...", + "108": "TODO", "109": "移除提醒完毕 [%s]", "110": "正在重命名...", "111": "正在保存文档 [%s]...", diff --git a/kernel/api/transaction.go b/kernel/api/transaction.go index d50a73fdd..a7f3c21d0 100644 --- a/kernel/api/transaction.go +++ b/kernel/api/transaction.go @@ -25,9 +25,7 @@ import ( "github.com/88250/gulu" "github.com/gin-gonic/gin" "github.com/siyuan-note/filelock" - "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/model" - "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -67,17 +65,6 @@ func performTransactions(c *gin.Context) { ret.Code = 1 return } - if nil != err { - tx, txErr := sql.BeginTx() - if nil != txErr { - logging.LogFatalf("transaction failed: %s", txErr) - return - } - sql.ClearBoxHash(tx) - sql.CommitTx(tx) - logging.LogFatalf("transaction failed: %s", err) - return - } ret.Data = transactions diff --git a/kernel/model/backlink.go b/kernel/model/backlink.go index 5272bf8e9..5cb07493a 100644 --- a/kernel/model/backlink.go +++ b/kernel/model/backlink.go @@ -40,24 +40,19 @@ import ( func RefreshBacklink(id string) { WaitForWritingFiles() - tx, err := sql.BeginTx() - if nil != err { - return - } - defer sql.CommitTx(tx) - refs := sql.QueryRefsByDefID(id, false) trees := map[string]*parse.Tree{} for _, ref := range refs { tree := trees[ref.RootID] if nil == tree { - tree, err = loadTreeByBlockID(ref.RootID) - if nil != err { - logging.LogErrorf("refresh tree refs failed: %s", err) + var loadErr error + tree, loadErr = loadTreeByBlockID(ref.RootID) + if nil != loadErr { + logging.LogErrorf("refresh tree refs failed: %s", loadErr) continue } trees[ref.RootID] = tree - sql.UpsertRefs(tx, tree) + sql.UpdateRefsTreeQueue(tree) } } } diff --git a/kernel/model/box.go b/kernel/model/box.go index 64caaf5c9..56f7f3d4e 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -339,22 +339,6 @@ func (box *Box) Remove(path string) error { return nil } -func (box *Box) Unindex() { - task.PrependTask(task.DatabaseIndex, unindex, box.ID) -} - -func unindex(boxID string) { - tx, err := sql.BeginTx() - if nil != err { - return - } - sql.RemoveBoxHash(tx, boxID) - sql.DeleteByBoxTx(tx, boxID) - sql.CommitTx(tx) - ids := treenode.RemoveBlockTreesByBoxID(boxID) - RemoveRecentDoc(ids) -} - func (box *Box) ListFiles(path string) (ret []*FileInfo) { fis, _, err := box.Ls(path) if nil != err { @@ -498,6 +482,7 @@ func genTreeID(tree *parse.Tree) { func FullReindex() { task.PrependTask(task.DatabaseIndexFull, fullReindex) + task.AppendTask(task.DatabaseIndexRef, IndexRefs) } func fullReindex() { @@ -510,11 +495,13 @@ func fullReindex() { } treenode.InitBlockTree(true) + sql.IndexMode() + defer sql.NormalMode() + openedBoxes := Conf.GetOpenedBoxes() for _, openedBox := range openedBoxes { - openedBox.Index(true) + index(openedBox.ID) } - IndexRefs() treenode.SaveBlockTree(true) LoadFlashcards() diff --git a/kernel/model/conf.go b/kernel/model/conf.go index 364821337..f1631195c 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -599,7 +599,7 @@ func InitBoxes() { box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 if !initialized { - box.Index(true) + index(box.ID) } } diff --git a/kernel/model/file.go b/kernel/model/file.go index b00162de3..b878e8646 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -912,15 +912,7 @@ func createTreeTx(tree *parse.Tree) { transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}} err := PerformTransactions(&[]*Transaction{transaction}) if nil != err { - tx, txErr := sql.BeginTx() - if nil != txErr { - logging.LogFatalf("transaction failed: %s", txErr) - return - } - sql.ClearBoxHash(tx) - sql.CommitTx(tx) logging.LogFatalf("transaction failed: %s", err) - return } } @@ -1467,13 +1459,6 @@ func createDoc(boxID, p, title, dom string) (err error) { transaction := &Transaction{DoOperations: []*Operation{{Action: "create", Data: tree}}} err = PerformTransactions(&[]*Transaction{transaction}) if nil != err { - tx, txErr := sql.BeginTx() - if nil != txErr { - logging.LogFatalf("transaction failed: %s", txErr) - return - } - sql.ClearBoxHash(tx) - sql.CommitTx(tx) logging.LogFatalf("transaction failed: %s", err) return } diff --git a/kernel/model/import.go b/kernel/model/import.go index 2edcd3be4..ddafbb75f 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -393,7 +393,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) { continue } - treenode.IndexBlockTree(tree) + treenode.ReindexBlockTree(tree) sql.UpsertTreeQueue(tree) } diff --git a/kernel/model/index.go b/kernel/model/index.go index 55b7995ba..f5b7ab140 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -17,11 +17,7 @@ package model import ( - "bytes" - "crypto/sha256" "fmt" - "runtime/debug" - "sort" "strings" "time" @@ -33,20 +29,31 @@ import ( "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) -func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) { - defer debug.FreeOSMemory() +func (box *Box) Unindex() { + task.PrependTask(task.DatabaseIndex, unindex, box.ID) +} - sql.IndexMode() - defer sql.NormalMode() +func unindex(boxID string) { + ids := treenode.RemoveBlockTreesByBoxID(boxID) + RemoveRecentDoc(ids) + sql.DeleteBoxQueue(boxID) +} - //os.MkdirAll("pprof", 0755) - //cpuProfile, _ := os.Create("pprof/cpu_profile_index") - //pprof.StartCPUProfile(cpuProfile) - //defer pprof.StopCPUProfile() +func (box *Box) Index() { + task.PrependTask(task.DatabaseIndex, index, box.ID) + task.AppendTask(task.DatabaseIndexRef, IndexRefs) +} + +func index(boxID string) { + box := Conf.Box(boxID) + if nil == box { + return + } util.SetBootDetails("Listing files...") files := box.ListFiles("/") @@ -54,104 +61,17 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) { if 1 > boxLen { boxLen = 1 } - bootProgressPart := 10.0 / float64(boxLen) / float64(len(files)) - - luteEngine := NewLute() - idTitleMap := map[string]string{} - idHashMap := map[string]string{} - - util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files))) - - i := 0 - // 读取并缓存路径映射 - for _, file := range files { - if file.isdir || !strings.HasSuffix(file.name, ".sy") { - continue - } - - p := file.path - - tree, err := filesys.LoadTree(box.ID, p, luteEngine) - if nil != err { - logging.LogErrorf("read box [%s] tree [%s] failed: %s", box.ID, p, err) - continue - } - - docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL) - if "" == docIAL["updated"] { - updated := util.TimeFromID(tree.Root.ID) - tree.Root.SetIALAttr("updated", updated) - docIAL["updated"] = updated - writeJSONQueue(tree) - } - - cache.PutDocIAL(p, docIAL) - - util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) - treeSize += file.size - treeCount++ - // 缓存文档标题，后面做 Path -> HPath 路径映射时需要 - idTitleMap[tree.ID] = tree.Root.IALAttr("title") - // 缓存块树 - treenode.IndexBlockTree(tree) - // 缓存 ID-Hash，后面需要用于判断是否要重建库 - idHashMap[tree.ID] = tree.Hash - if 1 < i && 0 == i%64 { - util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i)) - } - i++ - } - - box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 - - // 检查是否需要重新建库 - util.SetBootDetails("Checking data hashes...") - var ids []string - for id := range idTitleMap { - ids = append(ids, id) - } - sort.Slice(ids, func(i, j int) bool { return ids[i] >= ids[j] }) - buf := bytes.Buffer{} - for _, id := range ids { - hash, _ := idHashMap[id] - buf.WriteString(hash) - util.SetBootDetails("Checking hash " + hash) - } - boxHash := fmt.Sprintf("%x", sha256.Sum256(buf.Bytes())) - - dbBoxHash := sql.GetBoxHash(box.ID) - if boxHash == dbBoxHash { - //logging.LogInfof("use existing database for box [%s]", box.ID) - util.SetBootDetails("Use existing database for notebook " + box.ID) - return - } - - // 开始重建库 - - sql.DisableCache() - defer sql.EnableCache() + bootProgressPart := 30.0 / float64(boxLen) / float64(len(files)) start := time.Now() - if !fullRebuildIndex { - tx, err := sql.BeginTx() - if nil != err { - return - } - sql.PutBoxHash(tx, box.ID, boxHash) - util.SetBootDetails("Cleaning obsolete indexes...") - util.PushEndlessProgress(Conf.Language(108)) - sql.DeleteByBoxTx(tx, box.ID) - if err = sql.CommitTx(tx); nil != err { - return - } - } + luteEngine := NewLute() + var treeCount int + var treeSize int64 + i := 0 - bootProgressPart = 20.0 / float64(boxLen) / float64(treeCount) + util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(64), len(files))) + defer util.PushClearProgress() - context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress} - i = 0 - // 块级行级入库，缓存块 - // 这里不能并行插入，因为 SQLite 不支持 for _, file := range files { if file.isdir || !strings.HasSuffix(file.name, ".sy") { continue @@ -163,29 +83,33 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) { continue } - util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(93), util.ShortPathForBootingDisplay(tree.Path))) - tx, err := sql.BeginTx() - if nil != err { - continue - } - if err = sql.InsertBlocksSpans(tx, tree, context); nil != err { - sql.RollbackTx(tx) - continue - } - if err = sql.CommitTx(tx); nil != err { - continue + docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL) + if "" == docIAL["updated"] { // 早期的数据可能没有 updated 属性，这里进行订正 + updated := util.TimeFromID(tree.Root.ID) + tree.Root.SetIALAttr("updated", updated) + docIAL["updated"] = updated + if writeErr := filesys.WriteTree(tree); nil != writeErr { + logging.LogErrorf("write tree [%s] failed: %s", tree.Path, writeErr) + } } + + cache.PutDocIAL(file.path, docIAL) + treenode.ReindexBlockTree(tree) + sql.UpsertTreeQueue(tree) + + util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) + treeSize += file.size + treeCount++ if 1 < i && 0 == i%64 { - util.PushEndlessProgress(fmt.Sprintf("["+box.Name+"] "+Conf.Language(53), i, treeCount-i)) + util.PushEndlessProgress(fmt.Sprintf(Conf.Language(88), i, len(files)-i)) } i++ } + box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间 end := time.Now() elapsed := end.Sub(start).Seconds() logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize))) - - util.PushEndlessProgress(fmt.Sprintf(Conf.Language(56), treeCount)) return } @@ -195,7 +119,7 @@ func IndexRefs() { start := time.Now() util.SetBootDetails("Resolving refs...") - util.PushEndlessProgress(Conf.Language(54)) + util.PushStatusBar(Conf.Language(54)) // 引用入库 util.SetBootDetails("Indexing refs...") @@ -204,19 +128,15 @@ func IndexRefs() { for _, refBlock := range refBlocks { refTreeIDs.Add(refBlock.RootID) } + + i := 0 if 0 < refTreeIDs.Size() { luteEngine := NewLute() bootProgressPart := 10.0 / float64(refTreeIDs.Size()) for _, box := range Conf.GetOpenedBoxes() { - tx, err := sql.BeginTx() - if nil != err { - return - } - sql.DeleteRefsByBoxTx(tx, box.ID) - sql.CommitTx(tx) + sql.DeleteBoxRefsQueue(box.ID) files := box.ListFiles("/") - i := 0 for _, file := range files { if file.isdir || !strings.HasSuffix(file.name, ".sy") { continue @@ -239,22 +159,16 @@ func IndexRefs() { continue } - tx, err = sql.BeginTx() - if nil != err { - continue - } - sql.InsertRefs(tx, tree) - if err = sql.CommitTx(tx); nil != err { - continue - } + sql.InsertRefsTreeQueue(tree) if 1 < i && 0 == i%64 { - util.PushEndlessProgress(fmt.Sprintf(Conf.Language(55), i)) + util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) } i++ } } } logging.LogInfof("resolved refs [%d] in [%dms]", len(refBlocks), time.Now().Sub(start).Milliseconds()) + util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i)) } func init() { diff --git a/kernel/model/mount.go b/kernel/model/mount.go index e3ac12ef8..4dbaf0344 100644 --- a/kernel/model/mount.go +++ b/kernel/model/mount.go @@ -195,8 +195,7 @@ func Mount(boxID string) (alreadyMount bool, err error) { boxConf.Closed = false box.SaveConf(boxConf) - box.Index(false) - IndexRefs() + box.Index() // 缓存根一级的文档树展开 ListDocTree(box.ID, "/", Conf.FileTree.Sort) treenode.SaveBlockTree(false) diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 2702580f1..5834ed9a7 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -150,7 +150,11 @@ func syncData(boot, exit, byHand bool) { msg := fmt.Sprintf(Conf.Language(82), synced) Conf.Sync.Stat = msg Conf.Save() - util.BroadcastByType("main", "syncing", 1, msg, nil) + code := 1 + if nil != err { + code = 2 + } + util.BroadcastByType("main", "syncing", code, msg, nil) return } diff --git a/kernel/sql/block.go b/kernel/sql/block.go index eb0d11db3..fbd7392b6 100644 --- a/kernel/sql/block.go +++ b/kernel/sql/block.go @@ -65,10 +65,6 @@ func updateRootContent(tx *sql.Tx, content, updated, id string) { cache.RemoveBlockIAL(id) } -func InsertBlock(tx *sql.Tx, block *Block, context map[string]interface{}) (err error) { - return insertBlocks(tx, []*Block{block}, context) -} - func UpdateBlockContent(block *Block) { tx, err := BeginTx() if nil != err { diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index 2dd7349c8..b048633ca 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -17,17 +17,11 @@ package sql import ( - "bytes" - "crypto/sha256" - "database/sql" - "fmt" "path" "sync" "time" - "github.com/88250/lute/ast" "github.com/88250/lute/parse" - "github.com/emirpasic/gods/sets/hashset" "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/task" @@ -35,19 +29,20 @@ import ( ) var ( - operationQueue []*treeQueueOperation - upsertTreeQueueLock = sync.Mutex{} + operationQueue []*dbQueueOperation + dbQueueLock = sync.Mutex{} txLock = sync.Mutex{} ) -type treeQueueOperation struct { +type dbQueueOperation struct { inQueueTime time.Time - action string // upsert/delete/delete_id/rename + action string // upsert/delete/delete_id/rename/delete_box/delete_box_refs/insert_refs - upsertTree *parse.Tree // upsert + upsertTree *parse.Tree // upsert/insert_refs removeTreeBox, removeTreePath string // delete removeTreeIDBox, removeTreeID string // delete_id + box string // delete_box/delete_box_refs renameTree *parse.Tree // rename renameTreeOldHPath string // rename } @@ -88,8 +83,8 @@ func IsEmptyQueue() bool { } func ClearQueue() { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() + dbQueueLock.Lock() + defer dbQueueLock.Unlock() operationQueue = nil } @@ -108,7 +103,6 @@ func FlushQueue() { } context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} - boxes := hashset.New() for _, op := range ops { switch op.action { case "upsert": @@ -116,17 +110,21 @@ func FlushQueue() { if err = upsertTree(tx, tree, context); nil != err { logging.LogErrorf("upsert tree [%s] into database failed: %s", tree.Box+tree.Path, err) } - boxes.Add(op.upsertTree.Box) case "delete": batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath) - boxes.Add(op.removeTreeBox) case "delete_id": DeleteByRootID(tx, op.removeTreeID) - boxes.Add(op.removeTreeIDBox) case "rename": batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath) updateRootContent(tx, path.Base(op.renameTree.HPath), op.renameTree.Root.IALAttr("updated"), op.renameTree.ID) - boxes.Add(op.renameTree.Box) + case "delete_box": + DeleteByBoxTx(tx, op.box) + case "delete_box_refs": + DeleteRefsByBoxTx(tx, op.box) + case "insert_refs": + InsertRefs(tx, op.upsertTree) + case "update_refs": + UpsertRefs(tx, op.upsertTree) default: logging.LogErrorf("unknown operation [%s]", op.action) } @@ -136,39 +134,78 @@ func FlushQueue() { if 5000 < elapsed { logging.LogInfof("op tx [%dms]", elapsed) } - - start = time.Now() - tx, err = BeginTx() - if nil != err { - return - } - for _, box := range boxes.Values() { - if !ast.IsNodeIDPattern(box.(string)) { - continue - } - updateBoxHash(tx, box.(string)) - } - CommitTx(tx) - elapsed = time.Now().Sub(start).Milliseconds() - if 1000 < elapsed { - logging.LogInfof("hash tx [%dms]", elapsed) - } } -func mergeUpsertTrees() (ops []*treeQueueOperation) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() +func mergeUpsertTrees() (ops []*dbQueueOperation) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() ops = operationQueue operationQueue = nil return } -func UpsertTreeQueue(tree *parse.Tree) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() +func UpdateRefsTreeQueue(tree *parse.Tree) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() - newOp := &treeQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"} + newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "update_refs"} + for i, op := range operationQueue { + if "update_refs" == op.action && op.upsertTree.ID == tree.ID { + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + +func InsertRefsTreeQueue(tree *parse.Tree) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "insert_refs"} + for i, op := range operationQueue { + if "insert_refs" == op.action && op.upsertTree.ID == tree.ID { + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + +func DeleteBoxRefsQueue(boxID string) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box_refs"} + for i, op := range operationQueue { + if "delete_box_refs" == op.action && op.box == boxID { + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + +func DeleteBoxQueue(boxID string) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{box: boxID, inQueueTime: time.Now(), action: "delete_box"} + for i, op := range operationQueue { + if "delete_box" == op.action && op.box == boxID { + operationQueue[i] = newOp + return + } + } + operationQueue = append(operationQueue, newOp) +} + +func UpsertTreeQueue(tree *parse.Tree) { + dbQueueLock.Lock() + defer dbQueueLock.Unlock() + + newOp := &dbQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"} for i, op := range operationQueue { if "upsert" == op.action && op.upsertTree.ID == tree.ID { // 相同树则覆盖 operationQueue[i] = newOp @@ -179,10 +216,10 @@ func UpsertTreeQueue(tree *parse.Tree) { } func RenameTreeQueue(tree *parse.Tree, oldHPath string) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() + dbQueueLock.Lock() + defer dbQueueLock.Unlock() - newOp := &treeQueueOperation{ + newOp := &dbQueueOperation{ renameTree: tree, renameTreeOldHPath: oldHPath, inQueueTime: time.Now(), @@ -197,10 +234,10 @@ func RenameTreeQueue(tree *parse.Tree, oldHPath string) { } func RemoveTreeQueue(box, rootID string) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() + dbQueueLock.Lock() + defer dbQueueLock.Unlock() - var tmp []*treeQueueOperation + var tmp []*dbQueueOperation // 将已有的 upsert 操作去重 for _, op := range operationQueue { if "upsert" == op.action && op.upsertTree.ID != rootID { @@ -209,15 +246,15 @@ func RemoveTreeQueue(box, rootID string) { } operationQueue = tmp - newOp := &treeQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"} + newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"} operationQueue = append(operationQueue, newOp) } func RemoveTreePathQueue(treeBox, treePathPrefix string) { - upsertTreeQueueLock.Lock() - defer upsertTreeQueueLock.Unlock() + dbQueueLock.Lock() + defer dbQueueLock.Unlock() - var tmp []*treeQueueOperation + var tmp []*dbQueueOperation // 将已有的 upsert 操作去重 for _, op := range operationQueue { if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) { @@ -226,31 +263,6 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) { } operationQueue = tmp - newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"} + newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"} operationQueue = append(operationQueue, newOp) } - -func updateBoxHash(tx *sql.Tx, boxID string) { - sum := boxChecksum(boxID) - PutBoxHash(tx, boxID, sum) -} - -func boxChecksum(box string) (ret string) { - rows, err := query("SELECT hash FROM blocks WHERE type = 'd' AND box = ? ORDER BY id DESC", box) - if nil != err { - logging.LogErrorf("sql query failed: %s", err) - return - } - defer rows.Close() - buf := bytes.Buffer{} - for rows.Next() { - var hash string - if err = rows.Scan(&hash); nil != err { - logging.LogErrorf("query scan field failed: %s", err) - return - } - buf.WriteString(hash) - } - ret = fmt.Sprintf("%x", sha256.Sum256(buf.Bytes())) - return -} diff --git a/kernel/sql/stat.go b/kernel/sql/stat.go index e8222d8d5..1b1baa5ec 100644 --- a/kernel/sql/stat.go +++ b/kernel/sql/stat.go @@ -54,27 +54,6 @@ func setDatabaseVer() { CommitTx(tx) } -func ClearBoxHash(tx *sql.Tx) { - stmt := "DELETE FROM stat WHERE `key` LIKE '%_hash'" - execStmtTx(tx, stmt) -} - -func RemoveBoxHash(tx *sql.Tx, box string) { - key := box + "_hash" - stmt := "DELETE FROM stat WHERE `key` = '" + key + "'" - execStmtTx(tx, stmt) -} - -func PutBoxHash(tx *sql.Tx, box, hash string) { - key := box + "_hash" - putStat(tx, key, hash) -} - -func GetBoxHash(box string) string { - key := box + "_hash" - return getStat(key) -} - func putStat(tx *sql.Tx, key, value string) (err error) { stmt := "DELETE FROM stat WHERE `key` = '" + key + "'" if err = execStmtTx(tx, stmt); nil != err { diff --git a/kernel/sql/upsert.go b/kernel/sql/upsert.go index 24d2c7049..380a52475 100644 --- a/kernel/sql/upsert.go +++ b/kernel/sql/upsert.go @@ -36,13 +36,6 @@ func init() { luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial } -func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { - if err = insertBlocksSpans(tx, tree, context); nil != err { - logging.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err) - } - return -} - func InsertRefs(tx *sql.Tx, tree *parse.Tree) { if err := insertRef(tx, tree); nil != err { logging.LogErrorf("insert refs tree [%s] into database failed: %s", tree.Box+tree.Path, err) @@ -395,23 +388,6 @@ func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error return } -func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { - blocks, spans, assets, attributes := fromTree(tree.Root, tree) - if err = insertBlocks(tx, blocks, context); nil != err { - return - } - if err = insertSpans(tx, spans); nil != err { - return - } - if err = insertAssets(tx, assets); nil != err { - return - } - if err = insertAttributes(tx, attributes); nil != err { - return - } - return -} - func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) { refs, fileAnnotationRefs := refsFromTree(tree) if err = insertRefs(tx, refs); nil != err { diff --git a/kernel/task/queue.go b/kernel/task/queue.go index a95155d59..9609408a1 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -102,6 +102,7 @@ const ( RepoCheckout = "task.repo.checkout" // 从快照中检出 DatabaseIndexFull = "task.database.index.full" // 重建索引 DatabaseIndex = "task.database.index" // 数据库索引队列 + DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用 DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 OCRImage = "task.ocr.image" // 图片 OCR 提取文本 HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index 0f0f69787..67df96687 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -320,28 +320,6 @@ func ReindexBlockTree(tree *parse.Tree) { blockTreesChanged = true } -func IndexBlockTree(tree *parse.Tree) { - blockTreesLock.Lock() - defer blockTreesLock.Unlock() - - ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering || !n.IsBlock() { - return ast.WalkContinue - } - var parentID string - if nil != n.Parent { - parentID = n.Parent.ID - } - if "" == n.ID { - return ast.WalkContinue - } - blockTrees[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: tree.Root.IALAttr("updated")} - return ast.WalkContinue - }) - - // 新建索引不变更持久化文件，调用处会负责调用 SaveBlockTree() -} - func AutoFlushBlockTree() { for { SaveBlockTree(false)