diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 806a2a4ff..bfaf29930 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -181,7 +181,7 @@ "exportPDF3": "Page Scale", "exportPDF4": "Remove assets directory", "exportPDF5": "Keep folded", - "exportPDF6": "Fusionar subdocumentos", + "exportPDF6": "Merge subdocuments", "upload": "Upload", "reminderTip": "The reminder time cannot be less than the current time", "wechatTip": "The content block will be sent to the cloud in clear text, and pushed through the WeChat MP template message when it expires", @@ -1051,6 +1051,6 @@ "180": "Search content block does not exist", "181": "The document has been shared to Liandi, click to view", "182": "Sharing document, please wait...", - "183": "Validating index document tree [%s]" + "183": "Validating index document tree [%d/%d %s]" } } diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 1235f3f9f..057ca5592 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -1051,6 +1051,6 @@ "180": "El bloque de contenido de búsqueda no existe", "181": "El documento ha sido compartido con Liandi, haga clic para ver", "182": "Compartiendo documento, por favor espere...", - "183": "Validando el árbol del documento de índice [%s]" + "183": "Validando el árbol del documento de índice [%d/%d %s]" } } diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 899d04cdd..fec071b1e 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -1051,6 +1051,6 @@ "180": "Le bloc de contenu de recherche n'existe pas", "181": "Le document a été partagé avec Liandi, cliquez pour afficher", "182": "Partage du document, veuillez patienter...", - "183": "Validation de l'arborescence du document d'index [%s]" + "183": "Validation de l'arborescence du document d'index [%d/%d %s]" } } diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 035de57ab..c374f55f1 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -1051,6 +1051,6 @@ "180": "不存在符合條件的內容塊", "181": "已分享文檔到鏈滴,點擊查看", "182": "正在分享文檔,請稍等...", - "183": "正在校驗索引文檔樹 [%s]" + "183": "正在校驗索引文檔樹 [%d/%d %s]" } } diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 292099f6c..16a301d47 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -1051,6 +1051,6 @@ "180": "不存在符合条件的内容块", "181": "已分享文档到链滴,点击查看", "182": "正在分享文档,请稍等...", - "183": "正在校验索引文档树 [%s]" + "183": "正在校验索引文档树 [%d/%d %s]" } } diff --git a/kernel/model/box.go b/kernel/model/box.go index d9cb56573..bd29ac1d1 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -523,7 +523,10 @@ func genTreeID(tree *parse.Tree) { return } +var isFullReindexing = false + func FullReindex() { + isFullReindexing = true util.PushEndlessProgress(Conf.Language(35)) WaitForWritingFiles() @@ -542,6 +545,7 @@ func FullReindex() { InitFlashcards() util.PushEndlessProgress(Conf.Language(58)) + isFullReindexing = false go func() { time.Sleep(1 * time.Second) util.ReloadUI() diff --git a/kernel/model/index.go b/kernel/model/index.go index 3f0e64b3b..55b7995ba 100644 --- a/kernel/model/index.go +++ b/kernel/model/index.go @@ -78,6 +78,13 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) { } docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL) + if "" == docIAL["updated"] { + updated := util.TimeFromID(tree.Root.ID) + tree.Root.SetIALAttr("updated", updated) + docIAL["updated"] = updated + writeJSONQueue(tree) + } + cache.PutDocIAL(p, docIAL) util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path))) diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 727ca49b4..92dc9be46 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -1223,45 +1223,70 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha func AutoFixIndex() { for { autoFixIndex() - time.Sleep(30 * time.Second) + time.Sleep(10 * time.Minute) } } +var autoFixLock = sync.Mutex{} + func autoFixIndex() { - rootUpdated := treenode.GetRootUpdated() - for rootID, updated := range rootUpdated { + if util.IsMutexLocked(&autoFixLock) || isFullReindexing { + return + } + + autoFixLock.Lock() + defer autoFixLock.Unlock() + + rootUpdatedMap := treenode.GetRootUpdated() + dbRootUpdatedMap, err := sql.GetRootUpdated() + if nil == err { + i := -1 + size := len(rootUpdatedMap) + for rootID, updated := range rootUpdatedMap { + if isFullReindexing { + break + } + + i++ + + rootUpdated := dbRootUpdatedMap[rootID] + if "" == rootUpdated { + logging.LogWarnf("not found tree [%s] in database, reindex it", rootID) + reindexTree(rootID, i, size) + continue + } + + if "" == updated { + // BlockTree 迁移,v2.6.3 之前没有 updated 字段 + reindexTree(rootID, i, size) + continue + } + + btUpdated, _ := time.Parse("20060102150405", updated) + dbUpdated, _ := time.Parse("20060102150405", rootUpdated) + if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) { + logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID) + reindexTree(rootID, i, size) + continue + } + } + } + + duplicatedRootIDs := sql.GetDuplicatedRootIDs() + size := len(duplicatedRootIDs) + for i, rootID := range duplicatedRootIDs { root := sql.GetBlock(rootID) if nil == root { - logging.LogWarnf("not found tree [%s] in database, reindex it", rootID) - reindexTree(rootID) continue } - if "" == updated { - // BlockTree 迁移,v2.6.3 之前没有 updated 字段 - reindexTree(rootID) - continue - } - - btUpdated, _ := time.Parse("20060102150405", updated) - dbUpdated, _ := time.Parse("20060102150405", root.Updated) - if dbUpdated.Before(btUpdated.Add(-1 * time.Minute)) { - logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID) - reindexTree(rootID) - continue - } - - roots := sql.GetBlockRedundant(rootID) - if 1 < len(roots) { - logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) - sql.RemoveTreeQueue(root.Box, rootID) - reindexTree(rootID) - continue - } + logging.LogWarnf("exist more than one tree [%s], reindex it", rootID) + sql.RemoveTreeQueue(root.Box, rootID) + reindexTree(rootID, i, size) } } -func reindexTree(rootID string) { +func reindexTree(rootID string, i, size int) { root := treenode.GetBlockTree(rootID) if nil == root { logging.LogWarnf("root block not found", rootID) @@ -1273,7 +1298,14 @@ func reindexTree(rootID string) { return } - treenode.ReindexBlockTree(tree) - sql.UpsertTreeQueue(tree) - util.PushStatusBar(fmt.Sprintf(Conf.Language(183), path.Base(tree.HPath))) + updated := tree.Root.IALAttr("updated") + if "" == updated { + updated = util.TimeFromID(tree.Root.ID) + tree.Root.SetIALAttr("updated", updated) + indexWriteJSONQueue(tree) + } else { + treenode.ReindexBlockTree(tree) + sql.UpsertTreeQueue(tree) + } + util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, path.Base(tree.HPath))) } diff --git a/kernel/sql/block_query.go b/kernel/sql/block_query.go index 50c811e5f..71b5bd8ac 100644 --- a/kernel/sql/block_query.go +++ b/kernel/sql/block_query.go @@ -581,17 +581,34 @@ func GetBlock(id string) (ret *Block) { return } -func GetBlockRedundant(id string) (ret []*Block) { - rows, err := query("SELECT * FROM blocks WHERE id = ?", id) +func GetRootUpdated() (ret map[string]string, err error) { + rows, err := query("SELECT root_id, updated FROM blocks WHERE type = 'd'") + if nil != err { + logging.LogErrorf("sql query failed: %s", err) + return + } + defer rows.Close() + + ret = map[string]string{} + for rows.Next() { + var rootID, updated string + rows.Scan(&rootID, &updated) + ret[rootID] = updated + } + return +} + +func GetDuplicatedRootIDs() (ret []string) { + rows, err := query("SELECT DISTINCT root_id FROM blocks GROUP BY id HAVING COUNT(*) > 1") if nil != err { logging.LogErrorf("sql query failed: %s", err) return } defer rows.Close() for rows.Next() { - if block := scanBlockRows(rows); nil != block { - ret = append(ret, block) - } + var id string + rows.Scan(&id) + ret = append(ret, id) } return } diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 1f46dd6d5..35c83d946 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -64,6 +64,7 @@ func InitDatabase(forceRebuild bool) (err error) { util.IncBootProgress(2, "Initializing database...") if forceRebuild { + ClearQueue() WaitForWritingDatabase() } diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index 02e796089..c9c71cfa2 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -85,6 +85,12 @@ func IsEmptyQueue() bool { return 1 > len(operationQueue) && !util.IsMutexLocked(&txLock) } +func ClearQueue() { + upsertTreeQueueLock.Lock() + defer upsertTreeQueueLock.Unlock() + operationQueue = nil +} + func flushTreeQueue() { ops := mergeUpsertTrees() if 1 > len(ops) {