Merge remote-tracking branch 'origin/dev' into dev

app/appearance/langs/en_US.json

@@ -181,7 +181,7 @@
   "exportPDF3": "Page Scale",
   "exportPDF4": "Remove assets directory",
   "exportPDF5": "Keep folded",
-  "exportPDF6": "Fusionar subdocumentos",
+  "exportPDF6": "Merge subdocuments",
   "upload": "Upload",
   "reminderTip": "The reminder time cannot be less than the current time",
   "wechatTip": "The content block will be sent to the cloud in clear text, and pushed through the WeChat MP template message when it expires",
@@ -1051,6 +1051,6 @@
     "180": "Search content block does not exist",
     "181": "The document has been shared to Liandi, <a href=\"%s\" target=\"_blank\">click to view</a>",
     "182": "Sharing document, please wait...",
-    "183": "Validating index document tree [%s]"
+    "183": "Validating index document tree [%d/%d %s]"
   }
 }

app/appearance/langs/es_ES.json

@@ -1051,6 +1051,6 @@
     "180": "El bloque de contenido de búsqueda no existe",
     "181": "El documento ha sido compartido con Liandi, <a href=\"%s\" target=\"_blank\">haga clic para ver</a>",
     "182": "Compartiendo documento, por favor espere...",
-    "183": "Validando el árbol del documento de índice [%s]"
+    "183": "Validando el árbol del documento de índice [%d/%d %s]"
   }
 }

app/appearance/langs/fr_FR.json

@@ -1051,6 +1051,6 @@
     "180": "Le bloc de contenu de recherche n'existe pas",
     "181": "Le document a été partagé avec Liandi, <a href=\"%s\" target=\"_blank\">cliquez pour afficher</a>",
     "182": "Partage du document, veuillez patienter...",
-    "183": "Validation de l'arborescence du document d'index [%s]"
+    "183": "Validation de l'arborescence du document d'index [%d/%d %s]"
   }
 }

app/appearance/langs/zh_CHT.json

@@ -1051,6 +1051,6 @@
     "180": "不存在符合條件的內容塊",
     "181": "已分享文檔到鏈滴，<a href=\"%s\" target=\"_blank\">點擊查看</a>",
     "182": "正在分享文檔，請稍等...",
-    "183": "正在校驗索引文檔樹 [%s]"
+    "183": "正在校驗索引文檔樹 [%d/%d %s]"
   }
 }

app/appearance/langs/zh_CN.json

@@ -1051,6 +1051,6 @@
     "180": "不存在符合条件的内容块",
     "181": "已分享文档到链滴，<a href=\"%s\" target=\"_blank\">点击查看</a>",
     "182": "正在分享文档，请稍等...",
-    "183": "正在校验索引文档树 [%s]"
+    "183": "正在校验索引文档树 [%d/%d %s]"
   }
 }

kernel/model/box.go

@@ -523,7 +523,10 @@ func genTreeID(tree *parse.Tree) {
 	return
 }
 
+var isFullReindexing = false
+
 func FullReindex() {
+	isFullReindexing = true
 	util.PushEndlessProgress(Conf.Language(35))
 	WaitForWritingFiles()
 
@@ -542,6 +545,7 @@ func FullReindex() {
 	InitFlashcards()
 
 	util.PushEndlessProgress(Conf.Language(58))
+	isFullReindexing = false
 	go func() {
 		time.Sleep(1 * time.Second)
 		util.ReloadUI()

kernel/model/index.go

@@ -78,6 +78,13 @@ func (box *Box) Index(fullRebuildIndex bool) (treeCount int, treeSize int64) {
 		}
 
 		docIAL := parse.IAL2MapUnEsc(tree.Root.KramdownIAL)
+		if "" == docIAL["updated"] {
+			updated := util.TimeFromID(tree.Root.ID)
+			tree.Root.SetIALAttr("updated", updated)
+			docIAL["updated"] = updated
+			writeJSONQueue(tree)
+		}
+
 		cache.PutDocIAL(p, docIAL)
 
 		util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path)))

kernel/model/transaction.go

@@ -1223,45 +1223,70 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha
 func AutoFixIndex() {
 	for {
 		autoFixIndex()
-		time.Sleep(30 * time.Second)
+		time.Sleep(10 * time.Minute)
 	}
 }
 
+var autoFixLock = sync.Mutex{}
+
 func autoFixIndex() {
-	rootUpdated := treenode.GetRootUpdated()
-	for rootID, updated := range rootUpdated {
+	if util.IsMutexLocked(&autoFixLock) || isFullReindexing {
+		return
+	}
+
+	autoFixLock.Lock()
+	defer autoFixLock.Unlock()
+
+	rootUpdatedMap := treenode.GetRootUpdated()
+	dbRootUpdatedMap, err := sql.GetRootUpdated()
+	if nil == err {
+		i := -1
+		size := len(rootUpdatedMap)
+		for rootID, updated := range rootUpdatedMap {
+			if isFullReindexing {
+				break
+			}
+
+			i++
+
+			rootUpdated := dbRootUpdatedMap[rootID]
+			if "" == rootUpdated {
+				logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
+				reindexTree(rootID, i, size)
+				continue
+			}
+
+			if "" == updated {
+				// BlockTree 迁移，v2.6.3 之前没有 updated 字段
+				reindexTree(rootID, i, size)
+				continue
+			}
+
+			btUpdated, _ := time.Parse("20060102150405", updated)
+			dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
+			if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
+				logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
+				reindexTree(rootID, i, size)
+				continue
+			}
+		}
+	}
+
+	duplicatedRootIDs := sql.GetDuplicatedRootIDs()
+	size := len(duplicatedRootIDs)
+	for i, rootID := range duplicatedRootIDs {
 		root := sql.GetBlock(rootID)
 		if nil == root {
-			logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
-			reindexTree(rootID)
 			continue
 		}
 
-		if "" == updated {
-			// BlockTree 迁移，v2.6.3 之前没有 updated 字段
-			reindexTree(rootID)
-			continue
-		}
-
-		btUpdated, _ := time.Parse("20060102150405", updated)
-		dbUpdated, _ := time.Parse("20060102150405", root.Updated)
-		if dbUpdated.Before(btUpdated.Add(-1 * time.Minute)) {
-			logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
-			reindexTree(rootID)
-			continue
-		}
-
-		roots := sql.GetBlockRedundant(rootID)
-		if 1 < len(roots) {
-			logging.LogWarnf("exist more than one tree [%s], reindex it", rootID)
-			sql.RemoveTreeQueue(root.Box, rootID)
-			reindexTree(rootID)
-			continue
-		}
+		logging.LogWarnf("exist more than one tree [%s], reindex it", rootID)
+		sql.RemoveTreeQueue(root.Box, rootID)
+		reindexTree(rootID, i, size)
 	}
 }
 
-func reindexTree(rootID string) {
+func reindexTree(rootID string, i, size int) {
 	root := treenode.GetBlockTree(rootID)
 	if nil == root {
 		logging.LogWarnf("root block not found", rootID)
@@ -1273,7 +1298,14 @@ func reindexTree(rootID string) {
 		return
 	}
 
-	treenode.ReindexBlockTree(tree)
-	sql.UpsertTreeQueue(tree)
-	util.PushStatusBar(fmt.Sprintf(Conf.Language(183), path.Base(tree.HPath)))
+	updated := tree.Root.IALAttr("updated")
+	if "" == updated {
+		updated = util.TimeFromID(tree.Root.ID)
+		tree.Root.SetIALAttr("updated", updated)
+		indexWriteJSONQueue(tree)
+	} else {
+		treenode.ReindexBlockTree(tree)
+		sql.UpsertTreeQueue(tree)
+	}
+	util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, path.Base(tree.HPath)))
 }

kernel/sql/block_query.go

@@ -581,17 +581,34 @@ func GetBlock(id string) (ret *Block) {
 	return
 }
 
-func GetBlockRedundant(id string) (ret []*Block) {
-	rows, err := query("SELECT * FROM blocks WHERE id = ?", id)
+func GetRootUpdated() (ret map[string]string, err error) {
+	rows, err := query("SELECT root_id, updated FROM blocks WHERE type = 'd'")
+	if nil != err {
+		logging.LogErrorf("sql query failed: %s", err)
+		return
+	}
+	defer rows.Close()
+
+	ret = map[string]string{}
+	for rows.Next() {
+		var rootID, updated string
+		rows.Scan(&rootID, &updated)
+		ret[rootID] = updated
+	}
+	return
+}
+
+func GetDuplicatedRootIDs() (ret []string) {
+	rows, err := query("SELECT DISTINCT root_id FROM blocks GROUP BY id HAVING COUNT(*) > 1")
 	if nil != err {
 		logging.LogErrorf("sql query failed: %s", err)
 		return
 	}
 	defer rows.Close()
 	for rows.Next() {
-		if block := scanBlockRows(rows); nil != block {
-			ret = append(ret, block)
-		}
+		var id string
+		rows.Scan(&id)
+		ret = append(ret, id)
 	}
 	return
 }

kernel/sql/database.go

@@ -64,6 +64,7 @@ func InitDatabase(forceRebuild bool) (err error) {
 	util.IncBootProgress(2, "Initializing database...")
 
 	if forceRebuild {
+		ClearQueue()
 		WaitForWritingDatabase()
 	}
 

kernel/sql/queue.go

@@ -85,6 +85,12 @@ func IsEmptyQueue() bool {
 	return 1 > len(operationQueue) && !util.IsMutexLocked(&txLock)
 }
 
+func ClearQueue() {
+	upsertTreeQueueLock.Lock()
+	defer upsertTreeQueueLock.Unlock()
+	operationQueue = nil
+}
+
 func flushTreeQueue() {
 	ops := mergeUpsertTrees()
 	if 1 > len(ops) {