🎨 数据历史文档和资源文件支持分页和搜索 https://github.com/siyuan-note/siyuan/issues/4901

kernel/model/assets.go

@@ -393,7 +393,7 @@ func RemoveUnusedAssets() (ret []string) {
 	ret = []string{}
 	unusedAssets := UnusedAssets()
 
-	historyDir, err := util.GetHistoryDir("clean")
+	historyDir, err := GetHistoryDir(HistoryOpClean)
 	if nil != err {
 		logging.LogErrorf("get history dir failed: %s", err)
 		return
@@ -428,7 +428,7 @@ func RemoveUnusedAsset(p string) (ret string) {
 		return p
 	}
 
-	historyDir, err := util.GetHistoryDir("clean")
+	historyDir, err := GetHistoryDir(HistoryOpClean)
 	if nil != err {
 		logging.LogErrorf("get history dir failed: %s", err)
 		return

kernel/model/file.go

@@ -1185,7 +1185,7 @@ func RemoveDoc(boxID, p string) (err error) {
 		return
 	}
 
-	historyDir, err := util.GetHistoryDir("delete")
+	historyDir, err := GetHistoryDir(HistoryOpDelete)
 	if nil != err {
 		logging.LogErrorf("get history dir failed: %s", err)
 		return

kernel/model/format.go

@@ -79,7 +79,7 @@ func AutoSpace(rootID string) (err error) {
 }
 
 func generateFormatHistory(tree *parse.Tree) {
-	historyDir, err := util.GetHistoryDir("format")
+	historyDir, err := GetHistoryDir(HistoryOpFormat)
 	if nil != err {
 		logging.LogErrorf("get history dir failed: %s", err)
 		return

kernel/model/history.go

@@ -18,6 +18,7 @@ package model
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/fs"
 	"os"
@@ -31,6 +32,7 @@ import (
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/conf"
+	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
@@ -476,7 +478,7 @@ func (box *Box) generateDocHistory0() {
 		return
 	}
 
-	historyDir, err := util.GetHistoryDir("update")
+	historyDir, err := GetHistoryDir(HistoryOpUpdate)
 	if nil != err {
 		logging.LogErrorf("get history dir failed: %s", err)
 		return
@@ -562,3 +564,101 @@ func (box *Box) recentModifiedDocs() (ret []string) {
 	box.UpdateHistoryGenerated()
 	return
 }
+
+const (
+	HistoryOpClean  = "clean"
+	HistoryOpUpdate = "update"
+	HistoryOpDelete = "delete"
+	HistoryOpFormat = "format"
+)
+
+func GetHistoryDir(suffix string) (ret string, err error) {
+	ret = filepath.Join(util.HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix)
+	if err = os.MkdirAll(ret, 0755); nil != err {
+		logging.LogErrorf("make history dir failed: %s", err)
+		return
+	}
+	return
+}
+
+func indexHistory() {
+	historyDirs, err := os.ReadDir(util.HistoryDir)
+	if nil != err {
+		logging.LogErrorf("read history dir [%s] failed: %s", util.HistoryDir, err)
+		return
+	}
+
+	validOps := []string{HistoryOpClean, HistoryOpUpdate, HistoryOpDelete, HistoryOpFormat}
+	lutEngine := NewLute()
+	for _, historyDir := range historyDirs {
+		if !historyDir.IsDir() {
+			continue
+		}
+
+		name := historyDir.Name()
+		op := name[strings.LastIndex(name, "-")+1:]
+		if !gulu.Str.Contains(op, validOps) {
+			logging.LogWarnf("invalid history op [%s]", op)
+			continue
+		}
+		t := name[:strings.LastIndex(name, "-")]
+		tt, parseErr := time.Parse("2006-01-02-150405", t)
+		if nil != parseErr {
+			logging.LogWarnf("parse time [%s] failed: %s", t, parseErr)
+			continue
+		}
+		created := fmt.Sprintf("%d", tt.Unix())
+
+		entryPath := filepath.Join(util.HistoryDir, name)
+		var docs, assets []string
+		filepath.Walk(entryPath, func(path string, info os.FileInfo, err error) error {
+			if strings.HasSuffix(info.Name(), ".sy") {
+				docs = append(docs, path)
+			} else if strings.Contains(path, "assets"+string(os.PathSeparator)) {
+				assets = append(assets, path)
+			}
+			return nil
+		})
+
+		var histories []*sql.History
+		for _, doc := range docs {
+			tree, loadErr := loadTree(doc, lutEngine)
+			if nil != err {
+				logging.LogErrorf("load tree [%s] failed: %s", doc, loadErr)
+				continue
+			}
+
+			title := tree.Root.IALAttr("title")
+			content := tree.Root.Content()
+			histories = append(histories, &sql.History{
+				Type:    0,
+				Op:      op,
+				Title:   title,
+				Content: content,
+				Path:    doc,
+				Created: created,
+			})
+		}
+
+		for _, asset := range assets {
+			histories = append(histories, &sql.History{
+				Type:    1,
+				Op:      op,
+				Title:   filepath.Base(asset),
+				Path:    asset,
+				Created: created,
+			})
+		}
+
+		tx, txErr := sql.BeginHistoryTx()
+		if nil != txErr {
+			logging.LogErrorf("begin transaction failed: %s", txErr)
+			return
+		}
+		if err = sql.InsertHistories(tx, histories); nil != err {
+			logging.LogErrorf("insert histories failed: %s", err)
+			sql.RollbackTx(tx)
+			return
+		}
+	}
+}

kernel/model/mount.go

@@ -91,7 +91,7 @@ func RemoveBox(boxID string) (err error) {
 	filelock.ReleaseFileLocks(localPath)
 	if !IsUserGuide(boxID) {
 		var historyDir string
-		historyDir, err = util.GetHistoryDir("delete")
+		historyDir, err = GetHistoryDir(HistoryOpDelete)
 		if nil != err {
 			logging.LogErrorf("get history dir failed: %s", err)
 			return

kernel/sql/database.go

@@ -173,10 +173,10 @@ func InitHistoryDatabase(forceRebuild bool) {
 	historyDB.SetMaxOpenConns(1)
 	historyDB.SetConnMaxLifetime(365 * 24 * time.Hour)
 
-	historyDB.Exec("DROP TABLE history_fts_case_insensitive")
+	historyDB.Exec("DROP TABLE histories_fts_case_insensitive")
-	_, err = db.Exec("CREATE VIRTUAL TABLE history_fts_case_insensitive USING fts5(type UNINDEXED, op UNINDEXED, title, content, created UNINDEXED, path UNINDEXED, tokenize=\"siyuan case_insensitive\")")
+	_, err = historyDB.Exec("CREATE VIRTUAL TABLE histories_fts_case_insensitive USING fts5(type UNINDEXED, op UNINDEXED, title, content, path UNINDEXED, created UNINDEXED, tokenize=\"siyuan case_insensitive\")")
 	if nil != err {
-		logging.LogFatalf("create table [history_fts_case_insensitive] failed: %s", err)
+		logging.LogFatalf("create table [histories_fts_case_insensitive] failed: %s", err)
 	}
 }
 
@@ -1066,6 +1066,9 @@ func CloseDatabase() {
 	if err := db.Close(); nil != err {
 		logging.LogErrorf("close database failed: %s", err)
 	}
+	if err := historyDB.Close(); nil != err {
+		logging.LogErrorf("close history database failed: %s", err)
+	}
 }
 
 func queryRow(query string, args ...interface{}) *sql.Row {
@@ -1095,6 +1098,16 @@ func BeginTx() (tx *sql.Tx, err error) {
 	return
 }
 
+func BeginHistoryTx() (tx *sql.Tx, err error) {
+	if tx, err = historyDB.Begin(); nil != err {
+		logging.LogErrorf("begin history tx failed: %s\n  %s", err, logging.ShortStack())
+		if strings.Contains(err.Error(), "database is locked") {
+			os.Exit(util.ExitCodeReadOnlyDatabase)
+		}
+	}
+	return
+}
+
 func CommitTx(tx *sql.Tx) (err error) {
 	if nil == tx {
 		logging.LogErrorf("tx is nil")

kernel/sql/history.go

@@ -0,0 +1,82 @@
+// SiYuan - Build Your Eternal Digital Garden
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+)
+
+type History struct {
+	Type    int
+	Op      string
+	Title   string
+	Content string
+	Created string
+	Path    string
+}
+
+const (
+	HistoriesFTSCaseInsensitiveInsert = "INSERT INTO histories_fts_case_insensitive (type, op, title, content, path, created) VALUES %s"
+	HistoriesPlaceholder              = "(?, ?, ?, ?, ?, ?)"
+)
+
+func InsertHistories(tx *sql.Tx, histories []*History) (err error) {
+	if 1 > len(histories) {
+		return
+	}
+
+	var bulk []*History
+	for _, history := range histories {
+		bulk = append(bulk, history)
+		if 512 > len(bulk) {
+			continue
+		}
+
+		if err = insertHistories0(tx, bulk); nil != err {
+			return
+		}
+		bulk = []*History{}
+	}
+	if 0 < len(bulk) {
+		if err = insertHistories0(tx, bulk); nil != err {
+			return
+		}
+	}
+	return
+}
+
+func insertHistories0(tx *sql.Tx, bulk []*History) (err error) {
+	valueStrings := make([]string, 0, len(bulk))
+	valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?"))
+	for _, b := range bulk {
+		valueStrings = append(valueStrings, HistoriesPlaceholder)
+		valueArgs = append(valueArgs, b.Type)
+		valueArgs = append(valueArgs, b.Op)
+		valueArgs = append(valueArgs, b.Title)
+		valueArgs = append(valueArgs, b.Content)
+		valueArgs = append(valueArgs, b.Path)
+		valueArgs = append(valueArgs, b.Created)
+	}
+
+	stmt := fmt.Sprintf(HistoriesFTSCaseInsensitiveInsert, strings.Join(valueStrings, ","))
+	if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
+		return
+	}
+	return
+}

kernel/util/working.go

@@ -146,15 +146,6 @@ func SetBooted() {
 	logging.LogInfof("kernel booted")
 }
 
-func GetHistoryDir(suffix string) (ret string, err error) {
-	ret = filepath.Join(HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix)
-	if err = os.MkdirAll(ret, 0755); nil != err {
-		logging.LogErrorf("make history dir failed: %s", err)
-		return
-	}
-	return
-}
-
 var (
 	HomeDir, _    = gulu.OS.Home()
 	WorkingDir, _ = os.Getwd()