diff --git a/kernel/model/assets.go b/kernel/model/assets.go index f7932e8c6..8065fe42e 100644 --- a/kernel/model/assets.go +++ b/kernel/model/assets.go @@ -393,7 +393,7 @@ func RemoveUnusedAssets() (ret []string) { ret = []string{} unusedAssets := UnusedAssets() - historyDir, err := util.GetHistoryDir("clean") + historyDir, err := GetHistoryDir(HistoryOpClean) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return @@ -428,7 +428,7 @@ func RemoveUnusedAsset(p string) (ret string) { return p } - historyDir, err := util.GetHistoryDir("clean") + historyDir, err := GetHistoryDir(HistoryOpClean) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return diff --git a/kernel/model/file.go b/kernel/model/file.go index 51cbae967..d78fc9d5a 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -1185,7 +1185,7 @@ func RemoveDoc(boxID, p string) (err error) { return } - historyDir, err := util.GetHistoryDir("delete") + historyDir, err := GetHistoryDir(HistoryOpDelete) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return diff --git a/kernel/model/format.go b/kernel/model/format.go index a7e1b9c8e..ba2bd0589 100644 --- a/kernel/model/format.go +++ b/kernel/model/format.go @@ -79,7 +79,7 @@ func AutoSpace(rootID string) (err error) { } func generateFormatHistory(tree *parse.Tree) { - historyDir, err := util.GetHistoryDir("format") + historyDir, err := GetHistoryDir(HistoryOpFormat) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return diff --git a/kernel/model/history.go b/kernel/model/history.go index 2cba21235..b5f415f76 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -18,6 +18,7 @@ package model import ( "encoding/json" + "fmt" "io" "io/fs" "os" @@ -31,6 +32,7 @@ import ( "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/conf" + "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -476,7 +478,7 @@ func (box *Box) generateDocHistory0() { return } - historyDir, err := util.GetHistoryDir("update") + historyDir, err := GetHistoryDir(HistoryOpUpdate) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return @@ -562,3 +564,101 @@ func (box *Box) recentModifiedDocs() (ret []string) { box.UpdateHistoryGenerated() return } + +const ( + HistoryOpClean = "clean" + HistoryOpUpdate = "update" + HistoryOpDelete = "delete" + HistoryOpFormat = "format" +) + +func GetHistoryDir(suffix string) (ret string, err error) { + ret = filepath.Join(util.HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix) + if err = os.MkdirAll(ret, 0755); nil != err { + logging.LogErrorf("make history dir failed: %s", err) + return + } + return +} + +func indexHistory() { + historyDirs, err := os.ReadDir(util.HistoryDir) + if nil != err { + logging.LogErrorf("read history dir [%s] failed: %s", util.HistoryDir, err) + return + } + + validOps := []string{HistoryOpClean, HistoryOpUpdate, HistoryOpDelete, HistoryOpFormat} + lutEngine := NewLute() + for _, historyDir := range historyDirs { + if !historyDir.IsDir() { + continue + } + + name := historyDir.Name() + op := name[strings.LastIndex(name, "-")+1:] + if !gulu.Str.Contains(op, validOps) { + logging.LogWarnf("invalid history op [%s]", op) + continue + } + t := name[:strings.LastIndex(name, "-")] + tt, parseErr := time.Parse("2006-01-02-150405", t) + if nil != parseErr { + logging.LogWarnf("parse time [%s] failed: %s", t, parseErr) + continue + } + created := fmt.Sprintf("%d", tt.Unix()) + + entryPath := filepath.Join(util.HistoryDir, name) + var docs, assets []string + filepath.Walk(entryPath, func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(info.Name(), ".sy") { + docs = append(docs, path) + } else if strings.Contains(path, "assets"+string(os.PathSeparator)) { + assets = append(assets, path) + } + return nil + }) + + var histories []*sql.History + for _, doc := range docs { + tree, loadErr := loadTree(doc, lutEngine) + if nil != err { + logging.LogErrorf("load tree [%s] failed: %s", doc, loadErr) + continue + } + + title := tree.Root.IALAttr("title") + content := tree.Root.Content() + histories = append(histories, &sql.History{ + Type: 0, + Op: op, + Title: title, + Content: content, + Path: doc, + Created: created, + }) + } + + for _, asset := range assets { + histories = append(histories, &sql.History{ + Type: 1, + Op: op, + Title: filepath.Base(asset), + Path: asset, + Created: created, + }) + } + + tx, txErr := sql.BeginHistoryTx() + if nil != txErr { + logging.LogErrorf("begin transaction failed: %s", txErr) + return + } + if err = sql.InsertHistories(tx, histories); nil != err { + logging.LogErrorf("insert histories failed: %s", err) + sql.RollbackTx(tx) + return + } + } +} diff --git a/kernel/model/mount.go b/kernel/model/mount.go index d2e997e75..69b721d61 100644 --- a/kernel/model/mount.go +++ b/kernel/model/mount.go @@ -91,7 +91,7 @@ func RemoveBox(boxID string) (err error) { filelock.ReleaseFileLocks(localPath) if !IsUserGuide(boxID) { var historyDir string - historyDir, err = util.GetHistoryDir("delete") + historyDir, err = GetHistoryDir(HistoryOpDelete) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return diff --git a/kernel/sql/database.go b/kernel/sql/database.go index f116bf239..5d792c3f6 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -173,10 +173,10 @@ func InitHistoryDatabase(forceRebuild bool) { historyDB.SetMaxOpenConns(1) historyDB.SetConnMaxLifetime(365 * 24 * time.Hour) - historyDB.Exec("DROP TABLE history_fts_case_insensitive") - _, err = db.Exec("CREATE VIRTUAL TABLE history_fts_case_insensitive USING fts5(type UNINDEXED, op UNINDEXED, title, content, created UNINDEXED, path UNINDEXED, tokenize=\"siyuan case_insensitive\")") + historyDB.Exec("DROP TABLE histories_fts_case_insensitive") + _, err = historyDB.Exec("CREATE VIRTUAL TABLE histories_fts_case_insensitive USING fts5(type UNINDEXED, op UNINDEXED, title, content, path UNINDEXED, created UNINDEXED, tokenize=\"siyuan case_insensitive\")") if nil != err { - logging.LogFatalf("create table [history_fts_case_insensitive] failed: %s", err) + logging.LogFatalf("create table [histories_fts_case_insensitive] failed: %s", err) } } @@ -1066,6 +1066,9 @@ func CloseDatabase() { if err := db.Close(); nil != err { logging.LogErrorf("close database failed: %s", err) } + if err := historyDB.Close(); nil != err { + logging.LogErrorf("close history database failed: %s", err) + } } func queryRow(query string, args ...interface{}) *sql.Row { @@ -1095,6 +1098,16 @@ func BeginTx() (tx *sql.Tx, err error) { return } +func BeginHistoryTx() (tx *sql.Tx, err error) { + if tx, err = historyDB.Begin(); nil != err { + logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack()) + if strings.Contains(err.Error(), "database is locked") { + os.Exit(util.ExitCodeReadOnlyDatabase) + } + } + return +} + func CommitTx(tx *sql.Tx) (err error) { if nil == tx { logging.LogErrorf("tx is nil") diff --git a/kernel/sql/history.go b/kernel/sql/history.go new file mode 100644 index 000000000..ec2f6a99c --- /dev/null +++ b/kernel/sql/history.go @@ -0,0 +1,82 @@ +// SiYuan - Build Your Eternal Digital Garden +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "database/sql" + "fmt" + "strings" +) + +type History struct { + Type int + Op string + Title string + Content string + Created string + Path string +} + +const ( + HistoriesFTSCaseInsensitiveInsert = "INSERT INTO histories_fts_case_insensitive (type, op, title, content, path, created) VALUES %s" + HistoriesPlaceholder = "(?, ?, ?, ?, ?, ?)" +) + +func InsertHistories(tx *sql.Tx, histories []*History) (err error) { + if 1 > len(histories) { + return + } + + var bulk []*History + for _, history := range histories { + bulk = append(bulk, history) + if 512 > len(bulk) { + continue + } + + if err = insertHistories0(tx, bulk); nil != err { + return + } + bulk = []*History{} + } + if 0 < len(bulk) { + if err = insertHistories0(tx, bulk); nil != err { + return + } + } + return +} + +func insertHistories0(tx *sql.Tx, bulk []*History) (err error) { + valueStrings := make([]string, 0, len(bulk)) + valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?")) + for _, b := range bulk { + valueStrings = append(valueStrings, HistoriesPlaceholder) + valueArgs = append(valueArgs, b.Type) + valueArgs = append(valueArgs, b.Op) + valueArgs = append(valueArgs, b.Title) + valueArgs = append(valueArgs, b.Content) + valueArgs = append(valueArgs, b.Path) + valueArgs = append(valueArgs, b.Created) + } + + stmt := fmt.Sprintf(HistoriesFTSCaseInsensitiveInsert, strings.Join(valueStrings, ",")) + if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err { + return + } + return +} diff --git a/kernel/util/working.go b/kernel/util/working.go index fe1c9a670..3e0546ee6 100644 --- a/kernel/util/working.go +++ b/kernel/util/working.go @@ -146,15 +146,6 @@ func SetBooted() { logging.LogInfof("kernel booted") } -func GetHistoryDir(suffix string) (ret string, err error) { - ret = filepath.Join(HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix) - if err = os.MkdirAll(ret, 0755); nil != err { - logging.LogErrorf("make history dir failed: %s", err) - return - } - return -} - var ( HomeDir, _ = gulu.OS.Home() WorkingDir, _ = os.Getwd()