diff --git a/kernel/go.mod b/kernel/go.mod index ff274b368..5e9416ba8 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -29,7 +29,7 @@ require ( github.com/gin-contrib/gzip v0.0.6 github.com/gin-contrib/sessions v0.0.5 github.com/gin-gonic/gin v1.8.1 - github.com/imroc/req/v3 v3.19.0 + github.com/imroc/req/v3 v3.19.1 github.com/jinzhu/copier v0.3.5 github.com/mattn/go-sqlite3 v2.0.3+incompatible github.com/mitchellh/go-ps v1.0.0 @@ -38,7 +38,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/qiniu/go-sdk/v7 v7.13.0 github.com/radovskyb/watcher v1.0.7 - github.com/siyuan-note/dejavu v0.0.0-20220821052517-f8edbabd0423 + github.com/siyuan-note/dejavu v0.0.0-20220823012437-83b9401aeea3 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f github.com/siyuan-note/filelock v0.0.0-20220720144616-011221f7e128 @@ -113,7 +113,7 @@ require ( go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect + golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c // indirect golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect golang.org/x/tools v0.1.12 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index 6beaad897..ece40862c 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -200,8 +200,8 @@ github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/imroc/req/v3 v3.19.0 h1:CgUA54GlgeivADx8nodAFwlgifctXZSxf+V8XibzdDs= -github.com/imroc/req/v3 v3.19.0/go.mod h1:EluRnkfh8A39BmrCARYhcUrfGyR8qPw+O0BZyTy4j9k= +github.com/imroc/req/v3 v3.19.1 h1:djAwxGYmQcnvnE5rQViUSll0GrmxlexYhRkL9Chuat4= +github.com/imroc/req/v3 v3.19.1/go.mod h1:EluRnkfh8A39BmrCARYhcUrfGyR8qPw+O0BZyTy4j9k= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= @@ -349,8 +349,8 @@ github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1l github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= -github.com/siyuan-note/dejavu v0.0.0-20220821052517-f8edbabd0423 h1:isIyTU/P59uEY9Teol3F55Va0NWrcMF1cbu+LqyEkY8= -github.com/siyuan-note/dejavu v0.0.0-20220821052517-f8edbabd0423/go.mod h1:/7pAviNPlpJiwZkEg2eyLTEq2/8sfW/AU4eHBvyrHFk= +github.com/siyuan-note/dejavu v0.0.0-20220823012437-83b9401aeea3 h1:Jo6xIoRVswwki+TQK5t8fefd/wGTEAdmQiETKTh6YSo= +github.com/siyuan-note/dejavu v0.0.0-20220823012437-83b9401aeea3/go.mod h1:/7pAviNPlpJiwZkEg2eyLTEq2/8sfW/AU4eHBvyrHFk= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f h1:JMobMNZ7AqaKKyEK+WeWFhix/2TDQXgPZDajU00IybU= @@ -465,8 +465,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220802222814-0bcc04d9c69b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= -golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c h1:JVAXQ10yGGVbSyoer5VILysz6YKjdNT2bsvlayjqhes= +golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/kernel/main.go b/kernel/main.go index 3f39a34ae..aea697819 100644 --- a/kernel/main.go +++ b/kernel/main.go @@ -34,6 +34,7 @@ func main() { go server.Serve(false) model.InitAppearance() sql.InitDatabase(false) + sql.InitHistoryDatabase(false) sql.SetCaseSensitive(model.Conf.Search.CaseSensitive) model.SyncData(true, false, false) diff --git a/kernel/mobile/kernel.go b/kernel/mobile/kernel.go index edfc00dfb..24866015d 100644 --- a/kernel/mobile/kernel.go +++ b/kernel/mobile/kernel.go @@ -48,6 +48,7 @@ func StartKernel(container, appDir, workspaceDir, nativeLibDir, privateDataDir, go func() { model.InitAppearance() sql.InitDatabase(false) + sql.InitHistoryDatabase(false) sql.SetCaseSensitive(model.Conf.Search.CaseSensitive) model.SyncData(true, false, false) diff --git a/kernel/model/assets.go b/kernel/model/assets.go index b3af28811..8065fe42e 100644 --- a/kernel/model/assets.go +++ b/kernel/model/assets.go @@ -393,7 +393,7 @@ func RemoveUnusedAssets() (ret []string) { ret = []string{} unusedAssets := UnusedAssets() - historyDir, err := util.GetHistoryDir("delete") + historyDir, err := GetHistoryDir(HistoryOpClean) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return @@ -428,7 +428,7 @@ func RemoveUnusedAsset(p string) (ret string) { return p } - historyDir, err := util.GetHistoryDir("delete") + historyDir, err := GetHistoryDir(HistoryOpClean) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return diff --git a/kernel/model/file.go b/kernel/model/file.go index 51cbae967..d78fc9d5a 100644 --- a/kernel/model/file.go +++ b/kernel/model/file.go @@ -1185,7 +1185,7 @@ func RemoveDoc(boxID, p string) (err error) { return } - historyDir, err := util.GetHistoryDir("delete") + historyDir, err := GetHistoryDir(HistoryOpDelete) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return diff --git a/kernel/model/format.go b/kernel/model/format.go index a7e1b9c8e..ba2bd0589 100644 --- a/kernel/model/format.go +++ b/kernel/model/format.go @@ -79,7 +79,7 @@ func AutoSpace(rootID string) (err error) { } func generateFormatHistory(tree *parse.Tree) { - historyDir, err := util.GetHistoryDir("format") + historyDir, err := GetHistoryDir(HistoryOpFormat) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return diff --git a/kernel/model/history.go b/kernel/model/history.go index 2cba21235..b5f415f76 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -18,6 +18,7 @@ package model import ( "encoding/json" + "fmt" "io" "io/fs" "os" @@ -31,6 +32,7 @@ import ( "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/conf" + "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -476,7 +478,7 @@ func (box *Box) generateDocHistory0() { return } - historyDir, err := util.GetHistoryDir("update") + historyDir, err := GetHistoryDir(HistoryOpUpdate) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return @@ -562,3 +564,101 @@ func (box *Box) recentModifiedDocs() (ret []string) { box.UpdateHistoryGenerated() return } + +const ( + HistoryOpClean = "clean" + HistoryOpUpdate = "update" + HistoryOpDelete = "delete" + HistoryOpFormat = "format" +) + +func GetHistoryDir(suffix string) (ret string, err error) { + ret = filepath.Join(util.HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix) + if err = os.MkdirAll(ret, 0755); nil != err { + logging.LogErrorf("make history dir failed: %s", err) + return + } + return +} + +func indexHistory() { + historyDirs, err := os.ReadDir(util.HistoryDir) + if nil != err { + logging.LogErrorf("read history dir [%s] failed: %s", util.HistoryDir, err) + return + } + + validOps := []string{HistoryOpClean, HistoryOpUpdate, HistoryOpDelete, HistoryOpFormat} + lutEngine := NewLute() + for _, historyDir := range historyDirs { + if !historyDir.IsDir() { + continue + } + + name := historyDir.Name() + op := name[strings.LastIndex(name, "-")+1:] + if !gulu.Str.Contains(op, validOps) { + logging.LogWarnf("invalid history op [%s]", op) + continue + } + t := name[:strings.LastIndex(name, "-")] + tt, parseErr := time.Parse("2006-01-02-150405", t) + if nil != parseErr { + logging.LogWarnf("parse time [%s] failed: %s", t, parseErr) + continue + } + created := fmt.Sprintf("%d", tt.Unix()) + + entryPath := filepath.Join(util.HistoryDir, name) + var docs, assets []string + filepath.Walk(entryPath, func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(info.Name(), ".sy") { + docs = append(docs, path) + } else if strings.Contains(path, "assets"+string(os.PathSeparator)) { + assets = append(assets, path) + } + return nil + }) + + var histories []*sql.History + for _, doc := range docs { + tree, loadErr := loadTree(doc, lutEngine) + if nil != err { + logging.LogErrorf("load tree [%s] failed: %s", doc, loadErr) + continue + } + + title := tree.Root.IALAttr("title") + content := tree.Root.Content() + histories = append(histories, &sql.History{ + Type: 0, + Op: op, + Title: title, + Content: content, + Path: doc, + Created: created, + }) + } + + for _, asset := range assets { + histories = append(histories, &sql.History{ + Type: 1, + Op: op, + Title: filepath.Base(asset), + Path: asset, + Created: created, + }) + } + + tx, txErr := sql.BeginHistoryTx() + if nil != txErr { + logging.LogErrorf("begin transaction failed: %s", txErr) + return + } + if err = sql.InsertHistories(tx, histories); nil != err { + logging.LogErrorf("insert histories failed: %s", err) + sql.RollbackTx(tx) + return + } + } +} diff --git a/kernel/model/mount.go b/kernel/model/mount.go index d2e997e75..69b721d61 100644 --- a/kernel/model/mount.go +++ b/kernel/model/mount.go @@ -91,7 +91,7 @@ func RemoveBox(boxID string) (err error) { filelock.ReleaseFileLocks(localPath) if !IsUserGuide(boxID) { var historyDir string - historyDir, err = util.GetHistoryDir("delete") + historyDir, err = GetHistoryDir(HistoryOpDelete) if nil != err { logging.LogErrorf("get history dir failed: %s", err) return diff --git a/kernel/sql/database.go b/kernel/sql/database.go index 6e4616055..5d792c3f6 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -38,7 +38,10 @@ import ( "github.com/siyuan-note/siyuan/kernel/util" ) -var db *sql.DB +var ( + db *sql.DB + historyDB *sql.DB +) func init() { regex := func(re, s string) (bool, error) { @@ -143,6 +146,40 @@ func initDBTables() { } } +func InitHistoryDatabase(forceRebuild bool) { + if !forceRebuild && gulu.File.IsExist(util.HistoryDBPath) { + return + } + + if nil != historyDB { + historyDB.Close() + } + dsn := util.HistoryDBPath + "?_journal_mode=OFF" + + "&_synchronous=OFF" + + "&_secure_delete=OFF" + + "&_cache_size=-20480" + + "&_page_size=8192" + + "&_busy_timeout=7000" + + "&_ignore_check_constraints=ON" + + "&_temp_store=MEMORY" + + "&_case_sensitive_like=OFF" + + "&_locking_mode=EXCLUSIVE" + var err error + historyDB, err = sql.Open("sqlite3_extended", dsn) + if nil != err { + logging.LogFatalf("create database failed: %s", err) + } + historyDB.SetMaxIdleConns(1) + historyDB.SetMaxOpenConns(1) + historyDB.SetConnMaxLifetime(365 * 24 * time.Hour) + + historyDB.Exec("DROP TABLE histories_fts_case_insensitive") + _, err = historyDB.Exec("CREATE VIRTUAL TABLE histories_fts_case_insensitive USING fts5(type UNINDEXED, op UNINDEXED, title, content, path UNINDEXED, created UNINDEXED, tokenize=\"siyuan case_insensitive\")") + if nil != err { + logging.LogFatalf("create table [histories_fts_case_insensitive] failed: %s", err) + } +} + func IndexMode() { if nil != db { db.Close() @@ -1029,6 +1066,9 @@ func CloseDatabase() { if err := db.Close(); nil != err { logging.LogErrorf("close database failed: %s", err) } + if err := historyDB.Close(); nil != err { + logging.LogErrorf("close history database failed: %s", err) + } } func queryRow(query string, args ...interface{}) *sql.Row { @@ -1058,6 +1098,16 @@ func BeginTx() (tx *sql.Tx, err error) { return } +func BeginHistoryTx() (tx *sql.Tx, err error) { + if tx, err = historyDB.Begin(); nil != err { + logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack()) + if strings.Contains(err.Error(), "database is locked") { + os.Exit(util.ExitCodeReadOnlyDatabase) + } + } + return +} + func CommitTx(tx *sql.Tx) (err error) { if nil == tx { logging.LogErrorf("tx is nil") diff --git a/kernel/sql/history.go b/kernel/sql/history.go new file mode 100644 index 000000000..ec2f6a99c --- /dev/null +++ b/kernel/sql/history.go @@ -0,0 +1,82 @@ +// SiYuan - Build Your Eternal Digital Garden +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sql + +import ( + "database/sql" + "fmt" + "strings" +) + +type History struct { + Type int + Op string + Title string + Content string + Created string + Path string +} + +const ( + HistoriesFTSCaseInsensitiveInsert = "INSERT INTO histories_fts_case_insensitive (type, op, title, content, path, created) VALUES %s" + HistoriesPlaceholder = "(?, ?, ?, ?, ?, ?)" +) + +func InsertHistories(tx *sql.Tx, histories []*History) (err error) { + if 1 > len(histories) { + return + } + + var bulk []*History + for _, history := range histories { + bulk = append(bulk, history) + if 512 > len(bulk) { + continue + } + + if err = insertHistories0(tx, bulk); nil != err { + return + } + bulk = []*History{} + } + if 0 < len(bulk) { + if err = insertHistories0(tx, bulk); nil != err { + return + } + } + return +} + +func insertHistories0(tx *sql.Tx, bulk []*History) (err error) { + valueStrings := make([]string, 0, len(bulk)) + valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?")) + for _, b := range bulk { + valueStrings = append(valueStrings, HistoriesPlaceholder) + valueArgs = append(valueArgs, b.Type) + valueArgs = append(valueArgs, b.Op) + valueArgs = append(valueArgs, b.Title) + valueArgs = append(valueArgs, b.Content) + valueArgs = append(valueArgs, b.Path) + valueArgs = append(valueArgs, b.Created) + } + + stmt := fmt.Sprintf(HistoriesFTSCaseInsensitiveInsert, strings.Join(valueStrings, ",")) + if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err { + return + } + return +} diff --git a/kernel/util/working.go b/kernel/util/working.go index 1c66e5aae..3e0546ee6 100644 --- a/kernel/util/working.go +++ b/kernel/util/working.go @@ -146,15 +146,6 @@ func SetBooted() { logging.LogInfof("kernel booted") } -func GetHistoryDir(suffix string) (ret string, err error) { - ret = filepath.Join(HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix) - if err = os.MkdirAll(ret, 0755); nil != err { - logging.LogErrorf("make history dir failed: %s", err) - return - } - return -} - var ( HomeDir, _ = gulu.OS.Home() WorkingDir, _ = os.Getwd() @@ -168,6 +159,7 @@ var ( LogPath string // 配置目录下的日志文件 siyuan.log 路径 DBName = "siyuan.db" // SQLite 数据库文件名 DBPath string // SQLite 数据库文件路径 + HistoryDBPath string // SQLite 历史数据库文件路径 BlockTreePath string // 区块树文件路径 AppearancePath string // 配置目录下的外观目录 appearance/ 路径 ThemesPath string // 配置目录下的外观目录下的 themes/ 路径 @@ -271,6 +263,7 @@ func initWorkspaceDir(workspaceArg string) { os.Setenv("TEMP", osTmpDir) os.Setenv("TMP", osTmpDir) DBPath = filepath.Join(TempDir, DBName) + HistoryDBPath = filepath.Join(TempDir, "history.db") BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack") } diff --git a/kernel/util/working_mobile.go b/kernel/util/working_mobile.go index caf753912..e3beefc9b 100644 --- a/kernel/util/working_mobile.go +++ b/kernel/util/working_mobile.go @@ -51,6 +51,7 @@ func BootMobile(container, appDir, workspaceDir, nativeLibDir, privateDataDir, l os.RemoveAll(filepath.Join(TempDir, "repo")) os.Setenv("TMPDIR", osTmpDir) DBPath = filepath.Join(TempDir, DBName) + HistoryDBPath = filepath.Join(TempDir, "history.db") BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack") AndroidNativeLibDir = nativeLibDir AndroidPrivateDataDir = privateDataDir