From eaa83998a44e1b3af3ecf7307f20b417d17b4eca Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 30 Jun 2022 22:40:43 +0800 Subject: [PATCH] :art: Sync repo --- kernel/go.mod | 6 +-- kernel/go.sum | 12 ++--- kernel/model/conf.go | 10 ++-- kernel/model/repository.go | 38 +++++++++++++- kernel/model/sync.go | 98 +++++++++++++----------------------- kernel/treenode/blocktree.go | 3 +- 6 files changed, 86 insertions(+), 81 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 1405daf52..ef0cb391f 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -40,7 +40,7 @@ require ( github.com/qiniu/go-sdk/v7 v7.13.0 github.com/radovskyb/watcher v1.0.7 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 - github.com/siyuan-note/dejavu v0.0.0-20220627141634-53a7e4ef9840 + github.com/siyuan-note/dejavu v0.0.0-20220630143851-b239e9224291 github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f github.com/siyuan-note/filelock v0.0.0-20220616063212-74cfba0754ee @@ -83,7 +83,7 @@ require ( github.com/imdario/mergo v0.3.13 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/errors v0.0.0-20220331221717-b38fca44723b // indirect - github.com/klauspost/compress v1.15.6 // indirect + github.com/klauspost/compress v1.15.7 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -103,7 +103,7 @@ require ( golang.org/x/mod v0.5.1 // indirect golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect - golang.org/x/sys v0.0.0-20220624220833-87e55d714810 // indirect + golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect golang.org/x/tools v0.1.8 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.28.0 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index ebe631060..e387d5968 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -320,8 +320,8 @@ github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0t github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= -github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok= +github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -421,8 +421,8 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/siyuan-note/dejavu v0.0.0-20220627141634-53a7e4ef9840 h1:H/Gb6d/Q9/7LK/T9KuuqIXlZGdj5rLFIVNFsJ00r00U= -github.com/siyuan-note/dejavu v0.0.0-20220627141634-53a7e4ef9840/go.mod h1:0t+TpAdsrrv+6LkhDlJY2Gl92CQexgp8KWOBp5Ah9kU= +github.com/siyuan-note/dejavu v0.0.0-20220630143851-b239e9224291 h1:q6ZASwpPzlLseZIK0B37p3IQ8mTN4HnyFhD7NN3oVtI= +github.com/siyuan-note/dejavu v0.0.0-20220630143851-b239e9224291/go.mod h1:0t+TpAdsrrv+6LkhDlJY2Gl92CQexgp8KWOBp5Ah9kU= github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 h1:QB9TjJQFhXhZ6dAtPpY02DlzHAQm1C+WqZq6OadG8mI= github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f h1:JMobMNZ7AqaKKyEK+WeWFhix/2TDQXgPZDajU00IybU= @@ -662,8 +662,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810 h1:rHZQSjJdAI4Xf5Qzeh2bBc5YJIkPFVM6oDtMFYmgws0= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20180302201248-b7ef84aaf62a/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/kernel/model/conf.go b/kernel/model/conf.go index 8a677b610..d777c6802 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -488,13 +488,9 @@ func InitBoxes() { util.IncBootProgress(1, "Reading block trees...") } }() - if err := treenode.ReadBlockTree(); nil == err { - initialized = true - } else { - if err = os.RemoveAll(util.BlockTreePath); nil != err { - util.LogErrorf("remove block tree [%s] failed: %s", util.BlockTreePath, err) - } - } + + treenode.InitBlockTree() + initialized = true } } else { // 大于 1 的话说明在同步阶段已经加载过了 initialized = true diff --git a/kernel/model/repository.go b/kernel/model/repository.go index c6395072a..81e7b5560 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -29,6 +29,7 @@ import ( "github.com/siyuan-note/encryption" "github.com/siyuan-note/eventbus" "github.com/siyuan-note/filelock" + "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -317,7 +318,7 @@ func CheckoutRepo(id string) (err error) { Conf.Sync.Enabled = false Conf.Save() - err = repo.Checkout(id, map[string]interface{}{ + _, _, err = repo.Checkout(id, map[string]interface{}{ CtxPushMsg: CtxPushMsgToStatusBarAndProgress, }) if nil != err { @@ -444,12 +445,45 @@ func syncRepo() (err error) { } start := time.Now() - err = repo.Sync(Conf.Sync.CloudName, Conf.User.UserId, Conf.User.UserToken, Conf.System.NetworkProxy.String(), util.AliyunServer, map[string]interface{}{ + latest, fetchedFiles, err := repo.Sync(Conf.Sync.CloudName, Conf.User.UserId, Conf.User.UserToken, Conf.System.NetworkProxy.String(), util.AliyunServer, map[string]interface{}{ CtxPushMsg: CtxPushMsgToStatusBar, }) elapsed := time.Since(start) util.LogInfof("sync data repo elapsed [%.2fs]", elapsed.Seconds()) + if nil != err { + util.LogErrorf("sync data repo failed: %s", err) + util.PushStatusBar("Sync data repo failed: " + err.Error()) + return + } util.PushStatusBar(fmt.Sprintf(Conf.Language(149), elapsed.Seconds())) + if 1 > len(fetchedFiles) { + // 没有下载到新文件，直接返回 + return + } + + // 下载到文件后，需要恢复到工作区并重建索引 + + CloseWatchAssets() + defer WatchAssets() + + upsertFiles, removeFiles, err := repo.Checkout(latest.ID, map[string]interface{}{ + CtxPushMsg: CtxPushMsgToStatusBarAndProgress, + }) + if nil != err { + util.PushClearProgress() + return + } + + var upserts, removes []string + for _, file := range upsertFiles { + upserts = append(upserts, file.Path) + } + for _, file := range removeFiles { + removes = append(removes, file.Path) + } + incReindex(upserts, removes) + cache.ClearDocsIAL() + util.ReloadUI() return } diff --git a/kernel/model/sync.go b/kernel/model/sync.go index a47abbdd5..916762e54 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -401,71 +401,13 @@ func SyncData(boot, exit, byHand bool) { if boot && gulu.File.IsExist(util.BlockTreePath) { // 在 blocktree 存在的情况下不会重建索引，所以这里需要刷新 blocktree 和 database - - if err = treenode.ReadBlockTree(); nil != err { - os.RemoveAll(util.BlockTreePath) - util.LogWarnf("removed block tree [%s] due to %s", util.BlockTreePath, err) - return - } - - for _, upsertFile := range upsertFiles { - if !strings.HasSuffix(upsertFile, ".sy") { - continue - } - - upsertFile = filepath.ToSlash(upsertFile) - box := upsertFile[:strings.Index(upsertFile, "/")] - p := strings.TrimPrefix(upsertFile, box) - tree, err0 := LoadTree(box, p) - if nil != err0 { - continue - } - treenode.ReindexBlockTree(tree) - sql.UpsertTreeQueue(tree) - } - for _, removeFile := range removeFiles { - if !strings.HasSuffix(removeFile, ".sy") { - continue - } - id := strings.TrimSuffix(filepath.Base(removeFile), ".sy") - block := treenode.GetBlockTree(id) - if nil != block { - treenode.RemoveBlockTreesByRootID(block.RootID) - sql.RemoveTreeQueue(block.BoxID, block.RootID) - } - } + treenode.InitBlockTree() + incReindex(upsertFiles, removeFiles) + return } if !boot && !exit { - // 增量索引 - for _, upsertFile := range upsertFiles { - if !strings.HasSuffix(upsertFile, ".sy") { - continue - } - - upsertFile = filepath.ToSlash(upsertFile) - box := upsertFile[:strings.Index(upsertFile, "/")] - p := strings.TrimPrefix(upsertFile, box) - tree, err0 := LoadTree(box, p) - if nil != err0 { - continue - } - treenode.ReindexBlockTree(tree) - sql.UpsertTreeQueue(tree) - //util.LogInfof("sync index tree [%s]", tree.ID) - } - for _, removeFile := range removeFiles { - if !strings.HasSuffix(removeFile, ".sy") { - continue - } - id := strings.TrimSuffix(filepath.Base(removeFile), ".sy") - block := treenode.GetBlockTree(id) - if nil != block { - treenode.RemoveBlockTreesByRootID(block.RootID) - sql.RemoveTreeQueue(block.BoxID, block.RootID) - //util.LogInfof("sync remove tree [%s]", block.RootID) - } - } + incReindex(upsertFiles, removeFiles) cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939 util.ReloadUI() } @@ -495,6 +437,38 @@ func clearEmptyDirs(dir string) { } } +// incReindex 增量重建索引。 +func incReindex(upserts, removes []string) { + for _, upsertFile := range upserts { + if !strings.HasSuffix(upsertFile, ".sy") { + continue + } + + upsertFile = filepath.ToSlash(upsertFile) + box := upsertFile[:strings.Index(upsertFile, "/")] + p := strings.TrimPrefix(upsertFile, box) + tree, err0 := LoadTree(box, p) + if nil != err0 { + continue + } + treenode.ReindexBlockTree(tree) + sql.UpsertTreeQueue(tree) + //util.LogInfof("sync index tree [%s]", tree.ID) + } + for _, removeFile := range removes { + if !strings.HasSuffix(removeFile, ".sy") { + continue + } + id := strings.TrimSuffix(filepath.Base(removeFile), ".sy") + block := treenode.GetBlockTree(id) + if nil != block { + treenode.RemoveBlockTreesByRootID(block.RootID) + sql.RemoveTreeQueue(block.BoxID, block.RootID) + //util.LogInfof("sync remove tree [%s]", block.RootID) + } + } +} + func SetCloudSyncDir(name string) { if Conf.Sync.CloudName == name { return diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index 65e6be77b..9734a7c01 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -217,13 +217,14 @@ func AutoFlushBlockTree() { } } -func ReadBlockTree() (err error) { +func InitBlockTree() { start := time.Now() if nil == blocktreeFileLock { blocktreeFileLock = flock.New(util.BlockTreePath) } + var err error if err = blocktreeFileLock.Lock(); nil != err { util.LogErrorf("read block tree failed: %s", err) os.Exit(util.ExitCodeBlockTreeErr)