diff --git a/kernel/api/filetree.go b/kernel/api/filetree.go index 3890a7119..a0a9f6c9e 100644 --- a/kernel/api/filetree.go +++ b/kernel/api/filetree.go @@ -662,6 +662,9 @@ func getDoc(c *gin.Context) { return } + // 判断是否正在同步中 https://github.com/siyuan-note/siyuan/issues/6290 + isSyncing := model.IsSyncingFile(rootID) + ret.Data = map[string]interface{}{ "id": id, "mode": mode, @@ -675,6 +678,7 @@ func getDoc(c *gin.Context) { "eof": eof, "box": boxID, "path": docPath, + "isSyncing": isSyncing, } } diff --git a/kernel/api/sync.go b/kernel/api/sync.go index 97afc5c25..81a27fbca 100644 --- a/kernel/api/sync.go +++ b/kernel/api/sync.go @@ -45,7 +45,7 @@ func performSync(c *gin.Context) { func performBootSync(c *gin.Context) { ret := gulu.Ret.NewResult() defer c.JSON(http.StatusOK, ret) - model.SyncData(true, false, true) + model.BootSyncData() ret.Code = model.BootSyncSucc } diff --git a/kernel/go.mod b/kernel/go.mod index e44f28164..b3e70126a 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -37,7 +37,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/qiniu/go-sdk/v7 v7.13.0 github.com/radovskyb/watcher v1.0.7 - github.com/siyuan-note/dejavu v0.0.0-20221021094047-e0b46553c42c + github.com/siyuan-note/dejavu v0.0.0-20221022034239-a08f6ea40952 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da github.com/siyuan-note/filelock v0.0.0-20221007163134-7e64809023ef diff --git a/kernel/go.sum b/kernel/go.sum index 99b9b589a..4e0347200 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -357,6 +357,8 @@ github.com/siyuan-note/dejavu v0.0.0-20221021093518-60b24cf55189 h1:o7q3PVZtTTuV github.com/siyuan-note/dejavu v0.0.0-20221021093518-60b24cf55189/go.mod h1:i245FL1nmWaLlor+79tGDWZrAPeDvFOkHPJbZ844PyM= github.com/siyuan-note/dejavu v0.0.0-20221021094047-e0b46553c42c h1:A7zaN6Jfk3Z//qh2aOUoHXzxR6D1qp1epovtDtQWqwY= github.com/siyuan-note/dejavu v0.0.0-20221021094047-e0b46553c42c/go.mod h1:i245FL1nmWaLlor+79tGDWZrAPeDvFOkHPJbZ844PyM= +github.com/siyuan-note/dejavu v0.0.0-20221022034239-a08f6ea40952 h1:OQuOyyvrGLJ+Gj5z3/tUK6tGu+nRbAYelPTCeVrtY38= +github.com/siyuan-note/dejavu v0.0.0-20221022034239-a08f6ea40952/go.mod h1:i245FL1nmWaLlor+79tGDWZrAPeDvFOkHPJbZ844PyM= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da h1:/jNhl7LC+9BhkWvNxuJDdsNfA/2wvfuj9mqWx4CbV90= diff --git a/kernel/main.go b/kernel/main.go index 58f043df6..497492b7c 100644 --- a/kernel/main.go +++ b/kernel/main.go @@ -37,7 +37,7 @@ func main() { sql.InitHistoryDatabase(false) sql.SetCaseSensitive(model.Conf.Search.CaseSensitive) - model.SyncData(true, false, false) + model.BootSyncData() model.InitBoxes() go model.AutoGenerateDocHistory() diff --git a/kernel/mobile/kernel.go b/kernel/mobile/kernel.go index f63b1148f..4ab1d32e5 100644 --- a/kernel/mobile/kernel.go +++ b/kernel/mobile/kernel.go @@ -51,7 +51,7 @@ func StartKernel(container, appDir, workspaceDir, nativeLibDir, privateDataDir, sql.InitHistoryDatabase(false) sql.SetCaseSensitive(model.Conf.Search.CaseSensitive) - model.SyncData(true, false, false) + model.BootSyncData() model.InitBoxes() go model.AutoGenerateDocHistory() diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 6092bb497..9fd59d9cb 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -25,8 +25,10 @@ import ( "fmt" "math" "os" + "path" "path/filepath" "strings" + "sync" "time" "github.com/88250/gulu" @@ -468,7 +470,14 @@ func IndexRepo(memo string) (err error) { return } -func syncRepo(boot, exit, byHand bool) (err error) { +var syncingFiles = sync.Map{} + +func IsSyncingFile(rootID string) (ret bool) { + _, ret = syncingFiles.Load(rootID) + return +} + +func bootSyncRepo() (err error) { if 1 > len(Conf.Repo.Key) { syncDownloadErrCount++ planSyncAfter(fixSyncInterval) @@ -506,11 +515,101 @@ func syncRepo(boot, exit, byHand bool) (err error) { } syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} - _, mergeResult, trafficStat, err := repo.Sync(cloudInfo, syncContext) + fetchedFiles, err := repo.GetSyncCloudFiles(cloudInfo, syncContext) if errors.Is(err, dejavu.ErrRepoFatalErr) { // 重置仓库并再次尝试同步 if _, resetErr := resetRepository(repo); nil == resetErr { - _, mergeResult, trafficStat, err = repo.Sync(cloudInfo, syncContext) + fetchedFiles, err = repo.GetSyncCloudFiles(cloudInfo, syncContext) + } + } + + syncingFiles = sync.Map{} + for _, fetchedFile := range fetchedFiles { + name := path.Base(fetchedFile.Path) + if !(strings.HasSuffix(name, ".sy")) { + continue + } + + id := name[:len(name)-3] + syncingFiles.Store(id, true) + } + + elapsed := time.Since(start) + logging.LogInfof("boot get sync cloud files elapsed [%.2fs]", elapsed.Seconds()) + if nil != err { + syncDownloadErrCount++ + planSyncAfter(fixSyncInterval) + + logging.LogErrorf("sync data repo failed: %s", err) + msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) + if errors.Is(err, dejavu.ErrCloudStorageSizeExceeded) { + msg = fmt.Sprintf(Conf.Language(43), humanize.Bytes(uint64(Conf.User.UserSiYuanRepoSize))) + if 2 == Conf.User.UserSiYuanSubscriptionPlan { + msg = fmt.Sprintf(Conf.Language(68), humanize.Bytes(uint64(Conf.User.UserSiYuanRepoSize))) + } + } + Conf.Sync.Stat = msg + util.PushStatusBar(msg) + util.PushErrMsg(msg, 0) + BootSyncSucc = 1 + return + } + + go func() { + time.Sleep(7 * time.Second) // 等待一段时间后前端完成界面初始化后再同步 + syncErr := syncRepo(false, false) + if nil != err { + logging.LogErrorf("boot background sync repo failed: %s", syncErr) + return + } + syncingFiles = sync.Map{} + }() + return +} + +func syncRepo(exit, byHand bool) (err error) { + if 1 > len(Conf.Repo.Key) { + syncDownloadErrCount++ + planSyncAfter(fixSyncInterval) + + msg := Conf.Language(26) + util.PushStatusBar(msg) + util.PushErrMsg(msg, 0) + err = errors.New(msg) + return + } + + repo, err := newRepository() + if nil != err { + syncDownloadErrCount++ + planSyncAfter(fixSyncInterval) + + msg := fmt.Sprintf("sync repo failed: %s", err) + logging.LogErrorf(msg) + util.PushStatusBar(msg) + util.PushErrMsg(msg, 0) + return + } + + start := time.Now() + err = indexRepoBeforeCloudSync(repo) + if nil != err { + syncDownloadErrCount++ + planSyncAfter(fixSyncInterval) + return + } + + cloudInfo, err := buildCloudInfo() + if nil != err { + return + } + + syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar} + mergeResult, trafficStat, err := repo.Sync(cloudInfo, syncContext) + if errors.Is(err, dejavu.ErrRepoFatalErr) { + // 重置仓库并再次尝试同步 + if _, resetErr := resetRepository(repo); nil == resetErr { + mergeResult, trafficStat, err = repo.Sync(cloudInfo, syncContext) } } elapsed := time.Since(start) @@ -529,9 +628,6 @@ func syncRepo(boot, exit, byHand bool) (err error) { Conf.Sync.Stat = msg util.PushStatusBar(msg) util.PushErrMsg(msg, 0) - if boot { - BootSyncSucc = 1 - } if exit { ExitSyncSucc = 1 } @@ -610,10 +706,6 @@ func syncRepo(boot, exit, byHand bool) (err error) { removes = append(removes, file.Path) } - if boot && gulu.File.IsExist(util.BlockTreePath) { - treenode.InitBlockTree(false) - } - cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939 if needFullReindex(upsertTrees) { // 改进同步后全量重建索引判断 https://github.com/siyuan-note/siyuan/issues/5764 @@ -622,7 +714,7 @@ func syncRepo(boot, exit, byHand bool) (err error) { } incReindex(upserts, removes) - if !boot && !exit { + if !exit { util.ReloadUI() } diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 354ed946d..5ea421333 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -55,6 +55,52 @@ func AutoSync() { } } +func BootSyncData() { + defer logging.Recover() + + if util.IsMutexLocked(&syncLock) { + logging.LogWarnf("sync is in progress") + planSyncAfter(30 * time.Second) + return + } + + syncLock.Lock() + defer syncLock.Unlock() + + util.IncBootProgress(3, "Syncing data from the cloud...") + BootSyncSucc = 0 + + if !IsSubscriber() || !Conf.Sync.Enabled || "" == Conf.Sync.CloudName || !IsValidCloudDirName(Conf.Sync.CloudName) { + return + } + + logging.LogInfof("sync before boot") + + if 7 < syncDownloadErrCount { + logging.LogErrorf("sync download error too many times, cancel auto sync, try to sync by hand") + util.PushErrMsg(Conf.Language(125), 1000*60*60) + planSyncAfter(64 * time.Minute) + return + } + + now := util.CurrentTimeMillis() + Conf.Sync.Synced = now + + util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil) + err := bootSyncRepo() + synced := util.Millisecond2Time(Conf.Sync.Synced).Format("2006-01-02 15:04:05") + "\n\n" + if nil == err { + synced += Conf.Sync.Stat + } else { + synced += fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) + } + msg := fmt.Sprintf(Conf.Language(82), synced) + Conf.Sync.Stat = msg + Conf.Save() + util.BroadcastByType("main", "syncing", 1, msg, nil) + return +} + func SyncData(boot, exit, byHand bool) { defer logging.Recover() @@ -112,7 +158,7 @@ func SyncData(boot, exit, byHand bool) { Conf.Sync.Synced = now util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil) - err := syncRepo(boot, exit, byHand) + err := syncRepo(exit, byHand) synced := util.Millisecond2Time(Conf.Sync.Synced).Format("2006-01-02 15:04:05") + "\n\n" if nil == err { synced += Conf.Sync.Stat