From cd5c993fbde38b3d4f3d99b2a170d4c8164c0b22 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Tue, 31 May 2022 10:25:06 +0800 Subject: [PATCH] =?UTF-8?q?:art:=20=E5=90=8C=E6=AD=A5=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=96=AD=E7=82=B9=E7=BB=AD=E4=BC=A0=20https:?= =?UTF-8?q?//github.com/siyuan-note/siyuan/issues/5056?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/backup.go | 4 ++-- kernel/model/osssync.go | 19 ++++++++++++------- kernel/model/sync.go | 39 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/kernel/model/backup.go b/kernel/model/backup.go index 364586df9..a2fc332af 100644 --- a/kernel/model/backup.go +++ b/kernel/model/backup.go @@ -344,10 +344,10 @@ func DownloadBackup() (err error) { localDirPath := Conf.Backup.GetSaveDir() util.PushEndlessProgress(Conf.Language(68)) start := time.Now() - fetchedFiles, transferSize, err := ossDownload(localDirPath, "backup", false) + fetchedFilesCount, transferSize, _, err := ossDownload(localDirPath, "backup", false) if nil == err { elapsed := time.Now().Sub(start).Seconds() - util.LogInfof("downloaded backup [fetchedFiles=%d, transferSize=%s] in [%.2fs]", fetchedFiles, humanize.Bytes(transferSize), elapsed) + util.LogInfof("downloaded backup [fetchedFiles=%d, transferSize=%s] in [%.2fs]", fetchedFilesCount, humanize.Bytes(transferSize), elapsed) util.PushEndlessProgress(Conf.Language(69)) } return diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index d203134d2..010c49b31 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -161,7 +161,7 @@ func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error return } -func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFiles int, transferSize uint64, err error) { +func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles []string, err error) { if !gulu.File.IsExist(localDirPath) { return } @@ -187,24 +187,29 @@ func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFil waitGroup := &sync.WaitGroup{} var downloadErr error poolSize := 4 - if poolSize > len(cloudFetches) { + if poolSize > len(cloudFetches)-1 /* 不计入 /.siyuan/conf.json，配置文件最后单独下载 */ { poolSize = len(cloudFetches) } p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) { defer waitGroup.Done() + if nil != downloadErr { + return // 快速失败 + } fetch := arg.(string) - err = ossDownload0(localDirPath, cloudDirPath, fetch, &fetchedFiles, &transferSize, bootOrExit) + err = ossDownload0(localDirPath, cloudDirPath, fetch, &fetchedFilesCount, &transferSize, bootOrExit) if nil != err { downloadErr = err // 仅记录最后一次错误 return } + downloadedFiles = append(downloadedFiles, fetch) + if needPushProgress { - msg := fmt.Sprintf(Conf.Language(103), fetchedFiles, len(cloudFetches)-fetchedFiles) - util.PushProgress(util.PushProgressCodeProgressed, fetchedFiles, len(cloudFetches), msg) + msg := fmt.Sprintf(Conf.Language(103), fetchedFilesCount, len(cloudFetches)-fetchedFilesCount) + util.PushProgress(util.PushProgressCodeProgressed, fetchedFilesCount, len(cloudFetches), msg) } if bootOrExit { - msg := fmt.Sprintf("Downloading data from the cloud %d/%d", fetchedFiles, len(cloudFetches)) + msg := fmt.Sprintf("Downloading data from the cloud %d/%d", fetchedFilesCount, len(cloudFetches)) util.IncBootProgress(0, msg) } }) @@ -224,7 +229,7 @@ func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFil return } - err = ossDownload0(localDirPath, cloudDirPath, "/.siyuan/conf.json", &fetchedFiles, &transferSize, bootOrExit) + err = ossDownload0(localDirPath, cloudDirPath, "/.siyuan/conf.json", &fetchedFilesCount, &transferSize, bootOrExit) if nil != err { return } diff --git a/kernel/model/sync.go b/kernel/model/sync.go index f7ec9bcb4..350e76832 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -321,12 +321,18 @@ func SyncData(boot, exit, byHand bool) { return } - fetchedFiles, transferSize, err := ossDownload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit) + fetchedFilesCount, transferSize, downloadedFiles, err := ossDownload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit) if nil != err { util.PushClearMsg() msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) Conf.Sync.Stat = msg util.PushErrMsg(msg, 7000) + + err = syncDirUpsertWorkspaceData(downloadedFiles) + if nil != err { + util.LogErrorf("upsert partially downloaded files to workspace data failed: %s", err) + } + if boot { BootSyncSucc = 1 } @@ -360,8 +366,8 @@ func SyncData(boot, exit, byHand bool) { clearEmptyDirs(util.DataDir) elapsed := time.Now().Sub(start).Seconds() - stat := fmt.Sprintf(Conf.Language(129), fetchedFiles, humanize.Bytes(transferSize)) + fmt.Sprintf(Conf.Language(131), elapsed) - util.LogInfof("sync [cloud=%d, local=%d, fetchedFiles=%d, transferSize=%s] downloaded in [%.2fs]", cloudSyncVer, syncConf.SyncVer, fetchedFiles, humanize.Bytes(transferSize), elapsed) + stat := fmt.Sprintf(Conf.Language(129), fetchedFilesCount, humanize.Bytes(transferSize)) + fmt.Sprintf(Conf.Language(131), elapsed) + util.LogInfof("sync [cloud=%d, local=%d, fetchedFiles=%d, transferSize=%s] downloaded in [%.2fs]", cloudSyncVer, syncConf.SyncVer, fetchedFilesCount, humanize.Bytes(transferSize), elapsed) Conf.Sync.Downloaded = now Conf.Sync.Stat = stat @@ -490,6 +496,33 @@ func SetSyncEnable(b bool) (err error) { var syncLock = sync.Mutex{} +func syncDirUpsertWorkspaceData(downloadedFiles []string) (err error) { + start := time.Now() + + modified := map[string]bool{} + syncDir := Conf.Sync.GetSaveDir() + for _, file := range downloadedFiles { + file = filepath.Join(syncDir, file) + modified[file] = true + } + + decryptedDataDir, _, err := recoverSyncData(modified) + if nil != err { + util.LogErrorf("decrypt data dir failed: %s", err) + return + } + + dataDir := util.DataDir + if err = stableCopy(decryptedDataDir, dataDir); nil != err { + util.LogErrorf("copy decrypted data dir from [%s] to data dir [%s] failed: %s", decryptedDataDir, dataDir, err) + return + } + if elapsed := time.Since(start).Milliseconds(); 5000 < elapsed { + util.LogInfof("sync data to workspace data elapsed [%dms]", elapsed) + } + return +} + // syncDir2WorkspaceData 将 sync 的数据更新到 data 中。 // 1. 删除 data 中冗余的文件 // 2. 将 sync 中新增/修改的文件解密后拷贝到 data 中