From a18d48bdc4fa9508710fe3d758f1ab6d1d25362b Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 6 Jun 2022 15:50:33 +0800 Subject: [PATCH] =?UTF-8?q?:art:=20=E4=BC=98=E5=8C=96=E4=BA=91=E7=AB=AF?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E4=B8=8A=E4=BC=A0=E8=B5=84=E6=BA=90=E5=8D=A0?= =?UTF-8?q?=E7=94=A8=E5=92=8C=E8=80=97=E6=97=B6=20https://github.com/siyua?= =?UTF-8?q?n-note/siyuan/issues/5093?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/backup.go | 4 +- kernel/model/osssync.go | 98 +++++++++++------------------------------ kernel/model/sync.go | 17 ++++--- 3 files changed, 37 insertions(+), 82 deletions(-) diff --git a/kernel/model/backup.go b/kernel/model/backup.go index 4d6393ad6..3593d7e54 100644 --- a/kernel/model/backup.go +++ b/kernel/model/backup.go @@ -278,7 +278,7 @@ func CreateLocalBackup() (err error) { return } - _, err = genCloudIndex(newBackupDir, map[string]bool{}) + _, err = genCloudIndex(newBackupDir, map[string]bool{}, true) if nil != err { return } @@ -344,7 +344,7 @@ func DownloadBackup() (err error) { localDirPath := Conf.Backup.GetSaveDir() util.PushEndlessProgress(Conf.Language(68)) start := time.Now() - fetchedFilesCount, transferSize, _, err := ossDownload(true, localDirPath, "backup", false, map[string]bool{}, map[string]bool{}) + fetchedFilesCount, transferSize, _, err := ossDownload(localDirPath, "backup", false, map[string]bool{}, map[string]bool{}) if nil == err { elapsed := time.Now().Sub(start).Seconds() util.LogInfof("downloaded backup [fetchedFiles=%d, transferSize=%s] in [%.2fs]", fetchedFilesCount, humanize.Bytes(transferSize), elapsed) diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index fe7ec3482..50b0b9785 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -162,7 +162,7 @@ func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error return } -func ossDownload(isBackup bool, localDirPath, cloudDirPath string, bootOrExit bool, removeList, upsertList map[string]bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles map[string]bool, err error) { +func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool, removeList, upsertList map[string]bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles map[string]bool, err error) { if !gulu.File.IsExist(localDirPath) { return } @@ -183,66 +183,6 @@ func ossDownload(isBackup bool, localDirPath, cloudDirPath string, bootOrExit bo return } - if !isBackup && (0 < len(removeList) || 0 < len(upsertList)) { - // 上传合并本地变更 - - //var removed, upserted bool - //var removes []string - //for remove, _ := range removeList { - // removes = append(removes, remove) - //} - //err = ossRemove0(cloudDirPath, removes) - //if nil != err { - // util.LogErrorf("remove merge cloud file failed: %s", err) - // return - //} - //for remove, _ := range removeList { - // delete(cloudFileList, remove) - //} - //removed = 0 < len(removeList) - // - //var tmpWroteFiles int - //var tmpTransferSize uint64 - //for upsert, _ := range upsertList { - // if "/.siyuan/conf.json" == upsert { // 版本号不覆盖云端 - // continue - // } - // localUpsert := filepath.Join(localDirPath, upsert) - // var info os.FileInfo - // info, err = os.Stat(localUpsert) - // if nil != err { - // util.LogErrorf("stat file [%s] failed: %s", localUpsert, err) - // return - // } - // - // if err = ossUpload0(localDirPath, cloudDirPath, localUpsert, &tmpWroteFiles, &tmpTransferSize); nil != err { - // util.LogErrorf("upload merge cloud file [%s] failed: %s", upsert, err) - // return - // } - // cloudFileList[upsert] = &CloudIndex{ - // Size: info.Size(), - // Updated: info.ModTime().Unix(), - // } - // upserted = true - //} - // - //if removed || upserted { - // data, err = gulu.JSON.MarshalJSON(cloudFileList) - // if nil != err { - // util.LogErrorf("marshal cloud file list failed: %s", err) - // return - // } - // if err = os.WriteFile(tmpIndex, data, 0644); nil != err { - // util.LogErrorf("write cloud file list failed: %s", err) - // return - // } - // if err = ossUpload0(tmpSyncDir, cloudDirPath, tmpIndex, &tmpWroteFiles, &tmpTransferSize); nil != err { - // util.LogErrorf("upload merge cloud file [%s] failed: %s", tmpIndex, err) - // return - // } - //} - } - localRemoves, cloudFetches, err := localUpsertRemoveListOSS(localDirPath, cloudFileList) if nil != err { return @@ -406,22 +346,26 @@ func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, bo } localDevice := Conf.System.ID - excludes := getSyncExcludedList(localDirPath) - var localFileList, cloudFileList map[string]*CloudIndex if "" != localDevice && localDevice == cloudDevice && !isBackup { - cloudFileList, err = getLocalFileListOSS(isBackup, excludes) // 使用上一次的本地索引作为云端索引 - if nil != err { - return - } + // 同一台设备连续上传，使用上一次的本地索引作为云端索引 + cloudFileList, err = getLocalFileListOSS(isBackup) } else { cloudFileList, err = getCloudFileListOSS(cloudDirPath) - if nil != err { - return + } + if nil != err { + return + } + + calcHash := false + if 0 < len(cloudFileList) { + if idx := cloudFileList["/index.json"]; nil != idx { + calcHash = 0 == idx.Updated } } - localFileList, err = genCloudIndex(localDirPath, excludes) + excludes := getSyncExcludedList(localDirPath) + localFileList, err = genCloudIndex(localDirPath, excludes, calcHash) if nil != err { return } @@ -679,7 +623,8 @@ func getCloudSync(cloudDir string) (assetSize, backupSize int64, device string, return } -func getLocalFileListOSS(isBackup bool, excludes map[string]bool) (ret map[string]*CloudIndex, err error) { +func getLocalFileListOSS(isBackup bool) (ret map[string]*CloudIndex, err error) { + ret = map[string]*CloudIndex{} dir := "sync" if isBackup { dir = "backup" @@ -688,7 +633,6 @@ func getLocalFileListOSS(isBackup bool, excludes map[string]bool) (ret map[strin localDirPath := filepath.Join(util.WorkspaceDir, dir) indexPath := filepath.Join(localDirPath, "index.json") if !gulu.File.IsExist(indexPath) { - ret, err = genCloudIndex(localDirPath, excludes) return } @@ -776,8 +720,8 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo // 优先使用时间戳校验 if localModTime := info.ModTime().Unix(); cloudIdx.Updated == localModTime { unchanged[relPath] = true - return nil } + return nil } localHash, hashErr := util.GetEtag(path) @@ -814,6 +758,14 @@ func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList cloudRemoves = append(cloudRemoves, cloudFile) continue } + if 0 < cloudIdx.Updated { + // 优先使用时间戳校验 + if localIdx.Updated == cloudIdx.Updated { + unchanged[filepath.Join(localDirPath, cloudFile)] = true + } + continue + } + if localIdx.Hash == cloudIdx.Hash { unchanged[filepath.Join(localDirPath, cloudFile)] = true continue diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 811107ec4..3caad4eaa 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -329,7 +329,7 @@ func SyncData(boot, exit, byHand bool) { return } - fetchedFilesCount, transferSize, downloadedFiles, err := ossDownload(false, localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit, removeList, upsertList) + fetchedFilesCount, transferSize, downloadedFiles, err := ossDownload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit, removeList, upsertList) if nil != err { util.PushClearProgress() msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) @@ -613,7 +613,7 @@ type CloudIndex struct { } // genCloudIndex 生成云端索引文件。 -func genCloudIndex(localDirPath string, excludes map[string]bool) (cloudIndex map[string]*CloudIndex, err error) { +func genCloudIndex(localDirPath string, excludes map[string]bool, calcHash bool) (cloudIndex map[string]*CloudIndex, err error) { cloudIndex = map[string]*CloudIndex{} err = filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error { if nil != err { @@ -629,11 +629,14 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (cloudIndex ma p := strings.TrimPrefix(path, localDirPath) p = filepath.ToSlash(p) - // TODO: 优化云端同步上传资源占用和耗时 https://github.com/siyuan-note/siyuan/issues/5093 - hash, hashErr := util.GetEtag(path) - if nil != hashErr { - err = hashErr - return io.EOF + hash := "" + if calcHash { + var hashErr error + hash, hashErr = util.GetEtag(path) + if nil != hashErr { + err = hashErr + return io.EOF + } } cloudIndex[p] = &CloudIndex{Hash: hash, Size: info.Size(), Updated: info.ModTime().Unix()} return nil