From afd48a77652f1e05ec0d322685a9e1a838892721 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Sun, 5 Jun 2022 22:52:09 +0800 Subject: [PATCH] =?UTF-8?q?:art:=20=E6=94=AF=E6=8C=81=E5=A4=9A=E8=AE=BE?= =?UTF-8?q?=E5=A4=87=E6=93=8D=E4=BD=9C=E4=B8=8D=E5=90=8C=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E5=90=8E=E4=BA=91=E7=AB=AF=E5=90=8C=E6=AD=A5=E5=90=88=E5=B9=B6?= =?UTF-8?q?=20https://github.com/siyuan-note/siyuan/issues/5092?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/osssync.go | 30 +++++++++++++++++++++++++++--- kernel/model/sync.go | 40 +++++++++++++++++++++++++++------------- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index 967744698..cd28277d4 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -173,7 +173,19 @@ func ossDownload(isBackup bool, localDirPath, cloudDirPath string, bootOrExit bo return } + // 将云端索引文件临时保存一下，后面下载数据时如果部分成功，需要用索引文件恢复部分成功的文件 syncDirUpsertWorkspaceData() + data, err := gulu.JSON.MarshalJSON(cloudFileList) + if nil != err { + return + } + tmpIndex := filepath.Join(util.TempDir, "sync", "index.json") + if err = os.WriteFile(tmpIndex, data, 0644); nil != err { + return + } + if !isBackup && (0 < len(removeList) || 0 < len(upsertList)) { + // 上传合并本地变更 + var removed, upserted bool var removes []string for remove, _ := range removeList { @@ -217,11 +229,12 @@ func ossDownload(isBackup bool, localDirPath, cloudDirPath string, bootOrExit bo if nil != marshalErr { util.LogErrorf("marshal cloud file list failed: %s", marshalErr) } else { - tmpIndex := filepath.Join(util.TempDir, "sync", "cloud.json") + tmpMergeDir := filepath.Join(util.TempDir, "sync") + tmpIndex := filepath.Join(tmpMergeDir, "index.json") if writeErr := os.WriteFile(tmpIndex, data, 0644); nil != writeErr { util.LogErrorf("write cloud file list failed: %s", writeErr) } else { - if uploadErr := ossUpload0(localDirPath, cloudDirPath, tmpIndex, &tmpWroteFiles, &tmpTransferSize); nil != uploadErr { + if uploadErr := ossUpload0(tmpMergeDir, cloudDirPath, tmpIndex, &tmpWroteFiles, &tmpTransferSize); nil != uploadErr { util.LogErrorf("upload merge cloud file [%s] failed: %s", tmpIndex, uploadErr) } } @@ -415,6 +428,17 @@ func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, bo } if 0 < len(downloadList) && !isBackup { // 下载合并云端变更 + + var data []byte + data, err = gulu.JSON.MarshalJSON(cloudFileList) + if nil != err { + return + } + indexPath := filepath.Join(util.TempDir, "sync", "index.json") + if err = os.WriteFile(indexPath, data, 0644); nil != err { + return + } + var tmpFetchedFiles int var tmpTransferSize uint64 err = ossDownload0(util.TempDir+"/sync", "sync/"+Conf.Sync.CloudName, "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, false) @@ -423,7 +447,7 @@ func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, bo } metaPath := filepath.Join(util.TempDir, "/sync/"+pathJSON) - mergeErr := syncDirUpsertWorkspaceData(metaPath, downloadList) + mergeErr := syncDirUpsertWorkspaceData(metaPath, indexPath, downloadList) if nil != mergeErr { util.LogErrorf("download merge cloud file failed: %s", mergeErr) } else { diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 3e0505c51..d669a45c3 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -353,7 +353,8 @@ func SyncData(boot, exit, byHand bool) { util.PushErrMsg(msg, 7000) metaPath := filepath.Join(Conf.Sync.GetSaveDir(), pathJSON) - err = syncDirUpsertWorkspaceData(metaPath, downloadedFiles) + indexPath := filepath.Join(util.TempDir, "sync", "index.json") + err = syncDirUpsertWorkspaceData(metaPath, indexPath, downloadedFiles) if nil != err { util.LogErrorf("upsert partially downloaded files to workspace data failed: %s", err) } @@ -530,7 +531,7 @@ func SetSyncMode(mode int) (err error) { var syncLock = sync.Mutex{} -func syncDirUpsertWorkspaceData(metaPath string, downloadedFiles map[string]bool) (err error) { +func syncDirUpsertWorkspaceData(metaPath, indexPath string, downloadedFiles map[string]bool) (err error) { start := time.Now() modified := map[string]bool{} @@ -540,7 +541,7 @@ func syncDirUpsertWorkspaceData(metaPath string, downloadedFiles map[string]bool modified[file] = true } - decryptedDataDir, _, err := recoverSyncData(metaPath, modified) + decryptedDataDir, _, err := recoverSyncData(metaPath, indexPath, modified) if nil != err { util.LogErrorf("decrypt data dir failed: %s", err) return @@ -569,7 +570,8 @@ func syncDir2WorkspaceData(boot bool) (upsertFiles, removeFiles []string, err er modified := modifiedSyncList(unchanged) metaPath := filepath.Join(Conf.Sync.GetSaveDir(), pathJSON) - decryptedDataDir, upsertFiles, err := recoverSyncData(metaPath, modified) + indexPath := filepath.Join(Conf.Sync.GetSaveDir(), "index.json") + decryptedDataDir, upsertFiles, err := recoverSyncData(metaPath, indexPath, modified) if nil != err { util.LogErrorf("decrypt data dir failed: %s", err) return @@ -661,7 +663,7 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (cloudIndex ma return } -func recoverSyncData(metaPath string, modified map[string]bool) (decryptedDataDir string, upsertFiles []string, err error) { +func recoverSyncData(metaPath, indexPath string, modified map[string]bool) (decryptedDataDir string, upsertFiles []string, err error) { passwd := Conf.E2EEPasswd decryptedDataDir = filepath.Join(util.WorkspaceDir, "incremental", "sync-decrypt") if err = os.RemoveAll(decryptedDataDir); nil != err { @@ -687,7 +689,15 @@ func recoverSyncData(metaPath string, modified map[string]bool) (decryptedDataDi return } - modTimes := map[string]time.Time{} + index := map[string]*CloudIndex{} + data, err = os.ReadFile(indexPath) + if nil != err { + return + } + if err = gulu.JSON.UnmarshalJSON(data, &index); nil != err { + return + } + now := time.Now().Format("2006-01-02-150405") filepath.Walk(syncDir, func(path string, info fs.FileInfo, _ error) error { if syncDir == path || pathJSON == info.Name() { @@ -745,15 +755,19 @@ func recoverSyncData(metaPath string, modified map[string]bool) (decryptedDataDi err = err0 return io.EOF } - } - fi, err0 := os.Stat(path) - if nil != err0 { - util.LogErrorf("stat file [%s] failed: %s", path, err0) - err = err0 - return io.EOF + var modTime int64 + idx := index["/"+encryptedP] + if nil == idx { + util.LogErrorf("index file [%s] not found", encryptedP) + modTime = info.ModTime().Unix() + } else { + modTime = idx.Updated + } + if err0 = os.Chtimes(plainP, time.Unix(modTime, 0), time.Unix(modTime, 0)); nil != err0 { + util.LogErrorf("change file [%s] time failed: %s", plainP, err0) + } } - modTimes[plainP] = fi.ModTime() return nil }) return