diff --git a/kernel/model/backup.go b/kernel/model/backup.go index a2fc332af..fc41a6253 100644 --- a/kernel/model/backup.go +++ b/kernel/model/backup.go @@ -278,7 +278,7 @@ func CreateLocalBackup() (err error) { return } - err = genCloudIndex(newBackupDir, map[string]bool{}) + err = genFullCloudIndex(newBackupDir, map[string]bool{}) if nil != err { return } @@ -364,7 +364,7 @@ func UploadBackup() (err error) { util.PushEndlessProgress(Conf.Language(61)) util.LogInfof("uploading backup...") start := time.Now() - wroteFiles, transferSize, err := ossUpload(localDirPath, "backup", "", false) + wroteFiles, transferSize, err := ossUpload(localDirPath, "backup", "not exist", false, map[string]bool{}, map[string]bool{}) if nil == err { elapsed := time.Now().Sub(start).Seconds() util.LogInfof("uploaded backup [wroteFiles=%d, transferSize=%s] in [%.2fs]", wroteFiles, humanize.Bytes(transferSize), elapsed) diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index 572938a60..baa3adf1c 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -318,17 +318,36 @@ func ossDownload0(localDirPath, cloudDirPath, fetch string, fetchedFiles *int, t return } -func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool) (wroteFiles int, transferSize uint64, err error) { +func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool, removedSyncList, upsertedSyncList map[string]bool) (wroteFiles int, transferSize uint64, err error) { if !gulu.File.IsExist(localDirPath) { return } var cloudFileList map[string]*CloudIndex localDevice := Conf.System.ID + + syncIgnoreList := getSyncIgnoreList() + excludes := map[string]bool{} + ignores := syncIgnoreList.Values() + for _, p := range ignores { + relPath := p.(string) + relPath = pathSha256Short(relPath, "/") + relPath = filepath.Join(localDirPath, relPath) + excludes[relPath] = true + } + localFileList, getLocalFileListErr := getLocalFileListOSS(cloudDirPath, excludes) if "" != localDevice && localDevice == cloudDevice { //util.LogInfof("cloud device is the same as local device, get index from local") - cloudFileList, err = getLocalFileListOSS(cloudDirPath) - if nil != err { + if nil == getLocalFileListErr { + cloudFileList = map[string]*CloudIndex{} + // 深拷贝一次，避免后面和 localFileList 对比时引用值相同 + for p, idx := range localFileList { + cloudFileList[p] = &CloudIndex{ + Hash: idx.Hash, + Size: idx.Size, + } + } + } else { util.LogInfof("get local index failed [%s], get index from cloud", err) cloudFileList, err = getCloudFileListOSS(cloudDirPath) } @@ -339,7 +358,12 @@ func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool) (wrote return } - localUpserts, cloudRemoves, err := cloudUpsertRemoveListOSS(localDirPath, cloudFileList) + localUpserts, cloudRemoves, err := cloudUpsertRemoveListOSS(localDirPath, cloudFileList, localFileList, removedSyncList, upsertedSyncList, excludes) + if nil != err { + return + } + + err = ossRemove0(cloudDirPath, cloudRemoves) if nil != err { return } @@ -400,8 +424,6 @@ func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool) (wrote if needPushProgress { util.PushMsg(Conf.Language(105), 3000) } - - err = ossRemove0(cloudDirPath, cloudRemoves) return } @@ -584,20 +606,27 @@ func getCloudSync(cloudDir string) (assetSize, backupSize int64, device string, return } -func getLocalFileListOSS(dirPath string) (ret map[string]*CloudIndex, err error) { +func getLocalFileListOSS(dirPath string, excludes map[string]bool) (ret map[string]*CloudIndex, err error) { dir := "sync" if !strings.HasPrefix(dirPath, "sync") { dir = "backup" } - data, err := os.ReadFile(filepath.Join(util.WorkspaceDir, dir, "index.json")) + localDirPath := filepath.Join(util.WorkspaceDir, dir) + indexPath := filepath.Join(localDirPath, "index.json") + if !gulu.File.IsExist(indexPath) { + err = genFullCloudIndex(localDirPath, excludes) + if nil != err { + return + } + } + + data, err := os.ReadFile(indexPath) if nil != err { return } - if err = gulu.JSON.UnmarshalJSON(data, &ret); nil != err { - return - } + err = gulu.JSON.UnmarshalJSON(data, &ret) return } @@ -696,44 +725,26 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo return } -func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*CloudIndex) (localUpserts, cloudRemoves []string, err error) { +func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList map[string]*CloudIndex, removedSyncList, upsertedSyncList, excludes map[string]bool) (localUpserts, cloudRemoves []string, err error) { localUpserts, cloudRemoves = []string{}, []string{} - unchanged := map[string]bool{} - for cloudFile, cloudIdx := range cloudFileList { - localCheckPath := filepath.Join(localDirPath, cloudFile) - if !gulu.File.IsExist(localCheckPath) { - cloudRemoves = append(cloudRemoves, cloudFile) - continue - } - localHash, hashErr := util.GetEtag(localCheckPath) - if nil != hashErr { - util.LogErrorf("get local file [%s] hash failed: %s", localCheckPath, hashErr) - err = hashErr - return - } - - if localHash == cloudIdx.Hash { - unchanged[localCheckPath] = true - } - } - - syncIgnoreList := getSyncIgnoreList() - excludes := map[string]bool{} - ignores := syncIgnoreList.Values() - for _, p := range ignores { - relPath := p.(string) - relPath = pathSha256Short(relPath, "/") - relPath = filepath.Join(localDirPath, relPath) - excludes[relPath] = true - } - - delete(unchanged, filepath.Join(localDirPath, "index.json")) // 同步偶尔报错 `The system cannot find the path specified.` https://github.com/siyuan-note/siyuan/issues/4942 - err = genCloudIndex(localDirPath, excludes) - if nil != err { + if err = genIncCloudIndex(localDirPath, &localFileList, removedSyncList, upsertedSyncList, excludes); nil != err { return } + unchanged := map[string]bool{} + for cloudFile, cloudIdx := range cloudFileList { + localIdx := localFileList[cloudFile] + if nil == localIdx { + cloudRemoves = append(cloudRemoves, cloudFile) + continue + } + if localIdx.Hash == cloudIdx.Hash { + unchanged[filepath.Join(localDirPath, cloudFile)] = true + } + } + + delete(unchanged, filepath.Join(localDirPath, "index.json")) // 同步偶尔报错 `The system cannot find the path specified.` https://github.com/siyuan-note/siyuan/issues/4942 filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error { if localDirPath == path || info.IsDir() { return nil diff --git a/kernel/model/sync.go b/kernel/model/sync.go index de3039706..ef40a9b8d 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -135,8 +135,9 @@ func SyncData(boot, exit, byHand bool) { WaitForWritingFiles() writingDataLock.Lock() var err error + var removedSyncList, upsertedSyncList map[string]bool // 将 data 变更同步到 sync - if err = workspaceData2SyncDir(); nil != err { + if removedSyncList, upsertedSyncList, err = workspaceData2SyncDir(); nil != err { msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) Conf.Sync.Stat = msg util.PushErrMsg(msg, 7000) @@ -245,7 +246,7 @@ func SyncData(boot, exit, byHand bool) { return } - wroteFiles, transferSize, err := ossUpload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, device, boot) + wroteFiles, transferSize, err := ossUpload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, device, boot, removedSyncList, upsertedSyncList) if nil != err { util.PushClearMsg() IncWorkspaceDataVer() // 上传失败的话提升本地版本，以备下次上传 @@ -560,7 +561,7 @@ func syncDirUpsertWorkspaceData(downloadedFiles []string) (err error) { // 2. 将 sync 中新增/修改的文件解密后拷贝到 data 中 func syncDir2WorkspaceData(boot bool) (upsertFiles, removeFiles []string, err error) { start := time.Now() - unchanged, removeFiles, err := unchangedSyncList() + unchanged, removeFiles, err := calcUnchangedSyncList() if nil != err { return } @@ -590,7 +591,7 @@ func syncDir2WorkspaceData(boot bool) (upsertFiles, removeFiles []string, err er // workspaceData2SyncDir 将 data 的数据更新到 sync 中。 // 1. 删除 sync 中多余的文件 // 2. 将 data 中新增/修改的文件加密后拷贝到 sync 中 -func workspaceData2SyncDir() (err error) { +func workspaceData2SyncDir() (removedSyncList, upsertedSyncList map[string]bool, err error) { start := time.Now() filesys.ReleaseAllFileLocks() @@ -599,9 +600,8 @@ func workspaceData2SyncDir() (err error) { if nil != err { return } - _ = removedSyncList // TODO: 支持多设备操作不同文档后云端同步合并 https://github.com/siyuan-note/siyuan/issues/5092 - encryptedDataDir, err := prepareSyncData(passwd, unchangedDataList) + encryptedDataDir, upsertedSyncList, err := prepareSyncData(passwd, unchangedDataList) if nil != err { util.LogErrorf("encrypt data dir failed: %s", err) return @@ -623,9 +623,9 @@ type CloudIndex struct { Size int64 `json:"size"` } -func genCloudIndex(localDirPath string, excludes map[string]bool) (err error) { +// genCloudIndex 全量生成云端索引文件。 +func genFullCloudIndex(localDirPath string, excludes map[string]bool) (err error) { cloudIndex := map[string]*CloudIndex{} - // TODO: 优化云端同步资源占用 https://github.com/siyuan-note/siyuan/issues/5093 err = filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error { if nil != err { return err @@ -665,6 +665,46 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (err error) { return } +// genCloudIndex 增量生成云端索引文件。 +func genIncCloudIndex(localDirPath string, localFileList *map[string]*CloudIndex, removes, upserts, excludes map[string]bool) (err error) { + for remove, _ := range removes { + delete(*localFileList, remove) + } + for exclude, _ := range excludes { + delete(*localFileList, filepath.ToSlash(strings.TrimPrefix(exclude, localDirPath))) + } + + for upsert, _ := range upserts { + path := filepath.Join(localDirPath, upsert) + if excludes[path] { + continue + } + + info, statErr := os.Stat(path) + if nil != statErr { + util.LogErrorf("stat file [%s] failed: %s", path, statErr) + return statErr + } + hash, hashErr := util.GetEtag(path) + if nil != hashErr { + util.LogErrorf("get file [%s] hash failed: %s", path, hashErr) + return hashErr + } + (*localFileList)[upsert] = &CloudIndex{Hash: hash, Size: info.Size()} + } + + data, err := gulu.JSON.MarshalJSON(localFileList) + if nil != err { + util.LogErrorf("marshal sync cloud index failed: %s", err) + return + } + if err = os.WriteFile(filepath.Join(localDirPath, "index.json"), data, 0644); nil != err { + util.LogErrorf("write sync cloud index failed: %s", err) + return + } + return +} + func recoverSyncData(modified map[string]bool) (decryptedDataDir string, upsertFiles []string, err error) { passwd := Conf.E2EEPasswd decryptedDataDir = filepath.Join(util.WorkspaceDir, "incremental", "sync-decrypt") @@ -764,7 +804,7 @@ func recoverSyncData(modified map[string]bool) (decryptedDataDir string, upsertF return } -func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDataDir string, err error) { +func prepareSyncData(passwd string, unchangedDataList map[string]bool) (encryptedDataDir string, upsertedSyncList map[string]bool, err error) { encryptedDataDir = filepath.Join(util.WorkspaceDir, "incremental", "sync-encrypt") if err = os.RemoveAll(encryptedDataDir); nil != err { return @@ -797,7 +837,7 @@ func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDat metaJSON[filepath.ToSlash(p)] = filepath.ToSlash(plainP) // 如果不是新增或者修改则跳过 - if unchangedList[path] { + if unchangedDataList[path] { return nil } @@ -857,6 +897,7 @@ func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDat } } + upsertedSyncList = map[string]bool{} // 检查文件是否全部已经编入索引 err = filepath.Walk(encryptedDataDir, func(path string, info fs.FileInfo, _ error) error { if encryptedDataDir == path { @@ -869,6 +910,10 @@ func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDat util.LogErrorf("not found sync path in meta [%s]", path) return errors.New(Conf.Language(27)) } + + if !info.IsDir() { + upsertedSyncList["/"+path] = true + } return nil }) if nil != err { @@ -881,7 +926,8 @@ func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDat } data, err = encryption.AESGCMEncryptBinBytes(data, passwd) if nil != err { - return "", errors.New("encrypt file failed") + util.LogErrorf("encrypt file failed: %s", err) + return } meta := filepath.Join(encryptedDataDir, pathJSON) if err = os.WriteFile(meta, data, 0644); nil != err { @@ -907,8 +953,8 @@ func modifiedSyncList(unchangedList map[string]bool) (ret map[string]bool) { return } -// unchangedSyncList 获取 data 和 sync 一致（没有修改过）的文件列表，并删除 data 中不存在于 sync 中的多余文件。 -func unchangedSyncList() (ret map[string]bool, removes []string, err error) { +// calcUnchangedSyncList 获取 data 和 sync 一致（没有修改过）的文件列表，并删除 data 中不存在于 sync 中的多余文件。 +func calcUnchangedSyncList() (ret map[string]bool, removes []string, err error) { syncDir := Conf.Sync.GetSaveDir() meta := filepath.Join(syncDir, pathJSON) if !gulu.File.IsExist(meta) { @@ -1051,16 +1097,22 @@ func calcUnchangedDataList(passwd string) (unchangedDataList map[string]bool, re return nil }) + tmp := map[string]bool{} // 在 sync 中删除 data 中已经删除的文件 for remove, _ := range removedSyncList { if strings.HasSuffix(remove, "index.json") { continue } + p := strings.TrimPrefix(remove, syncDir) + p = filepath.ToSlash(p) + tmp[p] = true + if err = os.RemoveAll(remove); nil != err { util.LogErrorf("remove [%s] failed: %s", remove, err) } } + removedSyncList = tmp return }