mirror of
https://github.com/siyuan-note/siyuan.git
synced 2026-01-05 16:28:49 +01:00
🎨 优化云端同步上传资源占用和耗时 https://github.com/siyuan-note/siyuan/issues/5093
This commit is contained in:
Liang Ding
parent
a4b948daca
commit
a18d48bdc4
3 changed files with 37 additions and 82 deletions
|
|
@ -278,7 +278,7 @@ func CreateLocalBackup() (err error) {
|
|||
return
|
||||
}
|
||||
|
||||
_, err = genCloudIndex(newBackupDir, map[string]bool{})
|
||||
_, err = genCloudIndex(newBackupDir, map[string]bool{}, true)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
|
|
@ -344,7 +344,7 @@ func DownloadBackup() (err error) {
|
|||
localDirPath := Conf.Backup.GetSaveDir()
|
||||
util.PushEndlessProgress(Conf.Language(68))
|
||||
start := time.Now()
|
||||
fetchedFilesCount, transferSize, _, err := ossDownload(true, localDirPath, "backup", false, map[string]bool{}, map[string]bool{})
|
||||
fetchedFilesCount, transferSize, _, err := ossDownload(localDirPath, "backup", false, map[string]bool{}, map[string]bool{})
|
||||
if nil == err {
|
||||
elapsed := time.Now().Sub(start).Seconds()
|
||||
util.LogInfof("downloaded backup [fetchedFiles=%d, transferSize=%s] in [%.2fs]", fetchedFilesCount, humanize.Bytes(transferSize), elapsed)
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error
|
|||
return
|
||||
}
|
||||
|
||||
func ossDownload(isBackup bool, localDirPath, cloudDirPath string, bootOrExit bool, removeList, upsertList map[string]bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles map[string]bool, err error) {
|
||||
func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool, removeList, upsertList map[string]bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles map[string]bool, err error) {
|
||||
if !gulu.File.IsExist(localDirPath) {
|
||||
return
|
||||
}
|
||||
|
|
@ -183,66 +183,6 @@ func ossDownload(isBackup bool, localDirPath, cloudDirPath string, bootOrExit bo
|
|||
return
|
||||
}
|
||||
|
||||
if !isBackup && (0 < len(removeList) || 0 < len(upsertList)) {
|
||||
// 上传合并本地变更
|
||||
|
||||
//var removed, upserted bool
|
||||
//var removes []string
|
||||
//for remove, _ := range removeList {
|
||||
// removes = append(removes, remove)
|
||||
//}
|
||||
//err = ossRemove0(cloudDirPath, removes)
|
||||
//if nil != err {
|
||||
// util.LogErrorf("remove merge cloud file failed: %s", err)
|
||||
// return
|
||||
//}
|
||||
//for remove, _ := range removeList {
|
||||
// delete(cloudFileList, remove)
|
||||
//}
|
||||
//removed = 0 < len(removeList)
|
||||
//
|
||||
//var tmpWroteFiles int
|
||||
//var tmpTransferSize uint64
|
||||
//for upsert, _ := range upsertList {
|
||||
// if "/.siyuan/conf.json" == upsert { // 版本号不覆盖云端
|
||||
// continue
|
||||
// }
|
||||
// localUpsert := filepath.Join(localDirPath, upsert)
|
||||
// var info os.FileInfo
|
||||
// info, err = os.Stat(localUpsert)
|
||||
// if nil != err {
|
||||
// util.LogErrorf("stat file [%s] failed: %s", localUpsert, err)
|
||||
// return
|
||||
// }
|
||||
//
|
||||
// if err = ossUpload0(localDirPath, cloudDirPath, localUpsert, &tmpWroteFiles, &tmpTransferSize); nil != err {
|
||||
// util.LogErrorf("upload merge cloud file [%s] failed: %s", upsert, err)
|
||||
// return
|
||||
// }
|
||||
// cloudFileList[upsert] = &CloudIndex{
|
||||
// Size: info.Size(),
|
||||
// Updated: info.ModTime().Unix(),
|
||||
// }
|
||||
// upserted = true
|
||||
//}
|
||||
//
|
||||
//if removed || upserted {
|
||||
// data, err = gulu.JSON.MarshalJSON(cloudFileList)
|
||||
// if nil != err {
|
||||
// util.LogErrorf("marshal cloud file list failed: %s", err)
|
||||
// return
|
||||
// }
|
||||
// if err = os.WriteFile(tmpIndex, data, 0644); nil != err {
|
||||
// util.LogErrorf("write cloud file list failed: %s", err)
|
||||
// return
|
||||
// }
|
||||
// if err = ossUpload0(tmpSyncDir, cloudDirPath, tmpIndex, &tmpWroteFiles, &tmpTransferSize); nil != err {
|
||||
// util.LogErrorf("upload merge cloud file [%s] failed: %s", tmpIndex, err)
|
||||
// return
|
||||
// }
|
||||
//}
|
||||
}
|
||||
|
||||
localRemoves, cloudFetches, err := localUpsertRemoveListOSS(localDirPath, cloudFileList)
|
||||
if nil != err {
|
||||
return
|
||||
|
|
@ -406,22 +346,26 @@ func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, bo
|
|||
}
|
||||
|
||||
localDevice := Conf.System.ID
|
||||
excludes := getSyncExcludedList(localDirPath)
|
||||
|
||||
var localFileList, cloudFileList map[string]*CloudIndex
|
||||
if "" != localDevice && localDevice == cloudDevice && !isBackup {
|
||||
cloudFileList, err = getLocalFileListOSS(isBackup, excludes) // 使用上一次的本地索引作为云端索引
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
// 同一台设备连续上传，使用上一次的本地索引作为云端索引
|
||||
cloudFileList, err = getLocalFileListOSS(isBackup)
|
||||
} else {
|
||||
cloudFileList, err = getCloudFileListOSS(cloudDirPath)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
|
||||
calcHash := false
|
||||
if 0 < len(cloudFileList) {
|
||||
if idx := cloudFileList["/index.json"]; nil != idx {
|
||||
calcHash = 0 == idx.Updated
|
||||
}
|
||||
}
|
||||
|
||||
localFileList, err = genCloudIndex(localDirPath, excludes)
|
||||
excludes := getSyncExcludedList(localDirPath)
|
||||
localFileList, err = genCloudIndex(localDirPath, excludes, calcHash)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
|
|
@ -679,7 +623,8 @@ func getCloudSync(cloudDir string) (assetSize, backupSize int64, device string,
|
|||
return
|
||||
}
|
||||
|
||||
func getLocalFileListOSS(isBackup bool, excludes map[string]bool) (ret map[string]*CloudIndex, err error) {
|
||||
func getLocalFileListOSS(isBackup bool) (ret map[string]*CloudIndex, err error) {
|
||||
ret = map[string]*CloudIndex{}
|
||||
dir := "sync"
|
||||
if isBackup {
|
||||
dir = "backup"
|
||||
|
|
@ -688,7 +633,6 @@ func getLocalFileListOSS(isBackup bool, excludes map[string]bool) (ret map[strin
|
|||
localDirPath := filepath.Join(util.WorkspaceDir, dir)
|
||||
indexPath := filepath.Join(localDirPath, "index.json")
|
||||
if !gulu.File.IsExist(indexPath) {
|
||||
ret, err = genCloudIndex(localDirPath, excludes)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -776,8 +720,8 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo
|
|||
// 优先使用时间戳校验
|
||||
if localModTime := info.ModTime().Unix(); cloudIdx.Updated == localModTime {
|
||||
unchanged[relPath] = true
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
localHash, hashErr := util.GetEtag(path)
|
||||
|
|
@ -814,6 +758,14 @@ func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList
|
|||
cloudRemoves = append(cloudRemoves, cloudFile)
|
||||
continue
|
||||
}
|
||||
if 0 < cloudIdx.Updated {
|
||||
// 优先使用时间戳校验
|
||||
if localIdx.Updated == cloudIdx.Updated {
|
||||
unchanged[filepath.Join(localDirPath, cloudFile)] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if localIdx.Hash == cloudIdx.Hash {
|
||||
unchanged[filepath.Join(localDirPath, cloudFile)] = true
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -329,7 +329,7 @@ func SyncData(boot, exit, byHand bool) {
|
|||
return
|
||||
}
|
||||
|
||||
fetchedFilesCount, transferSize, downloadedFiles, err := ossDownload(false, localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit, removeList, upsertList)
|
||||
fetchedFilesCount, transferSize, downloadedFiles, err := ossDownload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit, removeList, upsertList)
|
||||
if nil != err {
|
||||
util.PushClearProgress()
|
||||
msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
|
||||
|
|
@ -613,7 +613,7 @@ type CloudIndex struct {
|
|||
}
|
||||
|
||||
// genCloudIndex 生成云端索引文件。
|
||||
func genCloudIndex(localDirPath string, excludes map[string]bool) (cloudIndex map[string]*CloudIndex, err error) {
|
||||
func genCloudIndex(localDirPath string, excludes map[string]bool, calcHash bool) (cloudIndex map[string]*CloudIndex, err error) {
|
||||
cloudIndex = map[string]*CloudIndex{}
|
||||
err = filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error {
|
||||
if nil != err {
|
||||
|
|
@ -629,11 +629,14 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (cloudIndex ma
|
|||
|
||||
p := strings.TrimPrefix(path, localDirPath)
|
||||
p = filepath.ToSlash(p)
|
||||
// TODO: 优化云端同步上传资源占用和耗时 https://github.com/siyuan-note/siyuan/issues/5093
|
||||
hash, hashErr := util.GetEtag(path)
|
||||
if nil != hashErr {
|
||||
err = hashErr
|
||||
return io.EOF
|
||||
hash := ""
|
||||
if calcHash {
|
||||
var hashErr error
|
||||
hash, hashErr = util.GetEtag(path)
|
||||
if nil != hashErr {
|
||||
err = hashErr
|
||||
return io.EOF
|
||||
}
|
||||
}
|
||||
cloudIndex[p] = &CloudIndex{Hash: hash, Size: info.Size(), Updated: info.ModTime().Unix()}
|
||||
return nil
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue