mirror of
https://github.com/siyuan-note/siyuan.git
synced 2026-01-04 07:48:49 +01:00
Merge remote-tracking branch 'origin/dev' into dev
This commit is contained in:
Vanessa
commit
bb09e1e102
3 changed files with 135 additions and 72 deletions
|
|
@ -278,7 +278,7 @@ func CreateLocalBackup() (err error) {
|
|||
return
|
||||
}
|
||||
|
||||
err = genCloudIndex(newBackupDir, map[string]bool{})
|
||||
err = genFullCloudIndex(newBackupDir, map[string]bool{})
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
|
|
@ -364,7 +364,7 @@ func UploadBackup() (err error) {
|
|||
util.PushEndlessProgress(Conf.Language(61))
|
||||
util.LogInfof("uploading backup...")
|
||||
start := time.Now()
|
||||
wroteFiles, transferSize, err := ossUpload(localDirPath, "backup", "", false)
|
||||
wroteFiles, transferSize, err := ossUpload(localDirPath, "backup", "not exist", false, map[string]bool{}, map[string]bool{})
|
||||
if nil == err {
|
||||
elapsed := time.Now().Sub(start).Seconds()
|
||||
util.LogInfof("uploaded backup [wroteFiles=%d, transferSize=%s] in [%.2fs]", wroteFiles, humanize.Bytes(transferSize), elapsed)
|
||||
|
|
|
|||
|
|
@ -318,17 +318,36 @@ func ossDownload0(localDirPath, cloudDirPath, fetch string, fetchedFiles *int, t
|
|||
return
|
||||
}
|
||||
|
||||
func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool) (wroteFiles int, transferSize uint64, err error) {
|
||||
func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool, removedSyncList, upsertedSyncList map[string]bool) (wroteFiles int, transferSize uint64, err error) {
|
||||
if !gulu.File.IsExist(localDirPath) {
|
||||
return
|
||||
}
|
||||
|
||||
var cloudFileList map[string]*CloudIndex
|
||||
localDevice := Conf.System.ID
|
||||
|
||||
syncIgnoreList := getSyncIgnoreList()
|
||||
excludes := map[string]bool{}
|
||||
ignores := syncIgnoreList.Values()
|
||||
for _, p := range ignores {
|
||||
relPath := p.(string)
|
||||
relPath = pathSha256Short(relPath, "/")
|
||||
relPath = filepath.Join(localDirPath, relPath)
|
||||
excludes[relPath] = true
|
||||
}
|
||||
localFileList, getLocalFileListErr := getLocalFileListOSS(cloudDirPath, excludes)
|
||||
if "" != localDevice && localDevice == cloudDevice {
|
||||
//util.LogInfof("cloud device is the same as local device, get index from local")
|
||||
cloudFileList, err = getLocalFileListOSS(cloudDirPath)
|
||||
if nil != err {
|
||||
if nil == getLocalFileListErr {
|
||||
cloudFileList = map[string]*CloudIndex{}
|
||||
// 深拷贝一次，避免后面和 localFileList 对比时引用值相同
|
||||
for p, idx := range localFileList {
|
||||
cloudFileList[p] = &CloudIndex{
|
||||
Hash: idx.Hash,
|
||||
Size: idx.Size,
|
||||
}
|
||||
}
|
||||
} else {
|
||||
util.LogInfof("get local index failed [%s], get index from cloud", err)
|
||||
cloudFileList, err = getCloudFileListOSS(cloudDirPath)
|
||||
}
|
||||
|
|
@ -339,7 +358,12 @@ func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool) (wrote
|
|||
return
|
||||
}
|
||||
|
||||
localUpserts, cloudRemoves, err := cloudUpsertRemoveListOSS(localDirPath, cloudFileList)
|
||||
localUpserts, cloudRemoves, err := cloudUpsertRemoveListOSS(localDirPath, cloudFileList, localFileList, removedSyncList, upsertedSyncList, excludes)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
|
||||
err = ossRemove0(cloudDirPath, cloudRemoves)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
|
|
@ -400,8 +424,6 @@ func ossUpload(localDirPath, cloudDirPath, cloudDevice string, boot bool) (wrote
|
|||
if needPushProgress {
|
||||
util.PushMsg(Conf.Language(105), 3000)
|
||||
}
|
||||
|
||||
err = ossRemove0(cloudDirPath, cloudRemoves)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -584,20 +606,27 @@ func getCloudSync(cloudDir string) (assetSize, backupSize int64, device string,
|
|||
return
|
||||
}
|
||||
|
||||
func getLocalFileListOSS(dirPath string) (ret map[string]*CloudIndex, err error) {
|
||||
func getLocalFileListOSS(dirPath string, excludes map[string]bool) (ret map[string]*CloudIndex, err error) {
|
||||
dir := "sync"
|
||||
if !strings.HasPrefix(dirPath, "sync") {
|
||||
dir = "backup"
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(util.WorkspaceDir, dir, "index.json"))
|
||||
localDirPath := filepath.Join(util.WorkspaceDir, dir)
|
||||
indexPath := filepath.Join(localDirPath, "index.json")
|
||||
if !gulu.File.IsExist(indexPath) {
|
||||
err = genFullCloudIndex(localDirPath, excludes)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(indexPath)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
|
||||
if err = gulu.JSON.UnmarshalJSON(data, &ret); nil != err {
|
||||
return
|
||||
}
|
||||
err = gulu.JSON.UnmarshalJSON(data, &ret)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -696,44 +725,26 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo
|
|||
return
|
||||
}
|
||||
|
||||
func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*CloudIndex) (localUpserts, cloudRemoves []string, err error) {
|
||||
func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList map[string]*CloudIndex, removedSyncList, upsertedSyncList, excludes map[string]bool) (localUpserts, cloudRemoves []string, err error) {
|
||||
localUpserts, cloudRemoves = []string{}, []string{}
|
||||
unchanged := map[string]bool{}
|
||||
for cloudFile, cloudIdx := range cloudFileList {
|
||||
localCheckPath := filepath.Join(localDirPath, cloudFile)
|
||||
if !gulu.File.IsExist(localCheckPath) {
|
||||
cloudRemoves = append(cloudRemoves, cloudFile)
|
||||
continue
|
||||
}
|
||||
|
||||
localHash, hashErr := util.GetEtag(localCheckPath)
|
||||
if nil != hashErr {
|
||||
util.LogErrorf("get local file [%s] hash failed: %s", localCheckPath, hashErr)
|
||||
err = hashErr
|
||||
return
|
||||
}
|
||||
|
||||
if localHash == cloudIdx.Hash {
|
||||
unchanged[localCheckPath] = true
|
||||
}
|
||||
}
|
||||
|
||||
syncIgnoreList := getSyncIgnoreList()
|
||||
excludes := map[string]bool{}
|
||||
ignores := syncIgnoreList.Values()
|
||||
for _, p := range ignores {
|
||||
relPath := p.(string)
|
||||
relPath = pathSha246(relPath, "/")
|
||||
relPath = filepath.Join(localDirPath, relPath)
|
||||
excludes[relPath] = true
|
||||
}
|
||||
|
||||
delete(unchanged, filepath.Join(localDirPath, "index.json")) // 同步偶尔报错 `The system cannot find the path specified.` https://github.com/siyuan-note/siyuan/issues/4942
|
||||
err = genCloudIndex(localDirPath, excludes)
|
||||
if nil != err {
|
||||
if err = genIncCloudIndex(localDirPath, &localFileList, removedSyncList, upsertedSyncList, excludes); nil != err {
|
||||
return
|
||||
}
|
||||
|
||||
unchanged := map[string]bool{}
|
||||
for cloudFile, cloudIdx := range cloudFileList {
|
||||
localIdx := localFileList[cloudFile]
|
||||
if nil == localIdx {
|
||||
cloudRemoves = append(cloudRemoves, cloudFile)
|
||||
continue
|
||||
}
|
||||
if localIdx.Hash == cloudIdx.Hash {
|
||||
unchanged[filepath.Join(localDirPath, cloudFile)] = true
|
||||
}
|
||||
}
|
||||
|
||||
delete(unchanged, filepath.Join(localDirPath, "index.json")) // 同步偶尔报错 `The system cannot find the path specified.` https://github.com/siyuan-note/siyuan/issues/4942
|
||||
filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error {
|
||||
if localDirPath == path || info.IsDir() {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -135,8 +135,9 @@ func SyncData(boot, exit, byHand bool) {
|
|||
WaitForWritingFiles()
|
||||
writingDataLock.Lock()
|
||||
var err error
|
||||
var removedSyncList, upsertedSyncList map[string]bool
|
||||
// 将 data 变更同步到 sync
|
||||
if err = workspaceData2SyncDir(); nil != err {
|
||||
if removedSyncList, upsertedSyncList, err = workspaceData2SyncDir(); nil != err {
|
||||
msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
|
||||
Conf.Sync.Stat = msg
|
||||
util.PushErrMsg(msg, 7000)
|
||||
|
|
@ -245,7 +246,7 @@ func SyncData(boot, exit, byHand bool) {
|
|||
return
|
||||
}
|
||||
|
||||
wroteFiles, transferSize, err := ossUpload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, device, boot)
|
||||
wroteFiles, transferSize, err := ossUpload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, device, boot, removedSyncList, upsertedSyncList)
|
||||
if nil != err {
|
||||
util.PushClearMsg()
|
||||
IncWorkspaceDataVer() // 上传失败的话提升本地版本，以备下次上传
|
||||
|
|
@ -560,7 +561,7 @@ func syncDirUpsertWorkspaceData(downloadedFiles []string) (err error) {
|
|||
// 2. 将 sync 中新增/修改的文件解密后拷贝到 data 中
|
||||
func syncDir2WorkspaceData(boot bool) (upsertFiles, removeFiles []string, err error) {
|
||||
start := time.Now()
|
||||
unchanged, removeFiles, err := unchangedSyncList()
|
||||
unchanged, removeFiles, err := calcUnchangedSyncList()
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
|
|
@ -590,18 +591,17 @@ func syncDir2WorkspaceData(boot bool) (upsertFiles, removeFiles []string, err er
|
|||
// workspaceData2SyncDir 将 data 的数据更新到 sync 中。
|
||||
// 1. 删除 sync 中多余的文件
|
||||
// 2. 将 data 中新增/修改的文件加密后拷贝到 sync 中
|
||||
func workspaceData2SyncDir() (err error) {
|
||||
func workspaceData2SyncDir() (removedSyncList, upsertedSyncList map[string]bool, err error) {
|
||||
start := time.Now()
|
||||
filesys.ReleaseAllFileLocks()
|
||||
|
||||
passwd := Conf.E2EEPasswd
|
||||
unchanged, removed, err := unchangedDataList(passwd)
|
||||
unchangedDataList, removedSyncList, err := calcUnchangedDataList(passwd)
|
||||
if nil != err {
|
||||
return
|
||||
}
|
||||
_ = removed // TODO: 支持多设备操作不同文档后云端同步合并 https://github.com/siyuan-note/siyuan/issues/5092
|
||||
|
||||
encryptedDataDir, err := prepareSyncData(passwd, unchanged)
|
||||
encryptedDataDir, upsertedSyncList, err := prepareSyncData(passwd, unchangedDataList)
|
||||
if nil != err {
|
||||
util.LogErrorf("encrypt data dir failed: %s", err)
|
||||
return
|
||||
|
|
@ -623,9 +623,9 @@ type CloudIndex struct {
|
|||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
func genCloudIndex(localDirPath string, excludes map[string]bool) (err error) {
|
||||
// genCloudIndex 全量生成云端索引文件。
|
||||
func genFullCloudIndex(localDirPath string, excludes map[string]bool) (err error) {
|
||||
cloudIndex := map[string]*CloudIndex{}
|
||||
// TODO: 优化云端同步资源占用 https://github.com/siyuan-note/siyuan/issues/5093
|
||||
err = filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error {
|
||||
if nil != err {
|
||||
return err
|
||||
|
|
@ -665,6 +665,46 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (err error) {
|
|||
return
|
||||
}
|
||||
|
||||
// genCloudIndex 增量生成云端索引文件。
|
||||
func genIncCloudIndex(localDirPath string, localFileList *map[string]*CloudIndex, removes, upserts, excludes map[string]bool) (err error) {
|
||||
for remove, _ := range removes {
|
||||
delete(*localFileList, remove)
|
||||
}
|
||||
for exclude, _ := range excludes {
|
||||
delete(*localFileList, filepath.ToSlash(strings.TrimPrefix(exclude, localDirPath)))
|
||||
}
|
||||
|
||||
for upsert, _ := range upserts {
|
||||
path := filepath.Join(localDirPath, upsert)
|
||||
if excludes[path] {
|
||||
continue
|
||||
}
|
||||
|
||||
info, statErr := os.Stat(path)
|
||||
if nil != statErr {
|
||||
util.LogErrorf("stat file [%s] failed: %s", path, statErr)
|
||||
return statErr
|
||||
}
|
||||
hash, hashErr := util.GetEtag(path)
|
||||
if nil != hashErr {
|
||||
util.LogErrorf("get file [%s] hash failed: %s", path, hashErr)
|
||||
return hashErr
|
||||
}
|
||||
(*localFileList)[upsert] = &CloudIndex{Hash: hash, Size: info.Size()}
|
||||
}
|
||||
|
||||
data, err := gulu.JSON.MarshalJSON(localFileList)
|
||||
if nil != err {
|
||||
util.LogErrorf("marshal sync cloud index failed: %s", err)
|
||||
return
|
||||
}
|
||||
if err = os.WriteFile(filepath.Join(localDirPath, "index.json"), data, 0644); nil != err {
|
||||
util.LogErrorf("write sync cloud index failed: %s", err)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func recoverSyncData(modified map[string]bool) (decryptedDataDir string, upsertFiles []string, err error) {
|
||||
passwd := Conf.E2EEPasswd
|
||||
decryptedDataDir = filepath.Join(util.WorkspaceDir, "incremental", "sync-decrypt")
|
||||
|
|
@ -764,7 +804,7 @@ func recoverSyncData(modified map[string]bool) (decryptedDataDir string, upsertF
|
|||
return
|
||||
}
|
||||
|
||||
func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDataDir string, err error) {
|
||||
func prepareSyncData(passwd string, unchangedDataList map[string]bool) (encryptedDataDir string, upsertedSyncList map[string]bool, err error) {
|
||||
encryptedDataDir = filepath.Join(util.WorkspaceDir, "incremental", "sync-encrypt")
|
||||
if err = os.RemoveAll(encryptedDataDir); nil != err {
|
||||
return
|
||||
|
|
@ -792,12 +832,12 @@ func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDat
|
|||
p := plainP
|
||||
|
||||
if !strings.HasPrefix(plainP, ".siyuan") { // 配置目录下都用明文，其他文件需要映射文件名
|
||||
p = pathSha246(p, string(os.PathSeparator))
|
||||
p = pathSha256Short(p, string(os.PathSeparator))
|
||||
}
|
||||
metaJSON[filepath.ToSlash(p)] = filepath.ToSlash(plainP)
|
||||
|
||||
// 如果不是新增或者修改则跳过
|
||||
if unchangedList[path] {
|
||||
if unchangedDataList[path] {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -857,6 +897,7 @@ func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDat
|
|||
}
|
||||
}
|
||||
|
||||
upsertedSyncList = map[string]bool{}
|
||||
// 检查文件是否全部已经编入索引
|
||||
err = filepath.Walk(encryptedDataDir, func(path string, info fs.FileInfo, _ error) error {
|
||||
if encryptedDataDir == path {
|
||||
|
|
@ -869,6 +910,10 @@ func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDat
|
|||
util.LogErrorf("not found sync path in meta [%s]", path)
|
||||
return errors.New(Conf.Language(27))
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
upsertedSyncList["/"+path] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if nil != err {
|
||||
|
|
@ -881,7 +926,8 @@ func prepareSyncData(passwd string, unchangedList map[string]bool) (encryptedDat
|
|||
}
|
||||
data, err = encryption.AESGCMEncryptBinBytes(data, passwd)
|
||||
if nil != err {
|
||||
return "", errors.New("encrypt file failed")
|
||||
util.LogErrorf("encrypt file failed: %s", err)
|
||||
return
|
||||
}
|
||||
meta := filepath.Join(encryptedDataDir, pathJSON)
|
||||
if err = os.WriteFile(meta, data, 0644); nil != err {
|
||||
|
|
@ -907,8 +953,8 @@ func modifiedSyncList(unchangedList map[string]bool) (ret map[string]bool) {
|
|||
return
|
||||
}
|
||||
|
||||
// unchangedSyncList 获取 data 和 sync 一致（没有修改过）的文件列表，并删除 data 中不存在于 sync 中的多余文件。
|
||||
func unchangedSyncList() (ret map[string]bool, removes []string, err error) {
|
||||
// calcUnchangedSyncList 获取 data 和 sync 一致（没有修改过）的文件列表，并删除 data 中不存在于 sync 中的多余文件。
|
||||
func calcUnchangedSyncList() (ret map[string]bool, removes []string, err error) {
|
||||
syncDir := Conf.Sync.GetSaveDir()
|
||||
meta := filepath.Join(syncDir, pathJSON)
|
||||
if !gulu.File.IsExist(meta) {
|
||||
|
|
@ -935,7 +981,7 @@ func unchangedSyncList() (ret map[string]bool, removes []string, err error) {
|
|||
ignores := syncIgnoreList.Values()
|
||||
for _, p := range ignores {
|
||||
relPath := p.(string)
|
||||
relPath = pathSha246(relPath, "/")
|
||||
relPath = pathSha256Short(relPath, "/")
|
||||
relPath = filepath.Join(syncDir, relPath)
|
||||
excludes[relPath] = true
|
||||
}
|
||||
|
|
@ -956,7 +1002,7 @@ func unchangedSyncList() (ret map[string]bool, removes []string, err error) {
|
|||
|
||||
plainP := strings.TrimPrefix(path, util.DataDir+sep)
|
||||
dataP := plainP
|
||||
dataP = pathSha246(dataP, sep)
|
||||
dataP = pathSha256Short(dataP, sep)
|
||||
syncP := filepath.Join(syncDir, dataP)
|
||||
|
||||
if excludes[syncP] {
|
||||
|
|
@ -993,8 +1039,8 @@ func unchangedSyncList() (ret map[string]bool, removes []string, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
// unchangedDataList 获取 sync 和 data 一致（没有修改过）的文件列表，并删除 sync 中不存在于 data 中的多余文件。
|
||||
func unchangedDataList(passwd string) (unchangedList map[string]bool, removeList map[string]bool, err error) {
|
||||
// calcUnchangedDataList 计算 sync 和 data 一致（没有修改过）的文件列表 unchangedDataList，并删除 sync 中不存在于 data 中的多余文件 removedSyncList。
|
||||
func calcUnchangedDataList(passwd string) (unchangedDataList map[string]bool, removedSyncList map[string]bool, err error) {
|
||||
syncDir := Conf.Sync.GetSaveDir()
|
||||
meta := filepath.Join(syncDir, pathJSON)
|
||||
if !gulu.File.IsExist(meta) {
|
||||
|
|
@ -1015,8 +1061,8 @@ func unchangedDataList(passwd string) (unchangedList map[string]bool, removeList
|
|||
return
|
||||
}
|
||||
|
||||
unchangedList = map[string]bool{}
|
||||
removeList = map[string]bool{}
|
||||
unchangedDataList = map[string]bool{}
|
||||
removedSyncList = map[string]bool{}
|
||||
filepath.Walk(syncDir, func(path string, info fs.FileInfo, _ error) error {
|
||||
if syncDir == path || pathJSON == info.Name() {
|
||||
return nil
|
||||
|
|
@ -1026,7 +1072,7 @@ func unchangedDataList(passwd string) (unchangedList map[string]bool, removeList
|
|||
encryptedP = filepath.ToSlash(encryptedP)
|
||||
decryptedP := metaJSON[encryptedP]
|
||||
if "" == decryptedP { // 理论上不会发生
|
||||
removeList[path] = true
|
||||
removedSyncList[path] = true
|
||||
if gulu.File.IsDir(path) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
|
@ -1035,7 +1081,7 @@ func unchangedDataList(passwd string) (unchangedList map[string]bool, removeList
|
|||
dataP := filepath.Join(util.DataDir, decryptedP)
|
||||
dataP = filepath.FromSlash(dataP)
|
||||
if !gulu.File.IsExist(dataP) { // data 已经删除的文件
|
||||
removeList[path] = true
|
||||
removedSyncList[path] = true
|
||||
if gulu.File.IsDir(path) {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
|
|
@ -1045,22 +1091,28 @@ func unchangedDataList(passwd string) (unchangedList map[string]bool, removeList
|
|||
stat, _ := os.Stat(dataP)
|
||||
dataModTime := stat.ModTime()
|
||||
if info.ModTime() == dataModTime {
|
||||
unchangedList[dataP] = true
|
||||
unchangedDataList[dataP] = true
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
tmp := map[string]bool{}
|
||||
// 在 sync 中删除 data 中已经删除的文件
|
||||
for remove, _ := range removeList {
|
||||
for remove, _ := range removedSyncList {
|
||||
if strings.HasSuffix(remove, "index.json") {
|
||||
continue
|
||||
}
|
||||
|
||||
p := strings.TrimPrefix(remove, syncDir)
|
||||
p = filepath.ToSlash(p)
|
||||
tmp[p] = true
|
||||
|
||||
if err = os.RemoveAll(remove); nil != err {
|
||||
util.LogErrorf("remove [%s] failed: %s", remove, err)
|
||||
}
|
||||
}
|
||||
removedSyncList = tmp
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -1291,7 +1343,7 @@ func getSyncIgnoreList() (ret *hashset.Set) {
|
|||
return
|
||||
}
|
||||
|
||||
func pathSha246(p, sep string) string {
|
||||
func pathSha256Short(p, sep string) string {
|
||||
buf := bytes.Buffer{}
|
||||
parts := strings.Split(p, sep)
|
||||
for i, part := range parts {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue