diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index fc0c1c9fc..8393c5c15 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "io" "io/fs" "os" "path" @@ -33,8 +34,6 @@ import ( "github.com/88250/gulu" "github.com/panjf2000/ants/v2" "github.com/qiniu/go-sdk/v7/storage" - "github.com/siyuan-note/siyuan/kernel/sql" - "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -187,61 +186,61 @@ func ossDownload(isBackup bool, localDirPath, cloudDirPath string, bootOrExit bo if !isBackup && (0 < len(removeList) || 0 < len(upsertList)) { // 上传合并本地变更 - var removed, upserted bool - var removes []string - for remove, _ := range removeList { - removes = append(removes, remove) - } - err = ossRemove0(cloudDirPath, removes) - if nil != err { - util.LogErrorf("remove merge cloud file failed: %s", err) - return - } - for remove, _ := range removeList { - delete(cloudFileList, remove) - } - removed = 0 < len(removeList) - - var tmpWroteFiles int - var tmpTransferSize uint64 - for upsert, _ := range upsertList { - if "/.siyuan/conf.json" == upsert { // 版本号不覆盖云端 - continue - } - localUpsert := filepath.Join(localDirPath, upsert) - var info os.FileInfo - info, err = os.Stat(localUpsert) - if nil != err { - util.LogErrorf("stat file [%s] failed: %s", localUpsert, err) - return - } - - if err = ossUpload0(localDirPath, cloudDirPath, localUpsert, &tmpWroteFiles, &tmpTransferSize); nil != err { - util.LogErrorf("upload merge cloud file [%s] failed: %s", upsert, err) - return - } - cloudFileList[upsert] = &CloudIndex{ - Size: info.Size(), - Updated: info.ModTime().Unix(), - } - upserted = true - } - - if removed || upserted { - data, err = gulu.JSON.MarshalJSON(cloudFileList) - if nil != err { - util.LogErrorf("marshal cloud file list failed: %s", err) - return - } - if err = os.WriteFile(tmpIndex, data, 0644); nil != err { - util.LogErrorf("write cloud file list failed: %s", err) - return - } - if err = ossUpload0(tmpSyncDir, cloudDirPath, tmpIndex, &tmpWroteFiles, &tmpTransferSize); nil != err { - util.LogErrorf("upload merge cloud file [%s] failed: %s", tmpIndex, err) - return - } - } + //var removed, upserted bool + //var removes []string + //for remove, _ := range removeList { + // removes = append(removes, remove) + //} + //err = ossRemove0(cloudDirPath, removes) + //if nil != err { + // util.LogErrorf("remove merge cloud file failed: %s", err) + // return + //} + //for remove, _ := range removeList { + // delete(cloudFileList, remove) + //} + //removed = 0 < len(removeList) + // + //var tmpWroteFiles int + //var tmpTransferSize uint64 + //for upsert, _ := range upsertList { + // if "/.siyuan/conf.json" == upsert { // 版本号不覆盖云端 + // continue + // } + // localUpsert := filepath.Join(localDirPath, upsert) + // var info os.FileInfo + // info, err = os.Stat(localUpsert) + // if nil != err { + // util.LogErrorf("stat file [%s] failed: %s", localUpsert, err) + // return + // } + // + // if err = ossUpload0(localDirPath, cloudDirPath, localUpsert, &tmpWroteFiles, &tmpTransferSize); nil != err { + // util.LogErrorf("upload merge cloud file [%s] failed: %s", upsert, err) + // return + // } + // cloudFileList[upsert] = &CloudIndex{ + // Size: info.Size(), + // Updated: info.ModTime().Unix(), + // } + // upserted = true + //} + // + //if removed || upserted { + // data, err = gulu.JSON.MarshalJSON(cloudFileList) + // if nil != err { + // util.LogErrorf("marshal cloud file list failed: %s", err) + // return + // } + // if err = os.WriteFile(tmpIndex, data, 0644); nil != err { + // util.LogErrorf("write cloud file list failed: %s", err) + // return + // } + // if err = ossUpload0(tmpSyncDir, cloudDirPath, tmpIndex, &tmpWroteFiles, &tmpTransferSize); nil != err { + // util.LogErrorf("upload merge cloud file [%s] failed: %s", tmpIndex, err) + // return + // } + //} } localRemoves, cloudFetches, err := localUpsertRemoveListOSS(localDirPath, cloudFileList) @@ -435,53 +434,53 @@ func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, bo if 0 < len(downloadList) && !isBackup { // 下载合并云端变更 - var data []byte - data, err = gulu.JSON.MarshalJSON(cloudFileList) - if nil != err { - return - } - tmpSyncDir := filepath.Join(util.TempDir, "sync") - indexPath := filepath.Join(tmpSyncDir, "index.json") - if err = os.WriteFile(indexPath, data, 0644); nil != err { - return - } - - var tmpFetchedFiles int - var tmpTransferSize uint64 - err = ossDownload0(tmpSyncDir, "sync/"+Conf.Sync.CloudName, "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, false) - if nil != err { - util.LogErrorf("download merge cloud file failed: %s", err) - return - } - - metaPath := filepath.Join(tmpSyncDir, pathJSON) - var upsertFiles []string - upsertFiles, err = syncDirUpsertWorkspaceData(metaPath, indexPath, downloadList) - if nil != err { - util.LogErrorf("download merge cloud file failed: %s", err) - return - } - // 增量索引 - for _, upsertFile := range upsertFiles { - if !strings.HasSuffix(upsertFile, ".sy") { - continue - } - - upsertFile = filepath.ToSlash(upsertFile) - box := upsertFile[:strings.Index(upsertFile, "/")] - p := strings.TrimPrefix(upsertFile, box) - tree, err0 := LoadTree(box, p) - if nil != err0 { - continue - } - treenode.ReindexBlockTree(tree) - sql.UpsertTreeQueue(tree) - } - - // 重新生成云端索引 - if _, err = genCloudIndex(localDirPath, excludes); nil != err { - return - } + //var data []byte + //data, err = gulu.JSON.MarshalJSON(cloudFileList) + //if nil != err { + // return + //} + //tmpSyncDir := filepath.Join(util.TempDir, "sync") + //indexPath := filepath.Join(tmpSyncDir, "index.json") + //if err = os.WriteFile(indexPath, data, 0644); nil != err { + // return + //} + // + //var tmpFetchedFiles int + //var tmpTransferSize uint64 + //err = ossDownload0(tmpSyncDir, "sync/"+Conf.Sync.CloudName, "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, false) + //if nil != err { + // util.LogErrorf("download merge cloud file failed: %s", err) + // return + //} + // + //metaPath := filepath.Join(tmpSyncDir, pathJSON) + //var upsertFiles []string + //upsertFiles, err = syncDirUpsertWorkspaceData(metaPath, indexPath, downloadList) + //if nil != err { + // util.LogErrorf("download merge cloud file failed: %s", err) + // return + //} + //// 增量索引 + //for _, upsertFile := range upsertFiles { + // if !strings.HasSuffix(upsertFile, ".sy") { + // continue + // } + // + // upsertFile = filepath.ToSlash(upsertFile) + // box := upsertFile[:strings.Index(upsertFile, "/")] + // p := strings.TrimPrefix(upsertFile, box) + // tree, err0 := LoadTree(box, p) + // if nil != err0 { + // continue + // } + // treenode.ReindexBlockTree(tree) + // sql.UpsertTreeQueue(tree) + //} + // + //// 重新生成云端索引 + //if _, err = genCloudIndex(localDirPath, excludes); nil != err { + // return + //} } } @@ -803,8 +802,20 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo return nil } - localModTime := info.ModTime().Unix() - if cloudIdx.Updated == localModTime { + if 0 < cloudIdx.Updated { + // 优先使用时间戳校验 + if localModTime := info.ModTime().Unix(); cloudIdx.Updated == localModTime { + unchanged[relPath] = true + return nil + } + } + + localHash, hashErr := util.GetEtag(path) + if nil != hashErr { + err = hashErr + return io.EOF + } + if cloudIdx.Hash == localHash { unchanged[relPath] = true } return nil @@ -861,10 +872,14 @@ func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList cloudRemoves = append(cloudRemoves, cloudFile) continue } - if localIdx.Updated == cloudIdx.Updated { + if localIdx.Hash == cloudIdx.Hash { unchanged[filepath.Join(localDirPath, cloudFile)] = true continue } + //if localIdx.Updated == cloudIdx.Updated { + // unchanged[filepath.Join(localDirPath, cloudFile)] = true + // continue + //} cloudChangedList[cloudFile] = true } diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 9467dc26b..811107ec4 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -607,8 +607,9 @@ func workspaceData2SyncDir() (removeList, upsertList map[string]bool, err error) } type CloudIndex struct { - Size int64 `json:"size"` - Updated int64 `json:"updated"` // Unix timestamp 秒 + Hash string `json:"hash"` + Size int64 `json:"size"` + Updated int64 `json:"updated"` // Unix timestamp 秒 } // genCloudIndex 生成云端索引文件。 @@ -628,7 +629,13 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (cloudIndex ma p := strings.TrimPrefix(path, localDirPath) p = filepath.ToSlash(p) - cloudIndex[p] = &CloudIndex{Size: info.Size(), Updated: info.ModTime().Unix()} + // TODO: 优化云端同步上传资源占用和耗时 https://github.com/siyuan-note/siyuan/issues/5093 + hash, hashErr := util.GetEtag(path) + if nil != hashErr { + err = hashErr + return io.EOF + } + cloudIndex[p] = &CloudIndex{Hash: hash, Size: info.Size(), Updated: info.ModTime().Unix()} return nil }) if nil != err {