From 0b54a97246da3003b585a46d1c343ac67a2ef16f Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 6 Jun 2022 13:26:36 +0800 Subject: [PATCH 1/3] =?UTF-8?q?:art:=20=E6=94=AF=E6=8C=81=E5=A4=9A?= =?UTF-8?q?=E8=AE=BE=E5=A4=87=E6=93=8D=E4=BD=9C=E4=B8=8D=E5=90=8C=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E5=90=8E=E4=BA=91=E7=AB=AF=E5=90=8C=E6=AD=A5=E5=90=88?= =?UTF-8?q?=E5=B9=B6=20https://github.com/siyuan-note/siyuan/issues/5092?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/osssync.go | 206 ++++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 104 deletions(-) diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index fc0c1c9fc..16381198d 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -33,8 +33,6 @@ import ( "github.com/88250/gulu" "github.com/panjf2000/ants/v2" "github.com/qiniu/go-sdk/v7/storage" - "github.com/siyuan-note/siyuan/kernel/sql" - "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -187,61 +185,61 @@ func ossDownload(isBackup bool, localDirPath, cloudDirPath string, bootOrExit bo if !isBackup && (0 < len(removeList) || 0 < len(upsertList)) { // 上传合并本地变更 - var removed, upserted bool - var removes []string - for remove, _ := range removeList { - removes = append(removes, remove) - } - err = ossRemove0(cloudDirPath, removes) - if nil != err { - util.LogErrorf("remove merge cloud file failed: %s", err) - return - } - for remove, _ := range removeList { - delete(cloudFileList, remove) - } - removed = 0 < len(removeList) - - var tmpWroteFiles int - var tmpTransferSize uint64 - for upsert, _ := range upsertList { - if "/.siyuan/conf.json" == upsert { // 版本号不覆盖云端 - continue - } - localUpsert := filepath.Join(localDirPath, upsert) - var info os.FileInfo - info, err = os.Stat(localUpsert) - if nil != err { - util.LogErrorf("stat file [%s] failed: %s", localUpsert, err) - return - } - - if err = ossUpload0(localDirPath, cloudDirPath, localUpsert, &tmpWroteFiles, &tmpTransferSize); nil != err { - util.LogErrorf("upload merge cloud file [%s] failed: %s", upsert, err) - return - } - cloudFileList[upsert] = &CloudIndex{ - Size: info.Size(), - Updated: info.ModTime().Unix(), - } - upserted = true - } - - if removed || upserted { - data, err = gulu.JSON.MarshalJSON(cloudFileList) - if nil != err { - util.LogErrorf("marshal cloud file list failed: %s", err) - return - } - if err = os.WriteFile(tmpIndex, data, 0644); nil != err { - util.LogErrorf("write cloud file list failed: %s", err) - return - } - if err = ossUpload0(tmpSyncDir, cloudDirPath, tmpIndex, &tmpWroteFiles, &tmpTransferSize); nil != err { - util.LogErrorf("upload merge cloud file [%s] failed: %s", tmpIndex, err) - return - } - } + //var removed, upserted bool + //var removes []string + //for remove, _ := range removeList { + // removes = append(removes, remove) + //} + //err = ossRemove0(cloudDirPath, removes) + //if nil != err { + // util.LogErrorf("remove merge cloud file failed: %s", err) + // return + //} + //for remove, _ := range removeList { + // delete(cloudFileList, remove) + //} + //removed = 0 < len(removeList) + // + //var tmpWroteFiles int + //var tmpTransferSize uint64 + //for upsert, _ := range upsertList { + // if "/.siyuan/conf.json" == upsert { // 版本号不覆盖云端 + // continue + // } + // localUpsert := filepath.Join(localDirPath, upsert) + // var info os.FileInfo + // info, err = os.Stat(localUpsert) + // if nil != err { + // util.LogErrorf("stat file [%s] failed: %s", localUpsert, err) + // return + // } + // + // if err = ossUpload0(localDirPath, cloudDirPath, localUpsert, &tmpWroteFiles, &tmpTransferSize); nil != err { + // util.LogErrorf("upload merge cloud file [%s] failed: %s", upsert, err) + // return + // } + // cloudFileList[upsert] = &CloudIndex{ + // Size: info.Size(), + // Updated: info.ModTime().Unix(), + // } + // upserted = true + //} + // + //if removed || upserted { + // data, err = gulu.JSON.MarshalJSON(cloudFileList) + // if nil != err { + // util.LogErrorf("marshal cloud file list failed: %s", err) + // return + // } + // if err = os.WriteFile(tmpIndex, data, 0644); nil != err { + // util.LogErrorf("write cloud file list failed: %s", err) + // return + // } + // if err = ossUpload0(tmpSyncDir, cloudDirPath, tmpIndex, &tmpWroteFiles, &tmpTransferSize); nil != err { + // util.LogErrorf("upload merge cloud file [%s] failed: %s", tmpIndex, err) + // return + // } + //} } localRemoves, cloudFetches, err := localUpsertRemoveListOSS(localDirPath, cloudFileList) @@ -435,53 +433,53 @@ func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, bo if 0 < len(downloadList) && !isBackup { // 下载合并云端变更 - var data []byte - data, err = gulu.JSON.MarshalJSON(cloudFileList) - if nil != err { - return - } - tmpSyncDir := filepath.Join(util.TempDir, "sync") - indexPath := filepath.Join(tmpSyncDir, "index.json") - if err = os.WriteFile(indexPath, data, 0644); nil != err { - return - } - - var tmpFetchedFiles int - var tmpTransferSize uint64 - err = ossDownload0(tmpSyncDir, "sync/"+Conf.Sync.CloudName, "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, false) - if nil != err { - util.LogErrorf("download merge cloud file failed: %s", err) - return - } - - metaPath := filepath.Join(tmpSyncDir, pathJSON) - var upsertFiles []string - upsertFiles, err = syncDirUpsertWorkspaceData(metaPath, indexPath, downloadList) - if nil != err { - util.LogErrorf("download merge cloud file failed: %s", err) - return - } - // 增量索引 - for _, upsertFile := range upsertFiles { - if !strings.HasSuffix(upsertFile, ".sy") { - continue - } - - upsertFile = filepath.ToSlash(upsertFile) - box := upsertFile[:strings.Index(upsertFile, "/")] - p := strings.TrimPrefix(upsertFile, box) - tree, err0 := LoadTree(box, p) - if nil != err0 { - continue - } - treenode.ReindexBlockTree(tree) - sql.UpsertTreeQueue(tree) - } - - // 重新生成云端索引 - if _, err = genCloudIndex(localDirPath, excludes); nil != err { - return - } + //var data []byte + //data, err = gulu.JSON.MarshalJSON(cloudFileList) + //if nil != err { + // return + //} + //tmpSyncDir := filepath.Join(util.TempDir, "sync") + //indexPath := filepath.Join(tmpSyncDir, "index.json") + //if err = os.WriteFile(indexPath, data, 0644); nil != err { + // return + //} + // + //var tmpFetchedFiles int + //var tmpTransferSize uint64 + //err = ossDownload0(tmpSyncDir, "sync/"+Conf.Sync.CloudName, "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, false) + //if nil != err { + // util.LogErrorf("download merge cloud file failed: %s", err) + // return + //} + // + //metaPath := filepath.Join(tmpSyncDir, pathJSON) + //var upsertFiles []string + //upsertFiles, err = syncDirUpsertWorkspaceData(metaPath, indexPath, downloadList) + //if nil != err { + // util.LogErrorf("download merge cloud file failed: %s", err) + // return + //} + //// 增量索引 + //for _, upsertFile := range upsertFiles { + // if !strings.HasSuffix(upsertFile, ".sy") { + // continue + // } + // + // upsertFile = filepath.ToSlash(upsertFile) + // box := upsertFile[:strings.Index(upsertFile, "/")] + // p := strings.TrimPrefix(upsertFile, box) + // tree, err0 := LoadTree(box, p) + // if nil != err0 { + // continue + // } + // treenode.ReindexBlockTree(tree) + // sql.UpsertTreeQueue(tree) + //} + // + //// 重新生成云端索引 + //if _, err = genCloudIndex(localDirPath, excludes); nil != err { + // return + //} } } From e8351aab047ed459f30ff1986fc4caff66452f4a Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 6 Jun 2022 13:44:54 +0800 Subject: [PATCH 2/3] =?UTF-8?q?:art:=20=E4=BC=98=E5=8C=96=E4=BA=91?= =?UTF-8?q?=E7=AB=AF=E5=90=8C=E6=AD=A5=E4=B8=8A=E4=BC=A0=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E5=8D=A0=E7=94=A8=E5=92=8C=E8=80=97=E6=97=B6=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/5093?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/osssync.go | 22 +++++++++++++++++++--- kernel/model/sync.go | 13 ++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index 16381198d..c83477cb2 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "io" "io/fs" "os" "path" @@ -801,10 +802,21 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo return nil } - localModTime := info.ModTime().Unix() - if cloudIdx.Updated == localModTime { + // TODO: 优化云端同步上传资源占用和耗时 https://github.com/siyuan-note/siyuan/issues/5093 + localHash, hashErr := util.GetEtag(path) + if nil != hashErr { + err = hashErr + return io.EOF + } + + if cloudIdx.Hash == localHash { unchanged[relPath] = true } + + //localModTime := info.ModTime().Unix() + //if cloudIdx.Updated == localModTime { + // unchanged[relPath] = true + //} return nil }) @@ -859,10 +871,14 @@ func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList cloudRemoves = append(cloudRemoves, cloudFile) continue } - if localIdx.Updated == cloudIdx.Updated { + if localIdx.Hash == cloudIdx.Hash { unchanged[filepath.Join(localDirPath, cloudFile)] = true continue } + //if localIdx.Updated == cloudIdx.Updated { + // unchanged[filepath.Join(localDirPath, cloudFile)] = true + // continue + //} cloudChangedList[cloudFile] = true } diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 9467dc26b..811107ec4 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -607,8 +607,9 @@ func workspaceData2SyncDir() (removeList, upsertList map[string]bool, err error) } type CloudIndex struct { - Size int64 `json:"size"` - Updated int64 `json:"updated"` // Unix timestamp 秒 + Hash string `json:"hash"` + Size int64 `json:"size"` + Updated int64 `json:"updated"` // Unix timestamp 秒 } // genCloudIndex 生成云端索引文件。 @@ -628,7 +629,13 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (cloudIndex ma p := strings.TrimPrefix(path, localDirPath) p = filepath.ToSlash(p) - cloudIndex[p] = &CloudIndex{Size: info.Size(), Updated: info.ModTime().Unix()} + // TODO: 优化云端同步上传资源占用和耗时 https://github.com/siyuan-note/siyuan/issues/5093 + hash, hashErr := util.GetEtag(path) + if nil != hashErr { + err = hashErr + return io.EOF + } + cloudIndex[p] = &CloudIndex{Hash: hash, Size: info.Size(), Updated: info.ModTime().Unix()} return nil }) if nil != err { From 7f5ddc58380f5cc06587746d841a8ab264588ef0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 6 Jun 2022 15:06:25 +0800 Subject: [PATCH 3/3] =?UTF-8?q?:art:=20=E4=BC=98=E5=8C=96=E4=BA=91?= =?UTF-8?q?=E7=AB=AF=E5=90=8C=E6=AD=A5=E4=B8=8A=E4=BC=A0=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E5=8D=A0=E7=94=A8=E5=92=8C=E8=80=97=E6=97=B6=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/5093?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/osssync.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index c83477cb2..8393c5c15 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -802,21 +802,22 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo return nil } - // TODO: 优化云端同步上传资源占用和耗时 https://github.com/siyuan-note/siyuan/issues/5093 + if 0 < cloudIdx.Updated { + // 优先使用时间戳校验 + if localModTime := info.ModTime().Unix(); cloudIdx.Updated == localModTime { + unchanged[relPath] = true + return nil + } + } + localHash, hashErr := util.GetEtag(path) if nil != hashErr { err = hashErr return io.EOF } - if cloudIdx.Hash == localHash { unchanged[relPath] = true } - - //localModTime := info.ModTime().Unix() - //if cloudIdx.Updated == localModTime { - // unchanged[relPath] = true - //} return nil })