This commit is contained in:
Liang Ding 2022-10-31 20:59:00 +08:00
parent ad8b04ccec
commit f02fd1ed93
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
4 changed files with 50 additions and 62 deletions

View file

@ -33,16 +33,16 @@ require (
github.com/mitchellh/go-ps v1.0.0
github.com/mssola/user_agent v0.5.3
github.com/olahol/melody v1.1.1
github.com/panjf2000/ants/v2 v2.5.0
github.com/panjf2000/ants/v2 v2.6.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/qiniu/go-sdk/v7 v7.13.0
github.com/radovskyb/watcher v1.0.7
github.com/siyuan-note/dejavu v0.0.0-20221023024107-17bf52da95a1
github.com/siyuan-note/dejavu v0.0.0-20221031125623-fe9358b6cc16
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da
github.com/siyuan-note/filelock v0.0.0-20221007163134-7e64809023ef
github.com/siyuan-note/httpclient v0.0.0-20221019094331-c904ac9be571
github.com/siyuan-note/logging v0.0.0-20220717040626-f796b05ee520
github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a
github.com/steambap/captcha v1.4.1
github.com/vmihailenco/msgpack/v5 v5.3.5
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673
@ -86,7 +86,7 @@ require (
github.com/imdario/mergo v0.3.13 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/juju/errors v1.0.0 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/compress v1.15.12 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lucas-clemente/quic-go v0.30.0 // indirect
github.com/marten-seemann/qpack v0.3.0 // indirect
@ -99,7 +99,7 @@ require (
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/onsi/ginkgo/v2 v2.3.1 // indirect
github.com/onsi/ginkgo/v2 v2.4.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/restic/chunker v0.4.0 // indirect
@ -111,7 +111,7 @@ require (
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/exp v0.0.0-20221019170559-20944726eadf // indirect
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f // indirect
golang.org/x/mod v0.6.0 // indirect
golang.org/x/net v0.1.0 // indirect
golang.org/x/sync v0.1.0 // indirect

View file

@ -216,6 +216,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@ -285,6 +287,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.3.1 h1:8SbseP7qM32WcvE6VaN6vfXxv698izmsJ1UQX9ve7T8=
github.com/onsi/ginkgo/v2 v2.3.1/go.mod h1:Sv4yQXwG5VmF7tm3Q5Z+RWUpPo24LF1mpnz2crUb8Ys=
github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
@ -292,6 +296,8 @@ github.com/onsi/gomega v1.22.0 h1:AIg2/OntwkBiCg5Tt1ayyiF1ArFrWFoCSMtMi/wdApk=
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
github.com/panjf2000/ants/v2 v2.5.0 h1:1rWGWSnxCsQBga+nQbA4/iY6VMeNoOIAM0ZWh9u3q2Q=
github.com/panjf2000/ants/v2 v2.5.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
github.com/panjf2000/ants/v2 v2.6.0 h1:xOSpw42m+BMiJ2I33we7h6fYzG4DAlpE1xyI7VS2gxU=
github.com/panjf2000/ants/v2 v2.6.0/go.mod h1:cU93usDlihJZ5CfRGNDYsiBYvoilLvBF5Qp/BT2GNRE=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
@ -354,6 +360,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
github.com/siyuan-note/dejavu v0.0.0-20221023024107-17bf52da95a1 h1:sk6pyVp22MfFXK8C5YOKD5eqWtvcMnkiEh1nyZlw4aU=
github.com/siyuan-note/dejavu v0.0.0-20221023024107-17bf52da95a1/go.mod h1:f+6y7KcTLlk2N2dLl5HeV/YzkyCTEqWNoMiWT0gkMi8=
github.com/siyuan-note/dejavu v0.0.0-20221031125623-fe9358b6cc16 h1:05p9tr+rJrKurRQG3NF9i/IPgGdBMPu5tKi1Ux427lY=
github.com/siyuan-note/dejavu v0.0.0-20221031125623-fe9358b6cc16/go.mod h1:+U86jfsvpacZBThE3Ouf/ZQ4EsB4jGPJsMO2iuRv0LQ=
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE=
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da h1:/jNhl7LC+9BhkWvNxuJDdsNfA/2wvfuj9mqWx4CbV90=
@ -364,6 +372,8 @@ github.com/siyuan-note/httpclient v0.0.0-20221019094331-c904ac9be571 h1:AclwUOWg
github.com/siyuan-note/httpclient v0.0.0-20221019094331-c904ac9be571/go.mod h1:FTPEFG3q0LAn33zON47HeRbLOWNk7HM+OPafmQgq4hU=
github.com/siyuan-note/logging v0.0.0-20220717040626-f796b05ee520 h1:kscYjMt7jXYdd7Qj2OSUoBnoHc5B0U/E6OSx86VRLr4=
github.com/siyuan-note/logging v0.0.0-20220717040626-f796b05ee520/go.mod h1:t1zRGxK13L/9ZFoGyTD39IbFCbee3CsypDj4b5dt4qM=
github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a h1:b9VJCE8IccYjsadwNBI11he+Wn25hI9lCma4uYoIYEM=
github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a/go.mod h1:t1zRGxK13L/9ZFoGyTD39IbFCbee3CsypDj4b5dt4qM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.7 h1:I6tZjLXD2Q1kjvNbIzB1wvQBsXmKXiVrhpRE8ZjP5jY=
@ -430,6 +440,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
golang.org/x/exp v0.0.0-20221019170559-20944726eadf h1:nFVjjKDgNY37+ZSYCJmtYf7tOlfQswHqplG2eosjOMg=
golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f h1:Al51T6tzvuh3oiwX11vex3QgJ2XTedFPGmbEVh8cdoc=
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20190823064033-3a9bac650e44/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

View file

@ -33,9 +33,9 @@ import (
"github.com/88250/gulu"
"github.com/dustin/go-humanize"
"github.com/qiniu/go-sdk/v7/storage"
"github.com/siyuan-note/dejavu"
"github.com/siyuan-note/dejavu/entity"
"github.com/siyuan-note/dejavu/transport"
"github.com/siyuan-note/encryption"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/httpclient"
@ -259,13 +259,8 @@ func DownloadCloudSnapshot(tag, id string) (err error) {
return
}
cloudInfo, err := buildCloudInfo()
if nil != err {
return
}
defer util.PushClearProgress()
downloadFileCount, downloadChunkCount, downloadBytes, err := repo.DownloadTagIndex(tag, id, cloudInfo, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
downloadFileCount, downloadChunkCount, downloadBytes, err := repo.DownloadTagIndex(tag, id, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
if nil != err {
return
}
@ -286,14 +281,9 @@ func UploadCloudSnapshot(tag, id string) (err error) {
return
}
cloudInfo, err := buildCloudInfo()
if nil != err {
return
}
util.PushEndlessProgress(Conf.Language(116))
defer util.PushClearProgress()
uploadFileCount, uploadChunkCount, uploadBytes, err := repo.UploadTagIndex(tag, id, cloudInfo, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
uploadFileCount, uploadChunkCount, uploadBytes, err := repo.UploadTagIndex(tag, id, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
if nil != err {
if errors.Is(err, dejavu.ErrCloudBackupCountExceeded) {
err = fmt.Errorf(Conf.Language(84), Conf.Language(154))
@ -324,12 +314,7 @@ func RemoveCloudRepoTag(tag string) (err error) {
return
}
cloudInfo, err := buildCloudInfo()
if nil != err {
return
}
err = repo.RemoveCloudRepoTag(tag, cloudInfo, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
err = repo.RemoveCloudRepoTag(tag, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
if nil != err {
return
}
@ -347,11 +332,7 @@ func GetCloudRepoTagSnapshots() (ret []*dejavu.Log, err error) {
return
}
cloudInfo, err := buildCloudInfo()
if nil != err {
return
}
ret, err = repo.GetCloudRepoTagLogs(cloudInfo, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
ret, err = repo.GetCloudRepoTagLogs(map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
if 1 > len(ret) {
ret = []*dejavu.Log{}
}
@ -509,17 +490,12 @@ func bootSyncRepo() (err error) {
return
}
cloudInfo, err := buildCloudInfo()
if nil != err {
return
}
syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
fetchedFiles, err := repo.GetSyncCloudFiles(cloudInfo, syncContext)
fetchedFiles, err := repo.GetSyncCloudFiles(syncContext)
if errors.Is(err, dejavu.ErrRepoFatalErr) {
// 重置仓库并再次尝试同步
if _, resetErr := resetRepository(repo); nil == resetErr {
fetchedFiles, err = repo.GetSyncCloudFiles(cloudInfo, syncContext)
fetchedFiles, err = repo.GetSyncCloudFiles(syncContext)
}
}
@ -601,17 +577,12 @@ func syncRepo(exit, byHand bool) (err error) {
return
}
cloudInfo, err := buildCloudInfo()
if nil != err {
return
}
syncContext := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
mergeResult, trafficStat, err := repo.Sync(cloudInfo, syncContext)
mergeResult, trafficStat, err := repo.Sync(syncContext)
if errors.Is(err, dejavu.ErrRepoFatalErr) {
// 重置仓库并再次尝试同步
if _, resetErr := resetRepository(repo); nil == resetErr {
mergeResult, trafficStat, err = repo.Sync(cloudInfo, syncContext)
mergeResult, trafficStat, err = repo.Sync(syncContext)
}
}
elapsed := time.Since(start)
@ -838,9 +809,17 @@ func resetRepository(repo *dejavu.Repo) (index *entity.Index, err error) {
}
func newRepository() (ret *dejavu.Repo, err error) {
transportConf, err := buildRepoTransportConf()
if nil != err {
return
}
// TODO: 数据同步支持接入第三方对象存储服务 https://github.com/siyuan-note/siyuan/issues/6426
siyuanTransport := &transport.SiYuan{Conf: transportConf}
ignoreLines := getIgnoreLines()
ignoreLines = append(ignoreLines, "/.siyuan/conf.json") // 忽略旧版同步配置
ret, err = dejavu.NewRepo(util.DataDir, util.RepoDir, util.HistoryDir, util.TempDir, Conf.Repo.Key, ignoreLines)
ret, err = dejavu.NewRepo(util.DataDir, util.RepoDir, util.HistoryDir, util.TempDir, Conf.Repo.Key, ignoreLines, siyuanTransport)
if nil != err {
logging.LogErrorf("init data repo failed: %s", err)
}
@ -1019,25 +998,25 @@ func subscribeEvents() {
})
}
func buildCloudInfo() (ret *dejavu.CloudInfo, err error) {
func buildRepoTransportConf() (ret *transport.Conf, err error) {
if !IsValidCloudDirName(Conf.Sync.CloudName) {
logging.LogWarnf("invalid cloud repo name, rename it to [main]")
Conf.Sync.CloudName = "main"
Conf.Save()
}
if nil == Conf.User {
err = errors.New("user auth failed")
return
userId, token := "0", ""
if nil != Conf.User {
userId = Conf.User.UserId
token = Conf.User.UserToken
}
ret = &dejavu.CloudInfo{
ret = &transport.Conf{
Dir: Conf.Sync.CloudName,
UserID: Conf.User.UserId,
Token: Conf.User.UserToken,
UserID: userId,
Token: token,
LimitSize: int64(Conf.User.UserSiYuanRepoSize - Conf.User.UserSiYuanAssetSize),
Server: util.AliyunServer,
Zone: &storage.ZoneHuadong, // TODO: 海外版需要条件编译
}
return
}

View file

@ -301,13 +301,12 @@ func CreateCloudSyncDir(name string) (err error) {
return errors.New(Conf.Language(37))
}
var cloudInfo *dejavu.CloudInfo
cloudInfo, err = buildCloudInfo()
repo, err := newRepository()
if nil != err {
return
}
err = dejavu.CreateCloudRepo(name, cloudInfo)
err = repo.CreateCloudRepo(name)
return
}
@ -321,13 +320,12 @@ func RemoveCloudSyncDir(name string) (err error) {
return
}
var cloudInfo *dejavu.CloudInfo
cloudInfo, err = buildCloudInfo()
repo, err := newRepository()
if nil != err {
return
}
err = dejavu.RemoveCloudRepo(name, cloudInfo)
err = repo.RemoveCloudRepo(name)
if nil != err {
err = errors.New(formatErrorMsg(err))
return
@ -348,13 +346,12 @@ func ListCloudSyncDir() (syncDirs []*Sync, hSize string, err error) {
var dirs []map[string]interface{}
var size int64
var cloudInfo *dejavu.CloudInfo
cloudInfo, err = buildCloudInfo()
repo, err := newRepository()
if nil != err {
return
}
dirs, size, err = dejavu.GetCloudRepos(cloudInfo)
dirs, size, err = repo.GetCloudRepos()
if nil != err {
err = errors.New(formatErrorMsg(err))
return