Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2025-12-29 21:08:49 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

🎨 WebDAV/S3 data sync and backup support configurable concurrent requests https://github.com/siyuan-note/siyuan/issues/12798

Browse source
This commit is contained in:
Daniel 2024-10-16 19:29:11 +08:00
parent 27618dd849
commit 488e87b70b
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
9 changed files with 75 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

28
kernel/conf/sync.go
View file

@ -41,22 +41,24 @@ func NewSync() *Sync {
}
type S3 struct {
Endpoint string `json:"endpoint"` // 服务端点
AccessKey string `json:"accessKey"` // Access Key
SecretKey string `json:"secretKey"` // Secret Key
Bucket string `json:"bucket"` // 存储空间
Region string `json:"region"` // 存储区域
PathStyle bool `json:"pathStyle"` // 是否使用路径风格
SkipTlsVerify bool `json:"skipTlsVerify"` // 是否跳过 TLS 验证
Timeout int `json:"timeout"` // 超时时间，单位：秒
Endpoint string `json:"endpoint"` // 服务端点
AccessKey string `json:"accessKey"` // Access Key
SecretKey string `json:"secretKey"` // Secret Key
Bucket string `json:"bucket"` // 存储空间
Region string `json:"region"` // 存储区域
PathStyle bool `json:"pathStyle"` // 是否使用路径风格
SkipTlsVerify bool `json:"skipTlsVerify"` // 是否跳过 TLS 验证
Timeout int `json:"timeout"` // 超时时间，单位：秒
ConcurrentReqs int `json:"concurrentReqs"` // 并发请求数
}
type WebDAV struct {
Endpoint string `json:"endpoint"` // 服务端点
Username string `json:"username"` // 用户名
Password string `json:"password"` // 密码
SkipTlsVerify bool `json:"skipTlsVerify"` // 是否跳过 TLS 验证
Timeout int `json:"timeout"` // 超时时间，单位：秒
Endpoint string `json:"endpoint"` // 服务端点
Username string `json:"username"` // 用户名
Password string `json:"password"` // 密码
SkipTlsVerify bool `json:"skipTlsVerify"` // 是否跳过 TLS 验证
Timeout int `json:"timeout"` // 超时时间，单位：秒
ConcurrentReqs int `json:"concurrentReqs"` // 并发请求数
}
const (

2
kernel/go.mod
View file

@ -54,7 +54,7 @@ require (
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/sashabaranov/go-openai v1.29.1
github.com/shirou/gopsutil/v3 v3.24.5
github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0
github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04

4
kernel/go.sum
View file

@ -332,8 +332,8 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA=
github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8=
github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0 h1:r2XSfIkA6rF0Cj4RqFlJzw9Am7x5xOlA6WcClV7vgm8=
github.com/siyuan-note/dejavu v0.0.0-20241014034432-7d61fd4d1fe0/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87 h1:EG8h35jYH8/iOukQlhB2OE7S35PwOBBn+XMOK1rtgtg=
github.com/siyuan-note/dejavu v0.0.0-20241016112457-5fc03573be87/go.mod h1:7E+jOWm75Me1ss7Sc5UM6O5xMiATAyCbN3g7MP6Cgxw=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc=
github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=

6
kernel/model/conf.go
View file

@ -339,15 +339,17 @@ func InitConf() {
Conf.Sync.Mode = 1
}
if nil == Conf.Sync.S3 {
Conf.Sync.S3 = &conf.S3{}
Conf.Sync.S3 = &conf.S3{SkipTlsVerify: true}
}
Conf.Sync.S3.Endpoint = util.NormalizeEndpoint(Conf.Sync.S3.Endpoint)
Conf.Sync.S3.Timeout = util.NormalizeTimeout(Conf.Sync.S3.Timeout)
Conf.Sync.S3.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.S3.ConcurrentReqs, conf.ProviderS3)
if nil == Conf.Sync.WebDAV {
Conf.Sync.WebDAV = &conf.WebDAV{}
Conf.Sync.WebDAV = &conf.WebDAV{SkipTlsVerify: true}
}
Conf.Sync.WebDAV.Endpoint = util.NormalizeEndpoint(Conf.Sync.WebDAV.Endpoint)
Conf.Sync.WebDAV.Timeout = util.NormalizeTimeout(Conf.Sync.WebDAV.Timeout)
Conf.Sync.WebDAV.ConcurrentReqs = util.NormalizeConcurrentReqs(Conf.Sync.WebDAV.ConcurrentReqs, conf.ProviderWebDAV)
if util.ContainerDocker == util.Container {
Conf.Sync.Perception = false
}

2
kernel/model/repository.go
View file

@ -1959,6 +1959,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
PathStyle: Conf.Sync.S3.PathStyle,
SkipTlsVerify: Conf.Sync.S3.SkipTlsVerify,
Timeout: Conf.Sync.S3.Timeout,
ConcurrentReqs: Conf.Sync.S3.ConcurrentReqs,
}
case conf.ProviderWebDAV:
ret.WebDAV = &cloud.ConfWebDAV{
@ -1967,6 +1968,7 @@ func buildCloudConf() (ret *cloud.Conf, err error) {
Password: Conf.Sync.WebDAV.Password,
SkipTlsVerify: Conf.Sync.WebDAV.SkipTlsVerify,
Timeout: Conf.Sync.WebDAV.Timeout,
ConcurrentReqs: Conf.Sync.WebDAV.ConcurrentReqs,
}
default:
err = fmt.Errorf("invalid provider [%d]", Conf.Sync.Provider)

2
kernel/model/sync.go
View file

@ -426,6 +426,7 @@ func SetSyncProviderS3(s3 *conf.S3) (err error) {
s3.Bucket = strings.TrimSpace(s3.Bucket)
s3.Region = strings.TrimSpace(s3.Region)
s3.Timeout = util.NormalizeTimeout(s3.Timeout)
s3.ConcurrentReqs = util.NormalizeConcurrentReqs(s3.ConcurrentReqs, conf.ProviderS3)
if !cloud.IsValidCloudDirName(s3.Bucket) {
util.PushErrMsg(Conf.Language(37), 5000)
@ -450,6 +451,7 @@ func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) {
webdav.Username = strings.TrimSpace(webdav.Username)
webdav.Password = strings.TrimSpace(webdav.Password)
webdav.Timeout = util.NormalizeTimeout(webdav.Timeout)
webdav.ConcurrentReqs = util.NormalizeConcurrentReqs(webdav.ConcurrentReqs, conf.ProviderWebDAV)
Conf.Sync.WebDAV = webdav
Conf.Save()

15
kernel/util/path.go
View file

@ -179,6 +179,21 @@ func GetChildDocDepth(treeAbsPath string) (ret int) {
return
}
func NormalizeConcurrentReqs(concurrentReqs int, provider int) int {
if 1 > concurrentReqs {
if 2 == provider { // S3
return 8
} else if 3 == provider { // WebDAV
return 1
}
return 8
}
if 16 < concurrentReqs {
return 16
}
return concurrentReqs
}
func NormalizeTimeout(timeout int) int {
if 7 > timeout {
if 1 > timeout {