From 779e4fc4ae48e6f0c5f8e4dba560120c7038b90f Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 23 Jun 2022 01:22:28 +0800 Subject: [PATCH 1/2] =?UTF-8?q?:recycle:=20=E5=86=85=E6=A0=B8=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=20HTTP=20=E5=AE=A2=E6=88=B7=E7=AB=AF=E6=8B=86?= =?UTF-8?q?=E5=88=86=E9=A1=B9=E7=9B=AE=20https://github.com/siyuan-note/si?= =?UTF-8?q?yuan/issues/5269?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/bazaar/icon.go | 5 +- kernel/bazaar/package.go | 9 +-- kernel/bazaar/template.go | 5 +- kernel/bazaar/theme.go | 5 +- kernel/bazaar/widget.go | 5 +- kernel/go.mod | 3 +- kernel/go.sum | 6 +- kernel/model/assets.go | 5 +- kernel/model/liandi.go | 23 +++---- kernel/model/osssync.go | 27 +++++---- kernel/util/rhy.go | 110 +--------------------------------- kernel/util/working.go | 4 +- kernel/util/working_mobile.go | 2 + 13 files changed, 59 insertions(+), 150 deletions(-) diff --git a/kernel/bazaar/icon.go b/kernel/bazaar/icon.go index 2a429bc6d..44c373cd2 100644 --- a/kernel/bazaar/icon.go +++ b/kernel/bazaar/icon.go @@ -25,6 +25,7 @@ import ( "github.com/dustin/go-humanize" ants "github.com/panjf2000/ants/v2" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -63,7 +64,7 @@ func Icons(proxyURL string) (icons []*Icon) { bazaarIndex := getBazaarIndex(proxyURL) bazaarHash := result["bazaar"].(string) result = map[string]interface{}{} - request := util.NewBrowserRequest(proxyURL) + request := httpclient.NewBrowserRequest(proxyURL) u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/icons.json" resp, err := request.SetResult(&result).Get(u) if nil != err { @@ -85,7 +86,7 @@ func Icons(proxyURL string) (icons []*Icon) { icon := &Icon{} innerU := util.BazaarOSSServer + "/package/" + repoURL + "/icon.json" - innerResp, innerErr := util.NewBrowserRequest(proxyURL).SetResult(icon).Get(innerU) + innerResp, innerErr := httpclient.NewBrowserRequest(proxyURL).SetResult(icon).Get(innerU) if nil != innerErr { util.LogErrorf("get bazaar package [%s] failed: %s", repoURL, innerErr) return diff --git a/kernel/bazaar/package.go b/kernel/bazaar/package.go index 225ab90c9..0677177f0 100644 --- a/kernel/bazaar/package.go +++ b/kernel/bazaar/package.go @@ -30,6 +30,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/araddon/dateparse" "github.com/imroc/req/v3" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/util" textUnicode "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" @@ -82,14 +83,14 @@ func downloadPackage(repoURLHash, proxyURL string, chinaCDN, pushProgress bool, u = util.BazaarOSSServer + "/package/" + repoURLHash } buf := &bytes.Buffer{} - resp, err := util.NewBrowserDownloadRequest(proxyURL).SetOutput(buf).SetDownloadCallback(func(info req.DownloadInfo) { + resp, err := httpclient.NewBrowserDownloadRequest(proxyURL).SetOutput(buf).SetDownloadCallback(func(info req.DownloadInfo) { if pushProgress { util.PushDownloadProgress(pushID, float32(info.DownloadedSize)/float32(info.Response.ContentLength)) } }).Get(u) if nil != err { u = util.BazaarOSSServer + "/package/" + repoURLHash - resp, err = util.NewBrowserDownloadRequest(proxyURL).SetOutput(buf).SetDownloadCallback(func(info req.DownloadInfo) { + resp, err = httpclient.NewBrowserDownloadRequest(proxyURL).SetOutput(buf).SetDownloadCallback(func(info req.DownloadInfo) { if pushProgress { util.PushDownloadProgress(pushID, float32(info.DownloadedSize)/float32(info.Response.ContentLength)) } @@ -116,7 +117,7 @@ func incPackageDownloads(repoURLHash, proxyURL, systemID string) { repo := strings.Split(repoURLHash, "@")[0] u := util.AliyunServer + "/apis/siyuan/bazaar/addBazaarPackageDownloadCount" - util.NewCloudRequest(proxyURL).SetBody( + httpclient.NewCloudRequest(proxyURL).SetBody( map[string]interface{}{ "systemID": systemID, "repo": repo, @@ -190,7 +191,7 @@ func getBazaarIndex(proxyURL string) map[string]*bazaarPackage { return cachedBazaarIndex } - request := util.NewBrowserRequest(proxyURL) + request := httpclient.NewBrowserRequest(proxyURL) u := util.BazaarStatServer + "/bazaar/index.json" resp, reqErr := request.SetResult(&cachedBazaarIndex).Get(u) if nil != reqErr { diff --git a/kernel/bazaar/template.go b/kernel/bazaar/template.go index 5d07f90e8..beba31f0a 100644 --- a/kernel/bazaar/template.go +++ b/kernel/bazaar/template.go @@ -26,6 +26,7 @@ import ( "github.com/dustin/go-humanize" "github.com/panjf2000/ants/v2" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -63,7 +64,7 @@ func Templates(proxyURL string) (templates []*Template) { bazaarIndex := getBazaarIndex(proxyURL) bazaarHash := result["bazaar"].(string) result = map[string]interface{}{} - request := util.NewBrowserRequest(proxyURL) + request := httpclient.NewBrowserRequest(proxyURL) u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/templates.json" resp, reqErr := request.SetResult(&result).Get(u) if nil != reqErr { @@ -86,7 +87,7 @@ func Templates(proxyURL string) (templates []*Template) { template := &Template{} innerU := util.BazaarOSSServer + "/package/" + repoURL + "/template.json" - innerResp, innerErr := util.NewBrowserRequest(proxyURL).SetResult(template).Get(innerU) + innerResp, innerErr := httpclient.NewBrowserRequest(proxyURL).SetResult(template).Get(innerU) if nil != innerErr { util.LogErrorf("get community template [%s] failed: %s", repoURL, innerErr) return diff --git a/kernel/bazaar/theme.go b/kernel/bazaar/theme.go index 6aca817ef..58b1cf919 100644 --- a/kernel/bazaar/theme.go +++ b/kernel/bazaar/theme.go @@ -25,6 +25,7 @@ import ( "github.com/dustin/go-humanize" ants "github.com/panjf2000/ants/v2" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -64,7 +65,7 @@ func Themes(proxyURL string) (ret []*Theme) { bazaarIndex := getBazaarIndex(proxyURL) bazaarHash := result["bazaar"].(string) result = map[string]interface{}{} - request := util.NewBrowserRequest(proxyURL) + request := httpclient.NewBrowserRequest(proxyURL) u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/themes.json" resp, reqErr := request.SetResult(&result).Get(u) if nil != reqErr { @@ -87,7 +88,7 @@ func Themes(proxyURL string) (ret []*Theme) { theme := &Theme{} innerU := util.BazaarOSSServer + "/package/" + repoURL + "/theme.json" - innerResp, innerErr := util.NewBrowserRequest(proxyURL).SetResult(theme).Get(innerU) + innerResp, innerErr := httpclient.NewBrowserRequest(proxyURL).SetResult(theme).Get(innerU) if nil != innerErr { util.LogErrorf("get bazaar package [%s] failed: %s", innerU, innerErr) return diff --git a/kernel/bazaar/widget.go b/kernel/bazaar/widget.go index fc76e777a..70f3ba4f7 100644 --- a/kernel/bazaar/widget.go +++ b/kernel/bazaar/widget.go @@ -25,6 +25,7 @@ import ( "github.com/dustin/go-humanize" ants "github.com/panjf2000/ants/v2" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -63,7 +64,7 @@ func Widgets(proxyURL string) (widgets []*Widget) { bazaarIndex := getBazaarIndex(proxyURL) bazaarHash := result["bazaar"].(string) result = map[string]interface{}{} - request := util.NewBrowserRequest(proxyURL) + request := httpclient.NewBrowserRequest(proxyURL) u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/widgets.json" resp, err := request.SetResult(&result).Get(u) if nil != err { @@ -86,7 +87,7 @@ func Widgets(proxyURL string) (widgets []*Widget) { widget := &Widget{} innerU := util.BazaarOSSServer + "/package/" + repoURL + "/widget.json" - innerResp, innerErr := util.NewBrowserRequest(proxyURL).SetResult(widget).Get(innerU) + innerResp, innerErr := httpclient.NewBrowserRequest(proxyURL).SetResult(widget).Get(innerU) if nil != innerErr { util.LogErrorf("get bazaar package [%s] failed: %s", repoURL, innerErr) return diff --git a/kernel/go.mod b/kernel/go.mod index 52918d32d..f560572e5 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -43,6 +43,7 @@ require ( github.com/siyuan-note/dejavu v0.0.0-20220620134708-bcbbf7f6478e github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 github.com/siyuan-note/filelock v0.0.0-20220616063212-74cfba0754ee + github.com/siyuan-note/httpclient v0.0.0-20220622170705-004622a8138d github.com/vmihailenco/msgpack/v5 v5.3.5 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 golang.org/x/image v0.0.0-20220601225756-64ec528b34cd @@ -98,7 +99,7 @@ require ( go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect golang.org/x/mod v0.5.1 // indirect - golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect + golang.org/x/net v0.0.0-20220621193019-9d032be2e588 // indirect golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect golang.org/x/tools v0.1.8 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index 2427a03c7..c00e284c9 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -425,6 +425,8 @@ github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 h1:QB9TjJQF github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/filelock v0.0.0-20220616063212-74cfba0754ee h1:8nBBJKHbXuMF2dWKUKsrnnFblgeNBfPjLgVm0aDas/s= github.com/siyuan-note/filelock v0.0.0-20220616063212-74cfba0754ee/go.mod h1:c4vwvWRrnfa75OXiO21K/76BFRJ4cFITKNoVs5lFdwc= +github.com/siyuan-note/httpclient v0.0.0-20220622170705-004622a8138d h1:zMkahXrTDPhBUg9W7HkGlYP6WY5AkQoJqj7Cmk++qoU= +github.com/siyuan-note/httpclient v0.0.0-20220622170705-004622a8138d/go.mod h1:W/5hysNTpI8a52hSMzNHvf29tbM+VmD/l+c2wvkqlL8= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= @@ -577,8 +579,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220621193019-9d032be2e588 h1:9ubFuySsnAJYGyJrZ3koiEv8FyqofCBdz3G9Mbf2YFc= +golang.org/x/net v0.0.0-20220621193019-9d032be2e588/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/kernel/model/assets.go b/kernel/model/assets.go index 138e92e8f..ddc20bc7e 100644 --- a/kernel/model/assets.go +++ b/kernel/model/assets.go @@ -36,6 +36,7 @@ import ( "github.com/88250/lute/parse" "github.com/gabriel-vasile/mimetype" "github.com/siyuan-note/filelock" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" @@ -91,7 +92,7 @@ func NetImg2LocalAssets(rootID string) (err error) { } } util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), u), 15000) - request := util.NewBrowserRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewBrowserRequest(Conf.System.NetworkProxy.String()) resp, reqErr := request.Get(u) if nil != reqErr { util.LogErrorf("download net img [%s] failed: %s", u, reqErr) @@ -285,7 +286,7 @@ func uploadCloud(sqlAssets []*sql.Asset) (err error) { } requestResult := gulu.Ret.NewResult() - request := util.NewCloudFileRequest2m(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudFileRequest2m(Conf.System.NetworkProxy.String()) resp, reqErr := request. SetResult(requestResult). SetFile("file[]", absAsset). diff --git a/kernel/model/liandi.go b/kernel/model/liandi.go index ab8318999..a215cbce3 100644 --- a/kernel/model/liandi.go +++ b/kernel/model/liandi.go @@ -25,6 +25,7 @@ import ( "time" "github.com/88250/gulu" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/conf" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -37,7 +38,7 @@ func StartFreeTrial() (err error) { } requestResult := gulu.Ret.NewResult() - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) _, err = request. SetResult(requestResult). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). @@ -54,7 +55,7 @@ func StartFreeTrial() (err error) { func DeactivateUser() (err error) { requestResult := gulu.Ret.NewResult() - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(requestResult). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). @@ -79,7 +80,7 @@ func DeactivateUser() (err error) { func SetCloudBlockReminder(id, data string, timed int64) (err error) { requestResult := gulu.Ret.NewResult() payload := map[string]interface{}{"dataId": id, "data": data, "timed": timed} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(requestResult). SetBody(payload). @@ -112,7 +113,7 @@ func LoadUploadToken() (err error) { } requestResult := gulu.Ret.NewResult() - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(requestResult). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). @@ -251,7 +252,7 @@ func loadUserFromConf() *conf.User { func RemoveCloudShorthands(ids []string) (err error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) body := map[string]interface{}{ "ids": ids, } @@ -282,7 +283,7 @@ func RemoveCloudShorthands(ids []string) (err error) { func GetCloudShorthands(page int) (result map[string]interface{}, err error) { result = map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(&result). SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}). @@ -319,7 +320,7 @@ var errInvalidUser = errors.New("invalid user") func getUser(token string) (*conf.User, error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) _, err := request. SetResult(&result). SetBody(map[string]string{"token": token}). @@ -350,7 +351,7 @@ func getUser(token string) (*conf.User, error) { func UseActivationcode(code string) (err error) { requestResult := gulu.Ret.NewResult() - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) _, err = request. SetResult(requestResult). SetBody(map[string]string{"data": code}). @@ -369,7 +370,7 @@ func UseActivationcode(code string) (err error) { func CheckActivationcode(code string) (retCode int, msg string) { retCode = 1 requestResult := gulu.Ret.NewResult() - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) _, err := request. SetResult(requestResult). SetBody(map[string]string{"data": code}). @@ -389,7 +390,7 @@ func CheckActivationcode(code string) (retCode int, msg string) { func Login(userName, password, captcha string) (ret *gulu.Result, err error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) _, err = request. SetResult(&result). SetBody(map[string]string{"userName": userName, "userPassword": password, "captcha": captcha}). @@ -415,7 +416,7 @@ func Login(userName, password, captcha string) (ret *gulu.Result, err error) { func Login2fa(token, code string) (map[string]interface{}, error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) _, err := request. SetResult(&result). SetBody(map[string]string{"twofactorAuthCode": code}). diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index 742b826e8..c1c254463 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -34,12 +34,13 @@ import ( "github.com/88250/gulu" "github.com/panjf2000/ants/v2" "github.com/qiniu/go-sdk/v7/storage" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/util" ) func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, err error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(&result). SetBody(map[string]string{"token": Conf.User.UserToken}). @@ -69,7 +70,7 @@ func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, e func removeCloudDirPath(dirPath string) (err error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(&result). SetBody(map[string]string{"dirPath": dirPath, "token": Conf.User.UserToken}). @@ -94,7 +95,7 @@ func removeCloudDirPath(dirPath string) (err error) { func createCloudSyncDirOSS(name string) (err error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(&result). SetBody(map[string]string{"name": name, "token": Conf.User.UserToken}). @@ -125,7 +126,7 @@ func createCloudSyncDirOSS(name string) (err error) { func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetBody(map[string]interface{}{"token": Conf.User.UserToken}). SetResult(&result). @@ -277,7 +278,7 @@ func ossDownload0(localDirPath, cloudDirPath, fetch string, fetchedFiles *int, t localFilePath := filepath.Join(localDirPath, fetch) remoteFileURL := path.Join(cloudDirPath, fetch) var result map[string]interface{} - resp, err := util.NewCloudRequest(Conf.System.NetworkProxy.String()). + resp, err := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()). SetResult(&result). SetBody(map[string]interface{}{"token": Conf.User.UserToken, "path": remoteFileURL}). Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanFile?uid=" + Conf.User.UserId) @@ -312,9 +313,9 @@ func ossDownload0(localDirPath, cloudDirPath, fetch string, fetchedFiles *int, t os.Remove(localFilePath) if bootORExit { - resp, err = util.NewCloudFileRequest15s(Conf.System.NetworkProxy.String()).Get(downloadURL) + resp, err = httpclient.NewCloudFileRequest15s(Conf.System.NetworkProxy.String()).Get(downloadURL) } else { - resp, err = util.NewCloudFileRequest2m(Conf.System.NetworkProxy.String()).Get(downloadURL) + resp, err = httpclient.NewCloudFileRequest2m(Conf.System.NetworkProxy.String()).Get(downloadURL) } if nil != err { util.LogErrorf("download request [%s] failed: %s", downloadURL, err) @@ -457,7 +458,7 @@ func ossRemove0(cloudDirPath string, removes []string) (err error) { return } - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetBody(map[string]interface{}{"token": Conf.User.UserToken, "dirPath": cloudDirPath, "paths": removes}). Post(util.AliyunServer + "/apis/siyuan/data/removeSiYuanFile?uid=" + Conf.User.UserId) @@ -511,7 +512,7 @@ func getOssUploadToken(filename, cloudDirPath string, length int64) (ret string, // 因为需要指定 key，所以每次上传文件都必须在云端生成 Token，否则有安全隐患 var result map[string]interface{} - req := util.NewCloudRequest(Conf.System.NetworkProxy.String()). + req := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()). SetResult(&result) req.SetBody(map[string]interface{}{ "token": Conf.User.UserToken, @@ -551,7 +552,7 @@ func getOssUploadToken(filename, cloudDirPath string, length int64) (ret string, func getCloudSyncVer(cloudDir string) (cloudSyncVer int64, err error) { start := time.Now() result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(&result). SetBody(map[string]string{"syncDir": cloudDir, "token": Conf.User.UserToken}). @@ -591,7 +592,7 @@ func getCloudSyncVer(cloudDir string) (cloudSyncVer int64, err error) { func getCloudSync(cloudDir string) (assetSize, backupSize int64, device string, err error) { start := time.Now() result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(&result). SetBody(map[string]string{"syncDir": cloudDir, "token": Conf.User.UserToken}). @@ -656,7 +657,7 @@ func getLocalFileListOSS(isBackup bool) (ret map[string]*CloudIndex, err error) func getCloudFileListOSS(cloudDirPath string) (ret map[string]*CloudIndex, err error) { result := map[string]interface{}{} - request := util.NewCloudRequest(Conf.System.NetworkProxy.String()) + request := httpclient.NewCloudRequest(Conf.System.NetworkProxy.String()) resp, err := request. SetResult(&result). SetBody(map[string]string{"dirPath": cloudDirPath, "token": Conf.User.UserToken}). @@ -681,7 +682,7 @@ func getCloudFileListOSS(cloudDirPath string) (ret map[string]*CloudIndex, err e retData := result["data"].(map[string]interface{}) downloadURL := retData["url"].(string) - resp, err = util.NewCloudFileRequest15s(Conf.System.NetworkProxy.String()).Get(downloadURL) + resp, err = httpclient.NewCloudFileRequest15s(Conf.System.NetworkProxy.String()).Get(downloadURL) if nil != err { util.LogErrorf("download request [%s] failed: %s", downloadURL, err) return diff --git a/kernel/util/rhy.go b/kernel/util/rhy.go index 1c29aad40..0bfaa0ed8 100644 --- a/kernel/util/rhy.go +++ b/kernel/util/rhy.go @@ -17,11 +17,10 @@ package util import ( - "net/http" "sync" "time" - "github.com/imroc/req/v3" + "github.com/siyuan-note/httpclient" ) var cachedRhyResult = map[string]interface{}{} @@ -37,7 +36,7 @@ func GetRhyResult(force bool, proxyURL string) (map[string]interface{}, error) { return cachedRhyResult, nil } - request := NewCloudRequest(proxyURL) + request := httpclient.NewCloudRequest(proxyURL) _, err := request.SetResult(&cachedRhyResult).Get(AliyunServer + "/apis/siyuan/version?ver=" + Ver) if nil != err { LogErrorf("get version meta info failed: %s", err) @@ -46,108 +45,3 @@ func GetRhyResult(force bool, proxyURL string) (map[string]interface{}, error) { rhyResultCacheTime = now return cachedRhyResult, nil } - -var ( - browserClient, browserDownloadClient, cloudAPIClient, cloudFileClientTimeout2Min, cloudFileClientTimeout15s *req.Client - - browserUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" -) - -func NewBrowserRequest(proxyURL string) (ret *req.Request) { - if nil == browserClient { - browserClient = req.C(). - SetUserAgent(browserUserAgent). - SetTimeout(7 * time.Second). - DisableInsecureSkipVerify() - } - if "" != proxyURL { - browserClient.SetProxyURL(proxyURL) - } - ret = browserClient.R() - ret.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second) - return -} - -func NewBrowserDownloadRequest(proxyURL string) *req.Request { - if nil == browserDownloadClient { - browserDownloadClient = req.C(). - SetUserAgent(browserUserAgent). - SetTimeout(2 * time.Minute). - SetCommonRetryCount(1). - SetCommonRetryFixedInterval(3 * time.Second). - SetCommonRetryCondition(retryCondition). - DisableInsecureSkipVerify() - } - if "" != proxyURL { - browserDownloadClient.SetProxyURL(proxyURL) - } - return browserDownloadClient.R() -} - -func NewCloudRequest(proxyURL string) *req.Request { - if nil == cloudAPIClient { - cloudAPIClient = req.C(). - SetUserAgent(UserAgent). - SetTimeout(7 * time.Second). - SetCommonRetryCount(1). - SetCommonRetryFixedInterval(3 * time.Second). - SetCommonRetryCondition(retryCondition). - DisableInsecureSkipVerify() - } - if "" != proxyURL { - cloudAPIClient.SetProxyURL(proxyURL) - } - return cloudAPIClient.R() -} - -func NewCloudFileRequest2m(proxyURL string) *req.Request { - if nil == cloudFileClientTimeout2Min { - cloudFileClientTimeout2Min = req.C(). - SetUserAgent(UserAgent). - SetTimeout(2 * time.Minute). - SetCommonRetryCount(1). - SetCommonRetryFixedInterval(3 * time.Second). - SetCommonRetryCondition(retryCondition). - DisableInsecureSkipVerify() - setTransport(cloudFileClientTimeout2Min.GetClient()) - } - if "" != proxyURL { - cloudFileClientTimeout2Min.SetProxyURL(proxyURL) - } - return cloudFileClientTimeout2Min.R() -} - -func NewCloudFileRequest15s(proxyURL string) *req.Request { - if nil == cloudFileClientTimeout15s { - cloudFileClientTimeout15s = req.C(). - SetUserAgent(UserAgent). - SetTimeout(15 * time.Second). - SetCommonRetryCount(1). - SetCommonRetryFixedInterval(3 * time.Second). - SetCommonRetryCondition(retryCondition). - DisableInsecureSkipVerify() - setTransport(cloudFileClientTimeout15s.GetClient()) - } - if "" != proxyURL { - cloudFileClientTimeout15s.SetProxyURL(proxyURL) - } - return cloudFileClientTimeout15s.R() -} - -func retryCondition(resp *req.Response, err error) bool { - if nil != err { - return true - } - if 503 == resp.StatusCode { // 负载均衡会返回 503，需要重试 - return true - } - return false -} - -func setTransport(client *http.Client) { - // 改进同步下载数据稳定性 https://github.com/siyuan-note/siyuan/issues/4994 - transport := client.Transport.(*req.Transport) - transport.MaxIdleConns = 10 - transport.MaxIdleConnsPerHost = 2 - transport.MaxConnsPerHost = 2 -} diff --git a/kernel/util/working.go b/kernel/util/working.go index 065b274d1..dbbfb4bdf 100644 --- a/kernel/util/working.go +++ b/kernel/util/working.go @@ -32,6 +32,7 @@ import ( "github.com/88250/gulu" figure "github.com/common-nighthawk/go-figure" goPS "github.com/mitchellh/go-ps" + "github.com/siyuan-note/httpclient" ) //var Mode = "dev" @@ -53,6 +54,7 @@ func Boot() { IncBootProgress(3, "Booting...") rand.Seed(time.Now().UTC().UnixNano()) initMime() + httpclient.SetUserAgent(UserAgent) workspacePath := flag.String("workspace", "", "dir path of the workspace, default to ~/Documents/SiYuan/") wdPath := flag.String("wd", WorkingDir, "working directory of SiYuan") @@ -323,7 +325,7 @@ func checkPort() { LogInfof("port [%s] is opened, try to check version of running kernel", ServerPort) result := NewResult() - _, err := NewBrowserRequest(""). + _, err := httpclient.NewBrowserRequest(""). SetResult(result). SetHeader("User-Agent", UserAgent). Get("http://127.0.0.1:" + ServerPort + "/api/system/version") diff --git a/kernel/util/working_mobile.go b/kernel/util/working_mobile.go index 46d007eb7..dfdc64944 100644 --- a/kernel/util/working_mobile.go +++ b/kernel/util/working_mobile.go @@ -23,12 +23,14 @@ import ( "time" figure "github.com/common-nighthawk/go-figure" + "github.com/siyuan-note/httpclient" ) func BootMobile(container, appDir, workspaceDir, nativeLibDir, privateDataDir, lang string) { IncBootProgress(3, "Booting...") rand.Seed(time.Now().UTC().UnixNano()) initMime() + httpclient.SetUserAgent(UserAgent) HomeDir = filepath.Join(workspaceDir, "home") WorkingDir = filepath.Join(appDir, "app") From ffd8a3b685cd4a2dc93ff25cd87b0425a163c9ce Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 23 Jun 2022 01:28:53 +0800 Subject: [PATCH 2/2] =?UTF-8?q?:recycle:=20=E5=86=85=E6=A0=B8=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=20HTTP=20=E5=AE=A2=E6=88=B7=E7=AB=AF=E6=8B=86?= =?UTF-8?q?=E5=88=86=E9=A1=B9=E7=9B=AE=20Fix=20https://github.com/siyuan-n?= =?UTF-8?q?ote/siyuan/issues/5269?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 + README_zh_CN.md | 2 + .../20201121224345-rc27qvo.sy | 143 +++++++++++++++++- .../20201121212605-9td1a62.sy | 143 +++++++++++++++++- .../20211226114929-08ap1r0.sy | 143 +++++++++++++++++- 5 files changed, 427 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7c043a0e7..4287712a1 100644 --- a/README.md +++ b/README.md @@ -290,6 +290,8 @@ SiYuan is made possible by the following open source projects. * [https://github.com/radovskyb/watcher](https://github.com/radovskyb/watcher) `BSD-3-Clause License` * [https://github.com/siyuan-note/dejavu](https://github.com/siyuan-note/dejavu) `Mulan PSL v2` * [https://github.com/siyuan-note/encryption](https://github.com/siyuan-note/encryption) `Mulan PSL v2` +* [https://github.com/siyuan-note/filelock](https://github.com/siyuan-note/filelock) `Mulan PSL v2` +* [https://github.com/siyuan-note/httpclient](https://github.com/siyuan-note/httpclient) `Mulan PSL v2` * [https://github.com/vmihailenco/msgpack](https://github.com/vmihailenco/msgpack) `BSD-2-Clause License` * [https://github.com/xrash/smetrics](https://github.com/xrash/smetrics) `MIT License` * [https://github.com/microsoft/TypeScript](https://github.com/microsoft/TypeScript) `Apache-2.0 License` diff --git a/README_zh_CN.md b/README_zh_CN.md index f46ae6c12..ab9388429 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -299,6 +299,8 @@ * [https://github.com/radovskyb/watcher](https://github.com/radovskyb/watcher) `BSD-3-Clause License` * [https://github.com/siyuan-note/dejavu](https://github.com/siyuan-note/dejavu) `Mulan PSL v2` * [https://github.com/siyuan-note/encryption](https://github.com/siyuan-note/encryption) `Mulan PSL v2` +* [https://github.com/siyuan-note/filelock](https://github.com/siyuan-note/filelock) `Mulan PSL v2` +* [https://github.com/siyuan-note/httpclient](https://github.com/siyuan-note/httpclient) `Mulan PSL v2` * [https://github.com/vmihailenco/msgpack](https://github.com/vmihailenco/msgpack) `BSD-2-Clause License` * [https://github.com/xrash/smetrics](https://github.com/xrash/smetrics) `MIT License` * [https://github.com/microsoft/TypeScript](https://github.com/microsoft/TypeScript) `Apache-2.0 License` diff --git a/app/guide/20210808180117-6v0mkxr/20201121224345-rc27qvo.sy b/app/guide/20210808180117-6v0mkxr/20201121224345-rc27qvo.sy index 7312dd64d..ef581f105 100644 --- a/app/guide/20210808180117-6v0mkxr/20201121224345-rc27qvo.sy +++ b/app/guide/20210808180117-6v0mkxr/20201121224345-rc27qvo.sy @@ -6,7 +6,7 @@ "id": "20201121224345-rc27qvo", "title": "Acknowledgements", "type": "doc", - "updated": "20220615154952" + "updated": "20220623012804" }, "Children": [ { @@ -29,7 +29,7 @@ "ListData": {}, "Properties": { "id": "20220522101213-z3hokh2", - "updated": "20220615154952" + "updated": "20220623012804" }, "Children": [ { @@ -2692,6 +2692,145 @@ } ] }, + { + "ID": "20220623012746-vt6mqv1", + "Type": "NodeListItem", + "ListData": { + "BulletChar": 42, + "Marker": "Kg==" + }, + "Properties": { + "id": "20220623012746-vt6mqv1" + }, + "Children": [ + { + "ID": "20220623012746-47iyf95", + "Type": "NodeParagraph", + "Properties": { + "id": "20220623012746-47iyf95", + "updated": "20220623012747" + }, + "Children": [ + { + "Type": "NodeLink", + "Data": "span", + "Children": [ + { + "Type": "NodeOpenBracket" + }, + { + "Type": "NodeLinkText", + "Data": "https://github.com/siyuan-note/filelock" + }, + { + "Type": "NodeCloseBracket" + }, + { + "Type": "NodeOpenParen" + }, + { + "Type": "NodeLinkDest", + "Data": "https://github.com/siyuan-note/filelock" + }, + { + "Type": "NodeCloseParen" + } + ] + }, + { + "Type": "NodeText", + "Data": " " + }, + { + "Type": "NodeCodeSpan", + "Data": "code", + "Children": [ + { + "Type": "NodeCodeSpanOpenMarker" + }, + { + "Type": "NodeCodeSpanContent", + "Data": "Mulan PSL v2" + }, + { + "Type": "NodeCodeSpanCloseMarker" + } + ] + } + ] + } + ] + }, + { + "ID": "20220623012804-rt72p38", + "Type": "NodeListItem", + "ListData": { + "BulletChar": 42, + "Marker": "Kg==" + }, + "Properties": { + "id": "20220623012804-rt72p38", + "updated": "20220623012804" + }, + "Children": [ + { + "ID": "20220623012804-s1gocv8", + "Type": "NodeParagraph", + "Properties": { + "id": "20220623012804-s1gocv8", + "updated": "20220623012804" + }, + "Children": [ + { + "Type": "NodeLink", + "Data": "span", + "Children": [ + { + "Type": "NodeOpenBracket" + }, + { + "Type": "NodeLinkText", + "Data": "https://github.com/siyuan-note/httpclient" + }, + { + "Type": "NodeCloseBracket" + }, + { + "Type": "NodeOpenParen" + }, + { + "Type": "NodeLinkDest", + "Data": "https://github.com/siyuan-note/httpclient" + }, + { + "Type": "NodeCloseParen" + } + ] + }, + { + "Type": "NodeText", + "Data": " " + }, + { + "Type": "NodeCodeSpan", + "Data": "code", + "Children": [ + { + "Type": "NodeCodeSpanOpenMarker" + }, + { + "Type": "NodeCodeSpanContent", + "Data": "Mulan PSL v2" + }, + { + "Type": "NodeCodeSpanCloseMarker" + } + ] + } + ] + } + ] + }, { "ID": "20220522101213-wxfscvi", "Type": "NodeListItem", diff --git a/app/guide/20210808180117-czj9bvb/20201121212605-9td1a62.sy b/app/guide/20210808180117-czj9bvb/20201121212605-9td1a62.sy index 207faf10d..94ca11e90 100644 --- a/app/guide/20210808180117-czj9bvb/20201121212605-9td1a62.sy +++ b/app/guide/20210808180117-czj9bvb/20201121212605-9td1a62.sy @@ -6,7 +6,7 @@ "id": "20201121212605-9td1a62", "title": "致谢", "type": "doc", - "updated": "20220615155003" + "updated": "20220623012801" }, "Children": [ { @@ -29,7 +29,7 @@ "ListData": {}, "Properties": { "id": "20220522101224-ctmcj3d", - "updated": "20220615155003" + "updated": "20220623012801" }, "Children": [ { @@ -2692,6 +2692,145 @@ } ] }, + { + "ID": "20220623012754-1t30exi", + "Type": "NodeListItem", + "ListData": { + "BulletChar": 42, + "Marker": "Kg==" + }, + "Properties": { + "id": "20220623012754-1t30exi" + }, + "Children": [ + { + "ID": "20220623012754-i5exfar", + "Type": "NodeParagraph", + "Properties": { + "id": "20220623012754-i5exfar", + "updated": "20220623012754" + }, + "Children": [ + { + "Type": "NodeLink", + "Data": "span", + "Children": [ + { + "Type": "NodeOpenBracket" + }, + { + "Type": "NodeLinkText", + "Data": "https://github.com/siyuan-note/filelock" + }, + { + "Type": "NodeCloseBracket" + }, + { + "Type": "NodeOpenParen" + }, + { + "Type": "NodeLinkDest", + "Data": "https://github.com/siyuan-note/filelock" + }, + { + "Type": "NodeCloseParen" + } + ] + }, + { + "Type": "NodeText", + "Data": " " + }, + { + "Type": "NodeCodeSpan", + "Data": "code", + "Children": [ + { + "Type": "NodeCodeSpanOpenMarker" + }, + { + "Type": "NodeCodeSpanContent", + "Data": "Mulan PSL v2" + }, + { + "Type": "NodeCodeSpanCloseMarker" + } + ] + } + ] + } + ] + }, + { + "ID": "20220623012800-bptg3o2", + "Type": "NodeListItem", + "ListData": { + "BulletChar": 42, + "Marker": "Kg==" + }, + "Properties": { + "id": "20220623012800-bptg3o2", + "updated": "20220623012801" + }, + "Children": [ + { + "ID": "20220623012800-u3f3ni2", + "Type": "NodeParagraph", + "Properties": { + "id": "20220623012800-u3f3ni2", + "updated": "20220623012801" + }, + "Children": [ + { + "Type": "NodeLink", + "Data": "span", + "Children": [ + { + "Type": "NodeOpenBracket" + }, + { + "Type": "NodeLinkText", + "Data": "https://github.com/siyuan-note/httpclient" + }, + { + "Type": "NodeCloseBracket" + }, + { + "Type": "NodeOpenParen" + }, + { + "Type": "NodeLinkDest", + "Data": "https://github.com/siyuan-note/httpclient" + }, + { + "Type": "NodeCloseParen" + } + ] + }, + { + "Type": "NodeText", + "Data": " " + }, + { + "Type": "NodeCodeSpan", + "Data": "code", + "Children": [ + { + "Type": "NodeCodeSpanOpenMarker" + }, + { + "Type": "NodeCodeSpanContent", + "Data": "Mulan PSL v2" + }, + { + "Type": "NodeCodeSpanCloseMarker" + } + ] + } + ] + } + ] + }, { "ID": "20220522101224-4u2mzml", "Type": "NodeListItem", diff --git a/app/guide/20211226090932-5lcq56f/20211226114929-08ap1r0.sy b/app/guide/20211226090932-5lcq56f/20211226114929-08ap1r0.sy index a599d4872..1e08a4d71 100644 --- a/app/guide/20211226090932-5lcq56f/20211226114929-08ap1r0.sy +++ b/app/guide/20211226090932-5lcq56f/20211226114929-08ap1r0.sy @@ -5,7 +5,7 @@ "icon": "1f64f", "id": "20211226114929-08ap1r0", "title": "致謝", - "updated": "20220615154945" + "updated": "20220623012807" }, "Children": [ { @@ -28,7 +28,7 @@ "ListData": {}, "Properties": { "id": "20220326092650-r61gazd", - "updated": "20220615154945" + "updated": "20220623012807" }, "Children": [ { @@ -2691,6 +2691,145 @@ } ] }, + { + "ID": "20220623012731-d9w9cov", + "Type": "NodeListItem", + "ListData": { + "BulletChar": 42, + "Marker": "Kg==" + }, + "Properties": { + "id": "20220623012731-d9w9cov" + }, + "Children": [ + { + "ID": "20220623012731-pgxr89c", + "Type": "NodeParagraph", + "Properties": { + "id": "20220623012731-pgxr89c", + "updated": "20220623012734" + }, + "Children": [ + { + "Type": "NodeLink", + "Data": "span", + "Children": [ + { + "Type": "NodeOpenBracket" + }, + { + "Type": "NodeLinkText", + "Data": "https://github.com/siyuan-note/filelock" + }, + { + "Type": "NodeCloseBracket" + }, + { + "Type": "NodeOpenParen" + }, + { + "Type": "NodeLinkDest", + "Data": "https://github.com/siyuan-note/filelock" + }, + { + "Type": "NodeCloseParen" + } + ] + }, + { + "Type": "NodeText", + "Data": " " + }, + { + "Type": "NodeCodeSpan", + "Data": "code", + "Children": [ + { + "Type": "NodeCodeSpanOpenMarker" + }, + { + "Type": "NodeCodeSpanContent", + "Data": "Mulan PSL v2" + }, + { + "Type": "NodeCodeSpanCloseMarker" + } + ] + } + ] + } + ] + }, + { + "ID": "20220623012806-apc6y0v", + "Type": "NodeListItem", + "ListData": { + "BulletChar": 42, + "Marker": "Kg==" + }, + "Properties": { + "id": "20220623012806-apc6y0v", + "updated": "20220623012807" + }, + "Children": [ + { + "ID": "20220623012806-vjhbdnn", + "Type": "NodeParagraph", + "Properties": { + "id": "20220623012806-vjhbdnn", + "updated": "20220623012807" + }, + "Children": [ + { + "Type": "NodeLink", + "Data": "span", + "Children": [ + { + "Type": "NodeOpenBracket" + }, + { + "Type": "NodeLinkText", + "Data": "https://github.com/siyuan-note/httpclient" + }, + { + "Type": "NodeCloseBracket" + }, + { + "Type": "NodeOpenParen" + }, + { + "Type": "NodeLinkDest", + "Data": "https://github.com/siyuan-note/httpclient" + }, + { + "Type": "NodeCloseParen" + } + ] + }, + { + "Type": "NodeText", + "Data": " " + }, + { + "Type": "NodeCodeSpan", + "Data": "code", + "Children": [ + { + "Type": "NodeCodeSpanOpenMarker" + }, + { + "Type": "NodeCodeSpanContent", + "Data": "Mulan PSL v2" + }, + { + "Type": "NodeCodeSpanCloseMarker" + } + ] + } + ] + } + ] + }, { "ID": "20220522101113-k9lx1nj", "Type": "NodeListItem",