diff --git a/kernel/bazaar/icon.go b/kernel/bazaar/icon.go index 96fa69218..cc5191e5c 100644 --- a/kernel/bazaar/icon.go +++ b/kernel/bazaar/icon.go @@ -21,11 +21,8 @@ import ( "path/filepath" "sort" "strings" - "sync" "github.com/88250/go-humanize" - ants "github.com/panjf2000/ants/v2" - "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -34,102 +31,69 @@ type Icon struct { *Package } +// Icons 返回集市图标列表 func Icons() (icons []*Icon) { icons = []*Icon{} + result := getStageAndBazaar("icons") - isOnline := isBazzarOnline() - if !isOnline { + if !result.Online { + return + } + if result.StageErr != nil { + return + } + if 1 > len(result.BazaarIndex) { return } - stageIndex, err := getStageIndex("icons") - if err != nil { - return + for _, repo := range result.StageIndex.Repos { + if nil == repo.Package { + continue + } + icon := buildIconFromStageRepo(repo, result.BazaarIndex) + if nil != icon { + icons = append(icons, icon) + } } - bazaarIndex := getBazaarIndex() - if 1 > len(bazaarIndex) { - return - } - - requestFailed := false - waitGroup := &sync.WaitGroup{} - lock := &sync.Mutex{} - p, _ := ants.NewPoolWithFunc(2, func(arg interface{}) { - defer waitGroup.Done() - - repo := arg.(*StageRepo) - repoURL := repo.URL - - if pkg, found := packageCache.Get(repoURL); found { - lock.Lock() - icons = append(icons, pkg.(*Icon)) - lock.Unlock() - return - } - - if requestFailed { - return - } - - icon := &Icon{} - innerU := util.BazaarOSSServer + "/package/" + repoURL + "/icon.json" - innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(icon).Get(innerU) - if nil != innerErr { - logging.LogErrorf("get bazaar package [%s] failed: %s", repoURL, innerErr) - requestFailed = true - return - } - if 200 != innerResp.StatusCode { - logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode) - requestFailed = true - return - } - - icon.DisallowInstall = disallowInstallBazaarPackage(icon.Package) - icon.DisallowUpdate = disallowInstallBazaarPackage(icon.Package) - icon.UpdateRequiredMinAppVer = icon.MinAppVersion - - icon.URL = strings.TrimSuffix(icon.URL, "/") - repoURLHash := strings.Split(repoURL, "@") - icon.RepoURL = "https://github.com/" + repoURLHash[0] - icon.RepoHash = repoURLHash[1] - icon.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim" - icon.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232" - icon.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png" - icon.Funding = repo.Package.Funding - icon.PreferredFunding = getPreferredFunding(icon.Funding) - icon.PreferredName = GetPreferredName(icon.Package) - icon.PreferredDesc = getPreferredDesc(icon.Description) - icon.Updated = repo.Updated - icon.Stars = repo.Stars - icon.OpenIssues = repo.OpenIssues - icon.Size = repo.Size - icon.HSize = humanize.BytesCustomCeil(uint64(icon.Size), 2) - icon.InstallSize = repo.InstallSize - icon.HInstallSize = humanize.BytesCustomCeil(uint64(icon.InstallSize), 2) - packageInstallSizeCache.SetDefault(icon.RepoURL, icon.InstallSize) - icon.HUpdated = formatUpdated(icon.Updated) - pkg := bazaarIndex[strings.Split(repoURL, "@")[0]] - if nil != pkg { - icon.Downloads = pkg.Downloads - } - lock.Lock() - icons = append(icons, icon) - lock.Unlock() - - packageCache.SetDefault(repoURL, icon) - }) - for _, repo := range stageIndex.Repos { - waitGroup.Add(1) - p.Invoke(repo) - } - waitGroup.Wait() - p.Release() sort.Slice(icons, func(i, j int) bool { return icons[i].Updated > icons[j].Updated }) return } +// buildIconFromStageRepo 使用 stage 内嵌的 package 构建 *Icon，不发起 HTTP 请求。 +func buildIconFromStageRepo(repo *StageRepo, bazaarIndex map[string]*bazaarPackage) *Icon { + pkg := *repo.Package + pkg.URL = strings.TrimSuffix(pkg.URL, "/") + repoURLHash := strings.Split(repo.URL, "@") + if 2 != len(repoURLHash) { + return nil + } + pkg.RepoURL = "https://github.com/" + repoURLHash[0] + pkg.RepoHash = repoURLHash[1] + pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim" + pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232" + pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png" + pkg.Updated = repo.Updated + pkg.Stars = repo.Stars + pkg.OpenIssues = repo.OpenIssues + pkg.Size = repo.Size + pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2) + pkg.InstallSize = repo.InstallSize + pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2) + pkg.HUpdated = formatUpdated(pkg.Updated) + pkg.PreferredFunding = getPreferredFunding(pkg.Funding) + pkg.PreferredName = GetPreferredName(&pkg) + pkg.PreferredDesc = getPreferredDesc(pkg.Description) + pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg) + pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg) + pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion + if bp := bazaarIndex[repoURLHash[0]]; nil != bp { + pkg.Downloads = bp.Downloads + } + packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize) + return &Icon{Package: &pkg} +} + func InstalledIcons() (ret []*Icon) { ret = []*Icon{} diff --git a/kernel/bazaar/package.go b/kernel/bazaar/package.go index 2cb8c9805..201eaa818 100644 --- a/kernel/bazaar/package.go +++ b/kernel/bazaar/package.go @@ -18,6 +18,7 @@ package bazaar import ( "bytes" + "context" "errors" "fmt" "os" @@ -36,6 +37,7 @@ import ( "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" "golang.org/x/mod/semver" + "golang.org/x/sync/singleflight" textUnicode "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" ) @@ -96,16 +98,6 @@ type Package struct { Incompatible bool `json:"incompatible"` } -type StagePackage struct { - Author string `json:"author"` - URL string `json:"url"` - Version string `json:"version"` - Description LocaleStrings `json:"description"` - Readme LocaleStrings `json:"readme"` - I18N []string `json:"i18n"` - Funding *Funding `json:"funding"` -} - type StageRepo struct { URL string `json:"url"` Updated string `json:"updated"` @@ -114,7 +106,8 @@ type StageRepo struct { Size int64 `json:"size"` InstallSize int64 `json:"installSize"` - Package *StagePackage `json:"package"` + // Package 与 stage/*.json 内嵌的完整 package 一致，可直接用于构建列表 + Package *Package `json:"package"` } type StageIndex struct { @@ -283,38 +276,120 @@ func ThemeJSON(themeDirName string) (ret *Theme, err error) { var cachedStageIndex = map[string]*StageIndex{} var stageIndexCacheTime int64 -var stageIndexLock = sync.Mutex{} +var stageIndexLock = sync.RWMutex{} -func getStageIndex(pkgType string) (ret *StageIndex, err error) { - rhyRet, err := util.GetRhyResult(false) +type StageBazaarResult struct { + StageIndex *StageIndex // stage 索引 + BazaarIndex map[string]*bazaarPackage // bazaar 索引 + Online bool // online 状态 + StageErr error // stage 错误 +} + +var stageBazaarFlight singleflight.Group + +// getStageAndBazaar 获取 stage 索引和 bazaar 索引，相同 pkgType 的并发调用会合并为一次实际请求 (single-flight) +func getStageAndBazaar(pkgType string) (result StageBazaarResult) { + key := "stageBazaar:" + pkgType + v, err, _ := stageBazaarFlight.Do(key, func() (interface{}, error) { + return getStageAndBazaar0(pkgType), nil + }) if err != nil { return } + result = v.(StageBazaarResult) + return +} + +// getStageAndBazaar0 执行一次 stage 和 bazaar 索引拉取 +func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { + stageIndex, stageErr := getStageIndexFromCache(pkgType) + bazaarIndex := getBazaarIndexFromCache() + if nil != stageIndex && nil != bazaarIndex { + // 两者都从缓存返回，不需要 online 检查 + return StageBazaarResult{ + StageIndex: stageIndex, + BazaarIndex: bazaarIndex, + Online: true, + StageErr: stageErr, + } + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var onlineResult bool + wg := &sync.WaitGroup{} + wg.Add(3) + go func() { + defer wg.Done() + onlineResult = isBazzarOnline() + if !onlineResult { + cancel() + } + }() + go func() { + defer wg.Done() + stageIndex, stageErr = getStageIndex(ctx, pkgType) + }() + go func() { + defer wg.Done() + bazaarIndex = getBazaarIndex(ctx) + }() + wg.Wait() + + return StageBazaarResult{ + StageIndex: stageIndex, + BazaarIndex: bazaarIndex, + Online: onlineResult, + StageErr: stageErr, + } +} + +// getStageIndexFromCache 仅从缓存获取 stage 索引，过期或无缓存时返回 nil +func getStageIndexFromCache(pkgType string) (ret *StageIndex, err error) { + stageIndexLock.RLock() + cacheTime := stageIndexCacheTime + cached := cachedStageIndex[pkgType] + stageIndexLock.RUnlock() + if util.RhyCacheDuration >= time.Now().Unix()-cacheTime && nil != cached { + ret = cached + } + return +} + +// getStageIndex 获取 stage 索引 +func getStageIndex(ctx context.Context, pkgType string) (ret *StageIndex, err error) { + if cached, cacheErr := getStageIndexFromCache(pkgType); nil != cached { + ret = cached + err = cacheErr + return + } + + var rhyRet map[string]interface{} + rhyRet, err = util.GetRhyResult(false) + if nil != err { + return + } stageIndexLock.Lock() defer stageIndexLock.Unlock() - now := time.Now().Unix() - if util.RhyCacheDuration >= now-stageIndexCacheTime && nil != cachedStageIndex[pkgType] { - ret = cachedStageIndex[pkgType] - return - } - bazaarHash := rhyRet["bazaar"].(string) ret = &StageIndex{} request := httpclient.NewBrowserRequest() u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/" + pkgType + ".json" - resp, reqErr := request.SetSuccessResult(ret).Get(u) + resp, reqErr := request.SetContext(ctx).SetSuccessResult(ret).Get(u) if nil != reqErr { logging.LogErrorf("get community stage index [%s] failed: %s", u, reqErr) + err = reqErr return } if 200 != resp.StatusCode { logging.LogErrorf("get community stage index [%s] failed: %d", u, resp.StatusCode) + err = errors.New("get stage index failed") return } - stageIndexCacheTime = now + stageIndexCacheTime = time.Now().Unix() cachedStageIndex[pkgType] = ret return } @@ -421,7 +496,7 @@ func isOutdatedTemplate(template *Template, bazaarTemplates []*Template) bool { func isBazzarOnline() (ret bool) { // Improve marketplace loading when offline https://github.com/siyuan-note/siyuan/issues/12050 - ret = util.IsOnline(util.BazaarOSSServer, true, 3000) + ret = util.IsOnline(util.BazaarOSSServer+"/204", true, 3000) if !ret { util.PushErrMsg(util.Langs[util.Lang][24], 5000) } @@ -431,7 +506,9 @@ func isBazzarOnline() (ret bool) { func GetPackageREADME(repoURL, repoHash, packageType string) (ret string) { repoURLHash := repoURL + "@" + repoHash + stageIndexLock.RLock() stageIndex := cachedStageIndex[packageType] + stageIndexLock.RUnlock() if nil == stageIndex { return } @@ -444,7 +521,7 @@ func GetPackageREADME(repoURL, repoHash, packageType string) (ret string) { break } } - if nil == repo { + if nil == repo || nil == repo.Package { return } @@ -683,20 +760,35 @@ type bazaarPackage struct { var cachedBazaarIndex = map[string]*bazaarPackage{} var bazaarIndexCacheTime int64 -var bazaarIndexLock = sync.Mutex{} +var bazaarIndexLock = sync.RWMutex{} + +// getBazaarIndexFromCache 仅从缓存获取 bazaar 索引，过期或无缓存时返回 nil +func getBazaarIndexFromCache() (ret map[string]*bazaarPackage) { + bazaarIndexLock.RLock() + cacheTime := bazaarIndexCacheTime + cached := cachedBazaarIndex + hasData := 0 < len(cached) + bazaarIndexLock.RUnlock() + if util.RhyCacheDuration >= time.Now().Unix()-cacheTime && hasData { + ret = cached + } else { + ret = nil + } + return +} + +// getBazaarIndex 获取 bazaar 索引 +func getBazaarIndex(ctx context.Context) map[string]*bazaarPackage { + if cached := getBazaarIndexFromCache(); nil != cached { + return cached + } -func getBazaarIndex() map[string]*bazaarPackage { bazaarIndexLock.Lock() defer bazaarIndexLock.Unlock() - now := time.Now().Unix() - if 3600 >= now-bazaarIndexCacheTime { - return cachedBazaarIndex - } - request := httpclient.NewBrowserRequest() u := util.BazaarStatServer + "/bazaar/index.json" - resp, reqErr := request.SetSuccessResult(&cachedBazaarIndex).Get(u) + resp, reqErr := request.SetContext(ctx).SetSuccessResult(&cachedBazaarIndex).Get(u) if nil != reqErr { logging.LogErrorf("get bazaar index [%s] failed: %s", u, reqErr) return cachedBazaarIndex @@ -705,19 +797,18 @@ func getBazaarIndex() map[string]*bazaarPackage { logging.LogErrorf("get bazaar index [%s] failed: %d", u, resp.StatusCode) return cachedBazaarIndex } - bazaarIndexCacheTime = now + bazaarIndexCacheTime = time.Now().Unix() return cachedBazaarIndex } -// defaultMinAppVersion 如果集市包中缺失 minAppVersion 项，则使用该值作为最低支持的版本号，小于该版本号时不显示集市包 // Add marketplace package config item `minAppVersion` https://github.com/siyuan-note/siyuan/issues/8330 -const defaultMinAppVersion = "2.9.0" - func disallowInstallBazaarPackage(pkg *Package) bool { + // 如果包没有指定 minAppVersion，则允许安装 if "" == pkg.MinAppVersion { - pkg.MinAppVersion = defaultMinAppVersion + return false } + // 如果包要求的 minAppVersion 大于当前版本，则不允许安装 if 0 < semver.Compare("v"+pkg.MinAppVersion, "v"+util.Ver) { return true } diff --git a/kernel/bazaar/plugin.go b/kernel/bazaar/plugin.go index e61e67791..8d159cb0a 100644 --- a/kernel/bazaar/plugin.go +++ b/kernel/bazaar/plugin.go @@ -22,11 +22,8 @@ import ( "runtime" "sort" "strings" - "sync" "github.com/88250/go-humanize" - ants "github.com/panjf2000/ants/v2" - "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -36,103 +33,70 @@ type Plugin struct { Enabled bool `json:"enabled"` } +// Plugins 返回集市插件列表 func Plugins(frontend string) (plugins []*Plugin) { plugins = []*Plugin{} + result := getStageAndBazaar("plugins") - isOnline := isBazzarOnline() - if !isOnline { + if !result.Online { + return + } + if result.StageErr != nil { + return + } + if 1 > len(result.BazaarIndex) { return } - stageIndex, err := getStageIndex("plugins") - if err != nil { - return + for _, repo := range result.StageIndex.Repos { + if nil == repo.Package { + continue + } + plugin := buildPluginFromStageRepo(repo, frontend, result.BazaarIndex) + if nil != plugin { + plugins = append(plugins, plugin) + } } - bazaarIndex := getBazaarIndex() - if 1 > len(bazaarIndex) { - return - } - - requestFailed := false - waitGroup := &sync.WaitGroup{} - lock := &sync.Mutex{} - p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) { - defer waitGroup.Done() - - repo := arg.(*StageRepo) - repoURL := repo.URL - - if pkg, found := packageCache.Get(repoURL); found { - lock.Lock() - plugins = append(plugins, pkg.(*Plugin)) - lock.Unlock() - return - } - - if requestFailed { - return - } - - plugin := &Plugin{} - innerU := util.BazaarOSSServer + "/package/" + repoURL + "/plugin.json" - innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(plugin).Get(innerU) - if nil != innerErr { - logging.LogErrorf("get bazaar package [%s] failed: %s", repoURL, innerErr) - requestFailed = true - return - } - if 200 != innerResp.StatusCode { - logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode) - requestFailed = true - return - } - - plugin.DisallowInstall = disallowInstallBazaarPackage(plugin.Package) - plugin.DisallowUpdate = disallowInstallBazaarPackage(plugin.Package) - plugin.UpdateRequiredMinAppVer = plugin.MinAppVersion - plugin.Incompatible = isIncompatiblePlugin(plugin, frontend) - - plugin.URL = strings.TrimSuffix(plugin.URL, "/") - repoURLHash := strings.Split(repoURL, "@") - plugin.RepoURL = "https://github.com/" + repoURLHash[0] - plugin.RepoHash = repoURLHash[1] - plugin.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim" - plugin.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232" - plugin.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png" - plugin.Funding = repo.Package.Funding - plugin.PreferredFunding = getPreferredFunding(plugin.Funding) - plugin.PreferredName = GetPreferredName(plugin.Package) - plugin.PreferredDesc = getPreferredDesc(plugin.Description) - plugin.Updated = repo.Updated - plugin.Stars = repo.Stars - plugin.OpenIssues = repo.OpenIssues - plugin.Size = repo.Size - plugin.HSize = humanize.BytesCustomCeil(uint64(plugin.Size), 2) - plugin.InstallSize = repo.InstallSize - plugin.HInstallSize = humanize.BytesCustomCeil(uint64(plugin.InstallSize), 2) - packageInstallSizeCache.SetDefault(plugin.RepoURL, plugin.InstallSize) - plugin.HUpdated = formatUpdated(plugin.Updated) - pkg := bazaarIndex[strings.Split(repoURL, "@")[0]] - if nil != pkg { - plugin.Downloads = pkg.Downloads - } - lock.Lock() - plugins = append(plugins, plugin) - lock.Unlock() - - packageCache.SetDefault(repoURL, plugin) - }) - for _, repo := range stageIndex.Repos { - waitGroup.Add(1) - p.Invoke(repo) - } - waitGroup.Wait() - p.Release() sort.Slice(plugins, func(i, j int) bool { return plugins[i].Updated > plugins[j].Updated }) return } +// buildPluginFromStageRepo 使用 stage 内嵌的 package 构建 *Plugin，不发起 HTTP 请求。 +func buildPluginFromStageRepo(repo *StageRepo, frontend string, bazaarIndex map[string]*bazaarPackage) *Plugin { + pkg := *repo.Package + pkg.URL = strings.TrimSuffix(pkg.URL, "/") + repoURLHash := strings.Split(repo.URL, "@") + if 2 != len(repoURLHash) { + return nil + } + pkg.RepoURL = "https://github.com/" + repoURLHash[0] + pkg.RepoHash = repoURLHash[1] + pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim" + pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232" + pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png" + pkg.Updated = repo.Updated + pkg.Stars = repo.Stars + pkg.OpenIssues = repo.OpenIssues + pkg.Size = repo.Size + pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2) + pkg.InstallSize = repo.InstallSize + pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2) + pkg.HUpdated = formatUpdated(pkg.Updated) + pkg.PreferredFunding = getPreferredFunding(pkg.Funding) + pkg.PreferredName = GetPreferredName(&pkg) + pkg.PreferredDesc = getPreferredDesc(pkg.Description) + pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg) + pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg) + pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion + pkg.Incompatible = isIncompatiblePlugin(&Plugin{Package: &pkg}, frontend) + if bp := bazaarIndex[repoURLHash[0]]; nil != bp { + pkg.Downloads = bp.Downloads + } + packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize) + return &Plugin{Package: &pkg} +} + func ParseInstalledPlugin(name, frontend string) (found bool, displayName string, incompatible, disabledInPublish, disallowInstall bool) { pluginsPath := filepath.Join(util.DataDir, "plugins") if !util.IsPathRegularDirOrSymlinkDir(pluginsPath) { diff --git a/kernel/bazaar/template.go b/kernel/bazaar/template.go index d77cc0ee5..07cfd3155 100644 --- a/kernel/bazaar/template.go +++ b/kernel/bazaar/template.go @@ -21,12 +21,9 @@ import ( "path/filepath" "sort" "strings" - "sync" "time" "github.com/88250/go-humanize" - "github.com/panjf2000/ants/v2" - "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -35,97 +32,30 @@ type Template struct { *Package } +// Templates 返回集市模板列表 func Templates() (templates []*Template) { templates = []*Template{} + result := getStageAndBazaar("templates") - isOnline := isBazzarOnline() - if !isOnline { + if !result.Online { + return + } + if result.StageErr != nil { + return + } + if 1 > len(result.BazaarIndex) { return } - stageIndex, err := getStageIndex("templates") - if err != nil { - return + for _, repo := range result.StageIndex.Repos { + if nil == repo.Package { + continue + } + template := buildTemplateFromStageRepo(repo, result.BazaarIndex) + if nil != template { + templates = append(templates, template) + } } - bazaarIndex := getBazaarIndex() - if 1 > len(bazaarIndex) { - return - } - - requestFailed := false - waitGroup := &sync.WaitGroup{} - lock := &sync.Mutex{} - p, _ := ants.NewPoolWithFunc(2, func(arg interface{}) { - defer waitGroup.Done() - - repo := arg.(*StageRepo) - repoURL := repo.URL - - if pkg, found := packageCache.Get(repoURL); found { - lock.Lock() - templates = append(templates, pkg.(*Template)) - lock.Unlock() - return - } - - if requestFailed { - return - } - - template := &Template{} - innerU := util.BazaarOSSServer + "/package/" + repoURL + "/template.json" - innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(template).Get(innerU) - if nil != innerErr { - logging.LogErrorf("get community template [%s] failed: %s", repoURL, innerErr) - requestFailed = true - return - } - if 200 != innerResp.StatusCode { - logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode) - requestFailed = true - return - } - - template.DisallowInstall = disallowInstallBazaarPackage(template.Package) - template.DisallowUpdate = disallowInstallBazaarPackage(template.Package) - template.UpdateRequiredMinAppVer = template.MinAppVersion - - template.URL = strings.TrimSuffix(template.URL, "/") - repoURLHash := strings.Split(repoURL, "@") - template.RepoURL = "https://github.com/" + repoURLHash[0] - template.RepoHash = repoURLHash[1] - template.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim" - template.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232" - template.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png" - template.Funding = repo.Package.Funding - template.PreferredFunding = getPreferredFunding(template.Funding) - template.PreferredName = GetPreferredName(template.Package) - template.PreferredDesc = getPreferredDesc(template.Description) - template.Updated = repo.Updated - template.Stars = repo.Stars - template.OpenIssues = repo.OpenIssues - template.Size = repo.Size - template.HSize = humanize.BytesCustomCeil(uint64(template.Size), 2) - template.InstallSize = repo.InstallSize - template.HInstallSize = humanize.BytesCustomCeil(uint64(template.InstallSize), 2) - packageInstallSizeCache.SetDefault(template.RepoURL, template.InstallSize) - template.HUpdated = formatUpdated(template.Updated) - pkg := bazaarIndex[strings.Split(repoURL, "@")[0]] - if nil != pkg { - template.Downloads = pkg.Downloads - } - lock.Lock() - templates = append(templates, template) - lock.Unlock() - - packageCache.SetDefault(repoURL, template) - }) - for _, repo := range stageIndex.Repos { - waitGroup.Add(1) - p.Invoke(repo) - } - waitGroup.Wait() - p.Release() templates = filterLegacyTemplates(templates) @@ -133,6 +63,40 @@ func Templates() (templates []*Template) { return } +// buildTemplateFromStageRepo 使用 stage 内嵌的 package 构建 *Template，不发起 HTTP 请求。 +func buildTemplateFromStageRepo(repo *StageRepo, bazaarIndex map[string]*bazaarPackage) *Template { + pkg := *repo.Package + pkg.URL = strings.TrimSuffix(pkg.URL, "/") + repoURLHash := strings.Split(repo.URL, "@") + if 2 != len(repoURLHash) { + return nil + } + pkg.RepoURL = "https://github.com/" + repoURLHash[0] + pkg.RepoHash = repoURLHash[1] + pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim" + pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232" + pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png" + pkg.Updated = repo.Updated + pkg.Stars = repo.Stars + pkg.OpenIssues = repo.OpenIssues + pkg.Size = repo.Size + pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2) + pkg.InstallSize = repo.InstallSize + pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2) + pkg.HUpdated = formatUpdated(pkg.Updated) + pkg.PreferredFunding = getPreferredFunding(pkg.Funding) + pkg.PreferredName = GetPreferredName(&pkg) + pkg.PreferredDesc = getPreferredDesc(pkg.Description) + pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg) + pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg) + pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion + if bp := bazaarIndex[repoURLHash[0]]; nil != bp { + pkg.Downloads = bp.Downloads + } + packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize) + return &Template{Package: &pkg} +} + func InstalledTemplates() (ret []*Template) { ret = []*Template{} diff --git a/kernel/bazaar/theme.go b/kernel/bazaar/theme.go index e25b85c96..efcc38bd8 100644 --- a/kernel/bazaar/theme.go +++ b/kernel/bazaar/theme.go @@ -21,11 +21,8 @@ import ( "path/filepath" "sort" "strings" - "sync" "github.com/88250/go-humanize" - ants "github.com/panjf2000/ants/v2" - "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -36,102 +33,70 @@ type Theme struct { Modes []string `json:"modes"` } +// Themes 返回集市主题列表 func Themes() (ret []*Theme) { ret = []*Theme{} + result := getStageAndBazaar("themes") - isOnline := isBazzarOnline() - if !isOnline { + if !result.Online { + return + } + if result.StageErr != nil { + return + } + if 1 > len(result.BazaarIndex) { return } - stageIndex, err := getStageIndex("themes") - if err != nil { - return + for _, repo := range result.StageIndex.Repos { + if nil == repo.Package { + continue + } + theme := buildThemeFromStageRepo(repo, result.BazaarIndex) + if nil != theme { + ret = append(ret, theme) + } } - bazaarIndex := getBazaarIndex() - if 1 > len(bazaarIndex) { - return - } - - requestFailed := false - waitGroup := &sync.WaitGroup{} - lock := &sync.Mutex{} - p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) { - defer waitGroup.Done() - - repo := arg.(*StageRepo) - repoURL := repo.URL - - if pkg, found := packageCache.Get(repoURL); found { - lock.Lock() - ret = append(ret, pkg.(*Theme)) - lock.Unlock() - return - } - - if requestFailed { - return - } - - theme := &Theme{} - innerU := util.BazaarOSSServer + "/package/" + repoURL + "/theme.json" - innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(theme).Get(innerU) - if nil != innerErr { - logging.LogErrorf("get bazaar package [%s] failed: %s", innerU, innerErr) - requestFailed = true - return - } - if 200 != innerResp.StatusCode { - logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode) - requestFailed = true - return - } - - theme.DisallowInstall = disallowInstallBazaarPackage(theme.Package) - theme.DisallowUpdate = disallowInstallBazaarPackage(theme.Package) - theme.UpdateRequiredMinAppVer = theme.MinAppVersion - - theme.URL = strings.TrimSuffix(theme.URL, "/") - repoURLHash := strings.Split(repoURL, "@") - theme.RepoURL = "https://github.com/" + repoURLHash[0] - theme.RepoHash = repoURLHash[1] - theme.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim" - theme.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232" - theme.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png" - theme.Funding = repo.Package.Funding - theme.PreferredFunding = getPreferredFunding(theme.Funding) - theme.PreferredName = GetPreferredName(theme.Package) - theme.PreferredDesc = getPreferredDesc(theme.Description) - theme.Updated = repo.Updated - theme.Stars = repo.Stars - theme.OpenIssues = repo.OpenIssues - theme.Size = repo.Size - theme.HSize = humanize.BytesCustomCeil(uint64(theme.Size), 2) - theme.InstallSize = repo.InstallSize - theme.HInstallSize = humanize.BytesCustomCeil(uint64(theme.InstallSize), 2) - packageInstallSizeCache.SetDefault(theme.RepoURL, theme.InstallSize) - theme.HUpdated = formatUpdated(theme.Updated) - pkg := bazaarIndex[strings.Split(repoURL, "@")[0]] - if nil != pkg { - theme.Downloads = pkg.Downloads - } - lock.Lock() - ret = append(ret, theme) - lock.Unlock() - - packageCache.SetDefault(repoURL, theme) - }) - for _, repo := range stageIndex.Repos { - waitGroup.Add(1) - p.Invoke(repo) - } - waitGroup.Wait() - p.Release() sort.Slice(ret, func(i, j int) bool { return ret[i].Updated > ret[j].Updated }) return } +// buildThemeFromStageRepo 使用 stage 内嵌的 package 构建 *Theme，不发起 HTTP 请求。 +func buildThemeFromStageRepo(repo *StageRepo, bazaarIndex map[string]*bazaarPackage) *Theme { + pkg := *repo.Package + pkg.URL = strings.TrimSuffix(pkg.URL, "/") + repoURLHash := strings.Split(repo.URL, "@") + if 2 != len(repoURLHash) { + return nil + } + pkg.RepoURL = "https://github.com/" + repoURLHash[0] + pkg.RepoHash = repoURLHash[1] + pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim" + pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232" + pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png" + pkg.Updated = repo.Updated + pkg.Stars = repo.Stars + pkg.OpenIssues = repo.OpenIssues + pkg.Size = repo.Size + pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2) + pkg.InstallSize = repo.InstallSize + pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2) + pkg.HUpdated = formatUpdated(pkg.Updated) + pkg.PreferredFunding = getPreferredFunding(pkg.Funding) + pkg.PreferredName = GetPreferredName(&pkg) + pkg.PreferredDesc = getPreferredDesc(pkg.Description) + pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg) + pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg) + pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion + if bp := bazaarIndex[repoURLHash[0]]; nil != bp { + pkg.Downloads = bp.Downloads + } + packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize) + theme := &Theme{Package: &pkg, Modes: []string{}} + return theme +} + func InstalledThemes() (ret []*Theme) { ret = []*Theme{} diff --git a/kernel/bazaar/widget.go b/kernel/bazaar/widget.go index 56196fbb4..405f5c1f5 100644 --- a/kernel/bazaar/widget.go +++ b/kernel/bazaar/widget.go @@ -21,11 +21,8 @@ import ( "path/filepath" "sort" "strings" - "sync" "github.com/88250/go-humanize" - ants "github.com/panjf2000/ants/v2" - "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -34,102 +31,69 @@ type Widget struct { *Package } +// Widgets 返回集市挂件列表 func Widgets() (widgets []*Widget) { widgets = []*Widget{} + result := getStageAndBazaar("widgets") - isOnline := isBazzarOnline() - if !isOnline { + if !result.Online { + return + } + if result.StageErr != nil { + return + } + if 1 > len(result.BazaarIndex) { return } - stageIndex, err := getStageIndex("widgets") - if err != nil { - return + for _, repo := range result.StageIndex.Repos { + if nil == repo.Package { + continue + } + widget := buildWidgetFromStageRepo(repo, result.BazaarIndex) + if nil != widget { + widgets = append(widgets, widget) + } } - bazaarIndex := getBazaarIndex() - if 1 > len(bazaarIndex) { - return - } - - requestFailed := false - waitGroup := &sync.WaitGroup{} - lock := &sync.Mutex{} - p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) { - defer waitGroup.Done() - - repo := arg.(*StageRepo) - repoURL := repo.URL - - if pkg, found := packageCache.Get(repoURL); found { - lock.Lock() - widgets = append(widgets, pkg.(*Widget)) - lock.Unlock() - return - } - - if requestFailed { - return - } - - widget := &Widget{} - innerU := util.BazaarOSSServer + "/package/" + repoURL + "/widget.json" - innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(widget).Get(innerU) - if nil != innerErr { - logging.LogErrorf("get bazaar package [%s] failed: %s", repoURL, innerErr) - requestFailed = true - return - } - if 200 != innerResp.StatusCode { - logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode) - requestFailed = true - return - } - - widget.DisallowInstall = disallowInstallBazaarPackage(widget.Package) - widget.DisallowUpdate = disallowInstallBazaarPackage(widget.Package) - widget.UpdateRequiredMinAppVer = widget.MinAppVersion - - widget.URL = strings.TrimSuffix(widget.URL, "/") - repoURLHash := strings.Split(repoURL, "@") - widget.RepoURL = "https://github.com/" + repoURLHash[0] - widget.RepoHash = repoURLHash[1] - widget.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim" - widget.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232" - widget.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png" - widget.Funding = repo.Package.Funding - widget.PreferredFunding = getPreferredFunding(widget.Funding) - widget.PreferredName = GetPreferredName(widget.Package) - widget.PreferredDesc = getPreferredDesc(widget.Description) - widget.Updated = repo.Updated - widget.Stars = repo.Stars - widget.OpenIssues = repo.OpenIssues - widget.Size = repo.Size - widget.HSize = humanize.BytesCustomCeil(uint64(widget.Size), 2) - widget.InstallSize = repo.InstallSize - widget.HInstallSize = humanize.BytesCustomCeil(uint64(widget.InstallSize), 2) - packageInstallSizeCache.SetDefault(widget.RepoURL, widget.InstallSize) - widget.HUpdated = formatUpdated(widget.Updated) - pkg := bazaarIndex[strings.Split(repoURL, "@")[0]] - if nil != pkg { - widget.Downloads = pkg.Downloads - } - lock.Lock() - widgets = append(widgets, widget) - lock.Unlock() - - packageCache.SetDefault(repoURL, widget) - }) - for _, repo := range stageIndex.Repos { - waitGroup.Add(1) - p.Invoke(repo) - } - waitGroup.Wait() - p.Release() sort.Slice(widgets, func(i, j int) bool { return widgets[i].Updated > widgets[j].Updated }) return } +// buildWidgetFromStageRepo 使用 stage 内嵌的 package 构建 *Widget，不发起 HTTP 请求。 +func buildWidgetFromStageRepo(repo *StageRepo, bazaarIndex map[string]*bazaarPackage) *Widget { + pkg := *repo.Package + pkg.URL = strings.TrimSuffix(pkg.URL, "/") + repoURLHash := strings.Split(repo.URL, "@") + if 2 != len(repoURLHash) { + return nil + } + pkg.RepoURL = "https://github.com/" + repoURLHash[0] + pkg.RepoHash = repoURLHash[1] + pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim" + pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232" + pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png" + pkg.Updated = repo.Updated + pkg.Stars = repo.Stars + pkg.OpenIssues = repo.OpenIssues + pkg.Size = repo.Size + pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2) + pkg.InstallSize = repo.InstallSize + pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2) + pkg.HUpdated = formatUpdated(pkg.Updated) + pkg.PreferredFunding = getPreferredFunding(pkg.Funding) + pkg.PreferredName = GetPreferredName(&pkg) + pkg.PreferredDesc = getPreferredDesc(pkg.Description) + pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg) + pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg) + pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion + if bp := bazaarIndex[repoURLHash[0]]; nil != bp { + pkg.Downloads = bp.Downloads + } + packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize) + return &Widget{Package: &pkg} +} + func InstalledWidgets() (ret []*Widget) { ret = []*Widget{}