Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2026-02-08 08:14:21 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

⚡ Improve marketplace list loading (#16965)

Browse source 
- 直接使用 stage 索引中的 package 数据，不再为每个包单独请求 JSON
- 统一索引获取逻辑，使用 singleflight 合并并发请求
- 优化在线状态检查耗时，改用 https://oss.b3logfile.com/204
- 改进 disallowInstallBazaarPackage 函数性能

Co-authored-by: D <845765@qq.com>
This commit is contained in:
Jeffrey Chen 2026-02-03 10:07:16 +08:00 • committed by GitHub
parent 9642f486bd
commit 4eec7b5944
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 385 additions and 473 deletions
Download patch file Download diff file Expand all files Collapse all files

138
kernel/bazaar/icon.go
View file

@ -21,11 +21,8 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"github.com/88250/go-humanize"
ants "github.com/panjf2000/ants/v2"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -34,102 +31,69 @@ type Icon struct {
*Package
}
// Icons 返回集市图标列表
func Icons() (icons []*Icon) {
icons = []*Icon{}
result := getStageAndBazaar("icons")
isOnline := isBazzarOnline()
if !isOnline {
if !result.Online {
return
}
if result.StageErr != nil {
return
}
if 1 > len(result.BazaarIndex) {
return
}
stageIndex, err := getStageIndex("icons")
if err != nil {
return
for _, repo := range result.StageIndex.Repos {
if nil == repo.Package {
continue
}
icon := buildIconFromStageRepo(repo, result.BazaarIndex)
if nil != icon {
icons = append(icons, icon)
}
}
bazaarIndex := getBazaarIndex()
if 1 > len(bazaarIndex) {
return
}
requestFailed := false
waitGroup := &sync.WaitGroup{}
lock := &sync.Mutex{}
p, _ := ants.NewPoolWithFunc(2, func(arg interface{}) {
defer waitGroup.Done()
repo := arg.(*StageRepo)
repoURL := repo.URL
if pkg, found := packageCache.Get(repoURL); found {
lock.Lock()
icons = append(icons, pkg.(*Icon))
lock.Unlock()
return
}
if requestFailed {
return
}
icon := &Icon{}
innerU := util.BazaarOSSServer + "/package/" + repoURL + "/icon.json"
innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(icon).Get(innerU)
if nil != innerErr {
logging.LogErrorf("get bazaar package [%s] failed: %s", repoURL, innerErr)
requestFailed = true
return
}
if 200 != innerResp.StatusCode {
logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode)
requestFailed = true
return
}
icon.DisallowInstall = disallowInstallBazaarPackage(icon.Package)
icon.DisallowUpdate = disallowInstallBazaarPackage(icon.Package)
icon.UpdateRequiredMinAppVer = icon.MinAppVersion
icon.URL = strings.TrimSuffix(icon.URL, "/")
repoURLHash := strings.Split(repoURL, "@")
icon.RepoURL = "https://github.com/" + repoURLHash[0]
icon.RepoHash = repoURLHash[1]
icon.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim"
icon.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232"
icon.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png"
icon.Funding = repo.Package.Funding
icon.PreferredFunding = getPreferredFunding(icon.Funding)
icon.PreferredName = GetPreferredName(icon.Package)
icon.PreferredDesc = getPreferredDesc(icon.Description)
icon.Updated = repo.Updated
icon.Stars = repo.Stars
icon.OpenIssues = repo.OpenIssues
icon.Size = repo.Size
icon.HSize = humanize.BytesCustomCeil(uint64(icon.Size), 2)
icon.InstallSize = repo.InstallSize
icon.HInstallSize = humanize.BytesCustomCeil(uint64(icon.InstallSize), 2)
packageInstallSizeCache.SetDefault(icon.RepoURL, icon.InstallSize)
icon.HUpdated = formatUpdated(icon.Updated)
pkg := bazaarIndex[strings.Split(repoURL, "@")[0]]
if nil != pkg {
icon.Downloads = pkg.Downloads
}
lock.Lock()
icons = append(icons, icon)
lock.Unlock()
packageCache.SetDefault(repoURL, icon)
})
for _, repo := range stageIndex.Repos {
waitGroup.Add(1)
p.Invoke(repo)
}
waitGroup.Wait()
p.Release()
sort.Slice(icons, func(i, j int) bool { return icons[i].Updated > icons[j].Updated })
return
}
// buildIconFromStageRepo 使用 stage 内嵌的 package 构建 *Icon，不发起 HTTP 请求。
func buildIconFromStageRepo(repo *StageRepo, bazaarIndex map[string]*bazaarPackage) *Icon {
pkg := *repo.Package
pkg.URL = strings.TrimSuffix(pkg.URL, "/")
repoURLHash := strings.Split(repo.URL, "@")
if 2 != len(repoURLHash) {
return nil
}
pkg.RepoURL = "https://github.com/" + repoURLHash[0]
pkg.RepoHash = repoURLHash[1]
pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim"
pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232"
pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png"
pkg.Updated = repo.Updated
pkg.Stars = repo.Stars
pkg.OpenIssues = repo.OpenIssues
pkg.Size = repo.Size
pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2)
pkg.InstallSize = repo.InstallSize
pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2)
pkg.HUpdated = formatUpdated(pkg.Updated)
pkg.PreferredFunding = getPreferredFunding(pkg.Funding)
pkg.PreferredName = GetPreferredName(&pkg)
pkg.PreferredDesc = getPreferredDesc(pkg.Description)
pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg)
pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg)
pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion
if bp := bazaarIndex[repoURLHash[0]]; nil != bp {
pkg.Downloads = bp.Downloads
}
packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize)
return &Icon{Package: &pkg}
}
func InstalledIcons() (ret []*Icon) {
ret = []*Icon{}

165
kernel/bazaar/package.go
View file

@ -18,6 +18,7 @@ package bazaar
import (
"bytes"
"context"
"errors"
"fmt"
"os"
@ -36,6 +37,7 @@ import (
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
"golang.org/x/mod/semver"
"golang.org/x/sync/singleflight"
textUnicode "golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
)
@ -96,16 +98,6 @@ type Package struct {
Incompatible bool `json:"incompatible"`
}
type StagePackage struct {
Author string `json:"author"`
URL string `json:"url"`
Version string `json:"version"`
Description LocaleStrings `json:"description"`
Readme LocaleStrings `json:"readme"`
I18N []string `json:"i18n"`
Funding *Funding `json:"funding"`
}
type StageRepo struct {
URL string `json:"url"`
Updated string `json:"updated"`
@ -114,7 +106,8 @@ type StageRepo struct {
Size int64 `json:"size"`
InstallSize int64 `json:"installSize"`
Package *StagePackage `json:"package"`
// Package 与 stage/*.json 内嵌的完整 package 一致，可直接用于构建列表
Package *Package `json:"package"`
}
type StageIndex struct {
@ -283,38 +276,120 @@ func ThemeJSON(themeDirName string) (ret *Theme, err error) {
var cachedStageIndex = map[string]*StageIndex{}
var stageIndexCacheTime int64
var stageIndexLock = sync.Mutex{}
var stageIndexLock = sync.RWMutex{}
func getStageIndex(pkgType string) (ret *StageIndex, err error) {
rhyRet, err := util.GetRhyResult(false)
type StageBazaarResult struct {
StageIndex *StageIndex // stage 索引
BazaarIndex map[string]*bazaarPackage // bazaar 索引
Online bool // online 状态
StageErr error // stage 错误
}
var stageBazaarFlight singleflight.Group
// getStageAndBazaar 获取 stage 索引和 bazaar 索引，相同 pkgType 的并发调用会合并为一次实际请求 (single-flight)
func getStageAndBazaar(pkgType string) (result StageBazaarResult) {
key := "stageBazaar:" + pkgType
v, err, _ := stageBazaarFlight.Do(key, func() (interface{}, error) {
return getStageAndBazaar0(pkgType), nil
})
if err != nil {
return
}
result = v.(StageBazaarResult)
return
}
// getStageAndBazaar0 执行一次 stage 和 bazaar 索引拉取
func getStageAndBazaar0(pkgType string) (result StageBazaarResult) {
stageIndex, stageErr := getStageIndexFromCache(pkgType)
bazaarIndex := getBazaarIndexFromCache()
if nil != stageIndex && nil != bazaarIndex {
// 两者都从缓存返回，不需要 online 检查
return StageBazaarResult{
StageIndex: stageIndex,
BazaarIndex: bazaarIndex,
Online: true,
StageErr: stageErr,
}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var onlineResult bool
wg := &sync.WaitGroup{}
wg.Add(3)
go func() {
defer wg.Done()
onlineResult = isBazzarOnline()
if !onlineResult {
cancel()
}
}()
go func() {
defer wg.Done()
stageIndex, stageErr = getStageIndex(ctx, pkgType)
}()
go func() {
defer wg.Done()
bazaarIndex = getBazaarIndex(ctx)
}()
wg.Wait()
return StageBazaarResult{
StageIndex: stageIndex,
BazaarIndex: bazaarIndex,
Online: onlineResult,
StageErr: stageErr,
}
}
// getStageIndexFromCache 仅从缓存获取 stage 索引，过期或无缓存时返回 nil
func getStageIndexFromCache(pkgType string) (ret *StageIndex, err error) {
stageIndexLock.RLock()
cacheTime := stageIndexCacheTime
cached := cachedStageIndex[pkgType]
stageIndexLock.RUnlock()
if util.RhyCacheDuration >= time.Now().Unix()-cacheTime && nil != cached {
ret = cached
}
return
}
// getStageIndex 获取 stage 索引
func getStageIndex(ctx context.Context, pkgType string) (ret *StageIndex, err error) {
if cached, cacheErr := getStageIndexFromCache(pkgType); nil != cached {
ret = cached
err = cacheErr
return
}
var rhyRet map[string]interface{}
rhyRet, err = util.GetRhyResult(false)
if nil != err {
return
}
stageIndexLock.Lock()
defer stageIndexLock.Unlock()
now := time.Now().Unix()
if util.RhyCacheDuration >= now-stageIndexCacheTime && nil != cachedStageIndex[pkgType] {
ret = cachedStageIndex[pkgType]
return
}
bazaarHash := rhyRet["bazaar"].(string)
ret = &StageIndex{}
request := httpclient.NewBrowserRequest()
u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/" + pkgType + ".json"
resp, reqErr := request.SetSuccessResult(ret).Get(u)
resp, reqErr := request.SetContext(ctx).SetSuccessResult(ret).Get(u)
if nil != reqErr {
logging.LogErrorf("get community stage index [%s] failed: %s", u, reqErr)
err = reqErr
return
}
if 200 != resp.StatusCode {
logging.LogErrorf("get community stage index [%s] failed: %d", u, resp.StatusCode)
err = errors.New("get stage index failed")
return
}
stageIndexCacheTime = now
stageIndexCacheTime = time.Now().Unix()
cachedStageIndex[pkgType] = ret
return
}
@ -421,7 +496,7 @@ func isOutdatedTemplate(template *Template, bazaarTemplates []*Template) bool {
func isBazzarOnline() (ret bool) {
// Improve marketplace loading when offline https://github.com/siyuan-note/siyuan/issues/12050
ret = util.IsOnline(util.BazaarOSSServer, true, 3000)
ret = util.IsOnline(util.BazaarOSSServer+"/204", true, 3000)
if !ret {
util.PushErrMsg(util.Langs[util.Lang][24], 5000)
}
@ -431,7 +506,9 @@ func isBazzarOnline() (ret bool) {
func GetPackageREADME(repoURL, repoHash, packageType string) (ret string) {
repoURLHash := repoURL + "@" + repoHash
stageIndexLock.RLock()
stageIndex := cachedStageIndex[packageType]
stageIndexLock.RUnlock()
if nil == stageIndex {
return
}
@ -444,7 +521,7 @@ func GetPackageREADME(repoURL, repoHash, packageType string) (ret string) {
break
}
}
if nil == repo {
if nil == repo || nil == repo.Package {
return
}
@ -683,20 +760,35 @@ type bazaarPackage struct {
var cachedBazaarIndex = map[string]*bazaarPackage{}
var bazaarIndexCacheTime int64
var bazaarIndexLock = sync.Mutex{}
var bazaarIndexLock = sync.RWMutex{}
// getBazaarIndexFromCache 仅从缓存获取 bazaar 索引，过期或无缓存时返回 nil
func getBazaarIndexFromCache() (ret map[string]*bazaarPackage) {
bazaarIndexLock.RLock()
cacheTime := bazaarIndexCacheTime
cached := cachedBazaarIndex
hasData := 0 < len(cached)
bazaarIndexLock.RUnlock()
if util.RhyCacheDuration >= time.Now().Unix()-cacheTime && hasData {
ret = cached
} else {
ret = nil
}
return
}
// getBazaarIndex 获取 bazaar 索引
func getBazaarIndex(ctx context.Context) map[string]*bazaarPackage {
if cached := getBazaarIndexFromCache(); nil != cached {
return cached
}
func getBazaarIndex() map[string]*bazaarPackage {
bazaarIndexLock.Lock()
defer bazaarIndexLock.Unlock()
now := time.Now().Unix()
if 3600 >= now-bazaarIndexCacheTime {
return cachedBazaarIndex
}
request := httpclient.NewBrowserRequest()
u := util.BazaarStatServer + "/bazaar/index.json"
resp, reqErr := request.SetSuccessResult(&cachedBazaarIndex).Get(u)
resp, reqErr := request.SetContext(ctx).SetSuccessResult(&cachedBazaarIndex).Get(u)
if nil != reqErr {
logging.LogErrorf("get bazaar index [%s] failed: %s", u, reqErr)
return cachedBazaarIndex
@ -705,19 +797,18 @@ func getBazaarIndex() map[string]*bazaarPackage {
logging.LogErrorf("get bazaar index [%s] failed: %d", u, resp.StatusCode)
return cachedBazaarIndex
}
bazaarIndexCacheTime = now
bazaarIndexCacheTime = time.Now().Unix()
return cachedBazaarIndex
}
// defaultMinAppVersion 如果集市包中缺失 minAppVersion 项，则使用该值作为最低支持的版本号，小于该版本号时不显示集市包
// Add marketplace package config item `minAppVersion` https://github.com/siyuan-note/siyuan/issues/8330
const defaultMinAppVersion = "2.9.0"
func disallowInstallBazaarPackage(pkg *Package) bool {
// 如果包没有指定 minAppVersion，则允许安装
if "" == pkg.MinAppVersion {
pkg.MinAppVersion = defaultMinAppVersion
return false
}
// 如果包要求的 minAppVersion 大于当前版本，则不允许安装
if 0 < semver.Compare("v"+pkg.MinAppVersion, "v"+util.Ver) {
return true
}

140
kernel/bazaar/plugin.go
View file

@ -22,11 +22,8 @@ import (
"runtime"
"sort"
"strings"
"sync"
"github.com/88250/go-humanize"
ants "github.com/panjf2000/ants/v2"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -36,103 +33,70 @@ type Plugin struct {
Enabled bool `json:"enabled"`
}
// Plugins 返回集市插件列表
func Plugins(frontend string) (plugins []*Plugin) {
plugins = []*Plugin{}
result := getStageAndBazaar("plugins")
isOnline := isBazzarOnline()
if !isOnline {
if !result.Online {
return
}
if result.StageErr != nil {
return
}
if 1 > len(result.BazaarIndex) {
return
}
stageIndex, err := getStageIndex("plugins")
if err != nil {
return
for _, repo := range result.StageIndex.Repos {
if nil == repo.Package {
continue
}
plugin := buildPluginFromStageRepo(repo, frontend, result.BazaarIndex)
if nil != plugin {
plugins = append(plugins, plugin)
}
}
bazaarIndex := getBazaarIndex()
if 1 > len(bazaarIndex) {
return
}
requestFailed := false
waitGroup := &sync.WaitGroup{}
lock := &sync.Mutex{}
p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) {
defer waitGroup.Done()
repo := arg.(*StageRepo)
repoURL := repo.URL
if pkg, found := packageCache.Get(repoURL); found {
lock.Lock()
plugins = append(plugins, pkg.(*Plugin))
lock.Unlock()
return
}
if requestFailed {
return
}
plugin := &Plugin{}
innerU := util.BazaarOSSServer + "/package/" + repoURL + "/plugin.json"
innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(plugin).Get(innerU)
if nil != innerErr {
logging.LogErrorf("get bazaar package [%s] failed: %s", repoURL, innerErr)
requestFailed = true
return
}
if 200 != innerResp.StatusCode {
logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode)
requestFailed = true
return
}
plugin.DisallowInstall = disallowInstallBazaarPackage(plugin.Package)
plugin.DisallowUpdate = disallowInstallBazaarPackage(plugin.Package)
plugin.UpdateRequiredMinAppVer = plugin.MinAppVersion
plugin.Incompatible = isIncompatiblePlugin(plugin, frontend)
plugin.URL = strings.TrimSuffix(plugin.URL, "/")
repoURLHash := strings.Split(repoURL, "@")
plugin.RepoURL = "https://github.com/" + repoURLHash[0]
plugin.RepoHash = repoURLHash[1]
plugin.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim"
plugin.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232"
plugin.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png"
plugin.Funding = repo.Package.Funding
plugin.PreferredFunding = getPreferredFunding(plugin.Funding)
plugin.PreferredName = GetPreferredName(plugin.Package)
plugin.PreferredDesc = getPreferredDesc(plugin.Description)
plugin.Updated = repo.Updated
plugin.Stars = repo.Stars
plugin.OpenIssues = repo.OpenIssues
plugin.Size = repo.Size
plugin.HSize = humanize.BytesCustomCeil(uint64(plugin.Size), 2)
plugin.InstallSize = repo.InstallSize
plugin.HInstallSize = humanize.BytesCustomCeil(uint64(plugin.InstallSize), 2)
packageInstallSizeCache.SetDefault(plugin.RepoURL, plugin.InstallSize)
plugin.HUpdated = formatUpdated(plugin.Updated)
pkg := bazaarIndex[strings.Split(repoURL, "@")[0]]
if nil != pkg {
plugin.Downloads = pkg.Downloads
}
lock.Lock()
plugins = append(plugins, plugin)
lock.Unlock()
packageCache.SetDefault(repoURL, plugin)
})
for _, repo := range stageIndex.Repos {
waitGroup.Add(1)
p.Invoke(repo)
}
waitGroup.Wait()
p.Release()
sort.Slice(plugins, func(i, j int) bool { return plugins[i].Updated > plugins[j].Updated })
return
}
// buildPluginFromStageRepo 使用 stage 内嵌的 package 构建 *Plugin，不发起 HTTP 请求。
func buildPluginFromStageRepo(repo *StageRepo, frontend string, bazaarIndex map[string]*bazaarPackage) *Plugin {
pkg := *repo.Package
pkg.URL = strings.TrimSuffix(pkg.URL, "/")
repoURLHash := strings.Split(repo.URL, "@")
if 2 != len(repoURLHash) {
return nil
}
pkg.RepoURL = "https://github.com/" + repoURLHash[0]
pkg.RepoHash = repoURLHash[1]
pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim"
pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232"
pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png"
pkg.Updated = repo.Updated
pkg.Stars = repo.Stars
pkg.OpenIssues = repo.OpenIssues
pkg.Size = repo.Size
pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2)
pkg.InstallSize = repo.InstallSize
pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2)
pkg.HUpdated = formatUpdated(pkg.Updated)
pkg.PreferredFunding = getPreferredFunding(pkg.Funding)
pkg.PreferredName = GetPreferredName(&pkg)
pkg.PreferredDesc = getPreferredDesc(pkg.Description)
pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg)
pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg)
pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion
pkg.Incompatible = isIncompatiblePlugin(&Plugin{Package: &pkg}, frontend)
if bp := bazaarIndex[repoURLHash[0]]; nil != bp {
pkg.Downloads = bp.Downloads
}
packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize)
return &Plugin{Package: &pkg}
}
func ParseInstalledPlugin(name, frontend string) (found bool, displayName string, incompatible, disabledInPublish, disallowInstall bool) {
pluginsPath := filepath.Join(util.DataDir, "plugins")
if !util.IsPathRegularDirOrSymlinkDir(pluginsPath) {

138
kernel/bazaar/template.go
View file

@ -21,12 +21,9 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/88250/go-humanize"
"github.com/panjf2000/ants/v2"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -35,97 +32,30 @@ type Template struct {
*Package
}
// Templates 返回集市模板列表
func Templates() (templates []*Template) {
templates = []*Template{}
result := getStageAndBazaar("templates")
isOnline := isBazzarOnline()
if !isOnline {
if !result.Online {
return
}
if result.StageErr != nil {
return
}
if 1 > len(result.BazaarIndex) {
return
}
stageIndex, err := getStageIndex("templates")
if err != nil {
return
for _, repo := range result.StageIndex.Repos {
if nil == repo.Package {
continue
}
template := buildTemplateFromStageRepo(repo, result.BazaarIndex)
if nil != template {
templates = append(templates, template)
}
}
bazaarIndex := getBazaarIndex()
if 1 > len(bazaarIndex) {
return
}
requestFailed := false
waitGroup := &sync.WaitGroup{}
lock := &sync.Mutex{}
p, _ := ants.NewPoolWithFunc(2, func(arg interface{}) {
defer waitGroup.Done()
repo := arg.(*StageRepo)
repoURL := repo.URL
if pkg, found := packageCache.Get(repoURL); found {
lock.Lock()
templates = append(templates, pkg.(*Template))
lock.Unlock()
return
}
if requestFailed {
return
}
template := &Template{}
innerU := util.BazaarOSSServer + "/package/" + repoURL + "/template.json"
innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(template).Get(innerU)
if nil != innerErr {
logging.LogErrorf("get community template [%s] failed: %s", repoURL, innerErr)
requestFailed = true
return
}
if 200 != innerResp.StatusCode {
logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode)
requestFailed = true
return
}
template.DisallowInstall = disallowInstallBazaarPackage(template.Package)
template.DisallowUpdate = disallowInstallBazaarPackage(template.Package)
template.UpdateRequiredMinAppVer = template.MinAppVersion
template.URL = strings.TrimSuffix(template.URL, "/")
repoURLHash := strings.Split(repoURL, "@")
template.RepoURL = "https://github.com/" + repoURLHash[0]
template.RepoHash = repoURLHash[1]
template.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim"
template.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232"
template.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png"
template.Funding = repo.Package.Funding
template.PreferredFunding = getPreferredFunding(template.Funding)
template.PreferredName = GetPreferredName(template.Package)
template.PreferredDesc = getPreferredDesc(template.Description)
template.Updated = repo.Updated
template.Stars = repo.Stars
template.OpenIssues = repo.OpenIssues
template.Size = repo.Size
template.HSize = humanize.BytesCustomCeil(uint64(template.Size), 2)
template.InstallSize = repo.InstallSize
template.HInstallSize = humanize.BytesCustomCeil(uint64(template.InstallSize), 2)
packageInstallSizeCache.SetDefault(template.RepoURL, template.InstallSize)
template.HUpdated = formatUpdated(template.Updated)
pkg := bazaarIndex[strings.Split(repoURL, "@")[0]]
if nil != pkg {
template.Downloads = pkg.Downloads
}
lock.Lock()
templates = append(templates, template)
lock.Unlock()
packageCache.SetDefault(repoURL, template)
})
for _, repo := range stageIndex.Repos {
waitGroup.Add(1)
p.Invoke(repo)
}
waitGroup.Wait()
p.Release()
templates = filterLegacyTemplates(templates)
@ -133,6 +63,40 @@ func Templates() (templates []*Template) {
return
}
// buildTemplateFromStageRepo 使用 stage 内嵌的 package 构建 *Template，不发起 HTTP 请求。
func buildTemplateFromStageRepo(repo *StageRepo, bazaarIndex map[string]*bazaarPackage) *Template {
pkg := *repo.Package
pkg.URL = strings.TrimSuffix(pkg.URL, "/")
repoURLHash := strings.Split(repo.URL, "@")
if 2 != len(repoURLHash) {
return nil
}
pkg.RepoURL = "https://github.com/" + repoURLHash[0]
pkg.RepoHash = repoURLHash[1]
pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim"
pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232"
pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png"
pkg.Updated = repo.Updated
pkg.Stars = repo.Stars
pkg.OpenIssues = repo.OpenIssues
pkg.Size = repo.Size
pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2)
pkg.InstallSize = repo.InstallSize
pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2)
pkg.HUpdated = formatUpdated(pkg.Updated)
pkg.PreferredFunding = getPreferredFunding(pkg.Funding)
pkg.PreferredName = GetPreferredName(&pkg)
pkg.PreferredDesc = getPreferredDesc(pkg.Description)
pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg)
pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg)
pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion
if bp := bazaarIndex[repoURLHash[0]]; nil != bp {
pkg.Downloads = bp.Downloads
}
packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize)
return &Template{Package: &pkg}
}
func InstalledTemplates() (ret []*Template) {
ret = []*Template{}

139
kernel/bazaar/theme.go
View file

@ -21,11 +21,8 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"github.com/88250/go-humanize"
ants "github.com/panjf2000/ants/v2"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -36,102 +33,70 @@ type Theme struct {
Modes []string `json:"modes"`
}
// Themes 返回集市主题列表
func Themes() (ret []*Theme) {
ret = []*Theme{}
result := getStageAndBazaar("themes")
isOnline := isBazzarOnline()
if !isOnline {
if !result.Online {
return
}
if result.StageErr != nil {
return
}
if 1 > len(result.BazaarIndex) {
return
}
stageIndex, err := getStageIndex("themes")
if err != nil {
return
for _, repo := range result.StageIndex.Repos {
if nil == repo.Package {
continue
}
theme := buildThemeFromStageRepo(repo, result.BazaarIndex)
if nil != theme {
ret = append(ret, theme)
}
}
bazaarIndex := getBazaarIndex()
if 1 > len(bazaarIndex) {
return
}
requestFailed := false
waitGroup := &sync.WaitGroup{}
lock := &sync.Mutex{}
p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) {
defer waitGroup.Done()
repo := arg.(*StageRepo)
repoURL := repo.URL
if pkg, found := packageCache.Get(repoURL); found {
lock.Lock()
ret = append(ret, pkg.(*Theme))
lock.Unlock()
return
}
if requestFailed {
return
}
theme := &Theme{}
innerU := util.BazaarOSSServer + "/package/" + repoURL + "/theme.json"
innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(theme).Get(innerU)
if nil != innerErr {
logging.LogErrorf("get bazaar package [%s] failed: %s", innerU, innerErr)
requestFailed = true
return
}
if 200 != innerResp.StatusCode {
logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode)
requestFailed = true
return
}
theme.DisallowInstall = disallowInstallBazaarPackage(theme.Package)
theme.DisallowUpdate = disallowInstallBazaarPackage(theme.Package)
theme.UpdateRequiredMinAppVer = theme.MinAppVersion
theme.URL = strings.TrimSuffix(theme.URL, "/")
repoURLHash := strings.Split(repoURL, "@")
theme.RepoURL = "https://github.com/" + repoURLHash[0]
theme.RepoHash = repoURLHash[1]
theme.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim"
theme.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232"
theme.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png"
theme.Funding = repo.Package.Funding
theme.PreferredFunding = getPreferredFunding(theme.Funding)
theme.PreferredName = GetPreferredName(theme.Package)
theme.PreferredDesc = getPreferredDesc(theme.Description)
theme.Updated = repo.Updated
theme.Stars = repo.Stars
theme.OpenIssues = repo.OpenIssues
theme.Size = repo.Size
theme.HSize = humanize.BytesCustomCeil(uint64(theme.Size), 2)
theme.InstallSize = repo.InstallSize
theme.HInstallSize = humanize.BytesCustomCeil(uint64(theme.InstallSize), 2)
packageInstallSizeCache.SetDefault(theme.RepoURL, theme.InstallSize)
theme.HUpdated = formatUpdated(theme.Updated)
pkg := bazaarIndex[strings.Split(repoURL, "@")[0]]
if nil != pkg {
theme.Downloads = pkg.Downloads
}
lock.Lock()
ret = append(ret, theme)
lock.Unlock()
packageCache.SetDefault(repoURL, theme)
})
for _, repo := range stageIndex.Repos {
waitGroup.Add(1)
p.Invoke(repo)
}
waitGroup.Wait()
p.Release()
sort.Slice(ret, func(i, j int) bool { return ret[i].Updated > ret[j].Updated })
return
}
// buildThemeFromStageRepo 使用 stage 内嵌的 package 构建 *Theme，不发起 HTTP 请求。
func buildThemeFromStageRepo(repo *StageRepo, bazaarIndex map[string]*bazaarPackage) *Theme {
pkg := *repo.Package
pkg.URL = strings.TrimSuffix(pkg.URL, "/")
repoURLHash := strings.Split(repo.URL, "@")
if 2 != len(repoURLHash) {
return nil
}
pkg.RepoURL = "https://github.com/" + repoURLHash[0]
pkg.RepoHash = repoURLHash[1]
pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim"
pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232"
pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png"
pkg.Updated = repo.Updated
pkg.Stars = repo.Stars
pkg.OpenIssues = repo.OpenIssues
pkg.Size = repo.Size
pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2)
pkg.InstallSize = repo.InstallSize
pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2)
pkg.HUpdated = formatUpdated(pkg.Updated)
pkg.PreferredFunding = getPreferredFunding(pkg.Funding)
pkg.PreferredName = GetPreferredName(&pkg)
pkg.PreferredDesc = getPreferredDesc(pkg.Description)
pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg)
pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg)
pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion
if bp := bazaarIndex[repoURLHash[0]]; nil != bp {
pkg.Downloads = bp.Downloads
}
packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize)
theme := &Theme{Package: &pkg, Modes: []string{}}
return theme
}
func InstalledThemes() (ret []*Theme) {
ret = []*Theme{}

138
kernel/bazaar/widget.go
View file

@ -21,11 +21,8 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"github.com/88250/go-humanize"
ants "github.com/panjf2000/ants/v2"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -34,102 +31,69 @@ type Widget struct {
*Package
}
// Widgets 返回集市挂件列表
func Widgets() (widgets []*Widget) {
widgets = []*Widget{}
result := getStageAndBazaar("widgets")
isOnline := isBazzarOnline()
if !isOnline {
if !result.Online {
return
}
if result.StageErr != nil {
return
}
if 1 > len(result.BazaarIndex) {
return
}
stageIndex, err := getStageIndex("widgets")
if err != nil {
return
for _, repo := range result.StageIndex.Repos {
if nil == repo.Package {
continue
}
widget := buildWidgetFromStageRepo(repo, result.BazaarIndex)
if nil != widget {
widgets = append(widgets, widget)
}
}
bazaarIndex := getBazaarIndex()
if 1 > len(bazaarIndex) {
return
}
requestFailed := false
waitGroup := &sync.WaitGroup{}
lock := &sync.Mutex{}
p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) {
defer waitGroup.Done()
repo := arg.(*StageRepo)
repoURL := repo.URL
if pkg, found := packageCache.Get(repoURL); found {
lock.Lock()
widgets = append(widgets, pkg.(*Widget))
lock.Unlock()
return
}
if requestFailed {
return
}
widget := &Widget{}
innerU := util.BazaarOSSServer + "/package/" + repoURL + "/widget.json"
innerResp, innerErr := httpclient.NewBrowserRequest().SetSuccessResult(widget).Get(innerU)
if nil != innerErr {
logging.LogErrorf("get bazaar package [%s] failed: %s", repoURL, innerErr)
requestFailed = true
return
}
if 200 != innerResp.StatusCode {
logging.LogErrorf("get bazaar package [%s] failed: %d", innerU, innerResp.StatusCode)
requestFailed = true
return
}
widget.DisallowInstall = disallowInstallBazaarPackage(widget.Package)
widget.DisallowUpdate = disallowInstallBazaarPackage(widget.Package)
widget.UpdateRequiredMinAppVer = widget.MinAppVersion
widget.URL = strings.TrimSuffix(widget.URL, "/")
repoURLHash := strings.Split(repoURL, "@")
widget.RepoURL = "https://github.com/" + repoURLHash[0]
widget.RepoHash = repoURLHash[1]
widget.PreviewURL = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageslim"
widget.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repoURL + "/preview.png?imageView2/2/w/436/h/232"
widget.IconURL = util.BazaarOSSServer + "/package/" + repoURL + "/icon.png"
widget.Funding = repo.Package.Funding
widget.PreferredFunding = getPreferredFunding(widget.Funding)
widget.PreferredName = GetPreferredName(widget.Package)
widget.PreferredDesc = getPreferredDesc(widget.Description)
widget.Updated = repo.Updated
widget.Stars = repo.Stars
widget.OpenIssues = repo.OpenIssues
widget.Size = repo.Size
widget.HSize = humanize.BytesCustomCeil(uint64(widget.Size), 2)
widget.InstallSize = repo.InstallSize
widget.HInstallSize = humanize.BytesCustomCeil(uint64(widget.InstallSize), 2)
packageInstallSizeCache.SetDefault(widget.RepoURL, widget.InstallSize)
widget.HUpdated = formatUpdated(widget.Updated)
pkg := bazaarIndex[strings.Split(repoURL, "@")[0]]
if nil != pkg {
widget.Downloads = pkg.Downloads
}
lock.Lock()
widgets = append(widgets, widget)
lock.Unlock()
packageCache.SetDefault(repoURL, widget)
})
for _, repo := range stageIndex.Repos {
waitGroup.Add(1)
p.Invoke(repo)
}
waitGroup.Wait()
p.Release()
sort.Slice(widgets, func(i, j int) bool { return widgets[i].Updated > widgets[j].Updated })
return
}
// buildWidgetFromStageRepo 使用 stage 内嵌的 package 构建 *Widget，不发起 HTTP 请求。
func buildWidgetFromStageRepo(repo *StageRepo, bazaarIndex map[string]*bazaarPackage) *Widget {
pkg := *repo.Package
pkg.URL = strings.TrimSuffix(pkg.URL, "/")
repoURLHash := strings.Split(repo.URL, "@")
if 2 != len(repoURLHash) {
return nil
}
pkg.RepoURL = "https://github.com/" + repoURLHash[0]
pkg.RepoHash = repoURLHash[1]
pkg.PreviewURL = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageslim"
pkg.PreviewURLThumb = util.BazaarOSSServer + "/package/" + repo.URL + "/preview.png?imageView2/2/w/436/h/232"
pkg.IconURL = util.BazaarOSSServer + "/package/" + repo.URL + "/icon.png"
pkg.Updated = repo.Updated
pkg.Stars = repo.Stars
pkg.OpenIssues = repo.OpenIssues
pkg.Size = repo.Size
pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2)
pkg.InstallSize = repo.InstallSize
pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2)
pkg.HUpdated = formatUpdated(pkg.Updated)
pkg.PreferredFunding = getPreferredFunding(pkg.Funding)
pkg.PreferredName = GetPreferredName(&pkg)
pkg.PreferredDesc = getPreferredDesc(pkg.Description)
pkg.DisallowInstall = disallowInstallBazaarPackage(&pkg)
pkg.DisallowUpdate = disallowInstallBazaarPackage(&pkg)
pkg.UpdateRequiredMinAppVer = pkg.MinAppVersion
if bp := bazaarIndex[repoURLHash[0]]; nil != bp {
pkg.Downloads = bp.Downloads
}
packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize)
return &Widget{Package: &pkg}
}
func InstalledWidgets() (ret []*Widget) {
ret = []*Widget{}