diff --git a/kernel/bazaar/bazaar.go b/kernel/bazaar/bazaar.go index 3db03b1d0..b8b6734d2 100644 --- a/kernel/bazaar/bazaar.go +++ b/kernel/bazaar/bazaar.go @@ -35,7 +35,7 @@ func GetBazaarPackages(pkgType string, frontend string) (packages []*Package) { packages = make([]*Package, 0, len(result.StageIndex.Repos)) for _, repo := range result.StageIndex.Repos { - pkg := buildBazaarPackageWithMetadata(repo, result.BazaarIndex, pkgType, frontend) + pkg := buildBazaarPackageWithMetadata(repo, result.BazaarStats, pkgType, frontend) if nil == pkg { continue } @@ -45,7 +45,7 @@ func GetBazaarPackages(pkgType string, frontend string) (packages []*Package) { } // buildBazaarPackageWithMetadata 从 StageRepo 构建带有在线元数据的集市包。 -func buildBazaarPackageWithMetadata(repo *StageRepo, bazaarIndex map[string]*bazaarPackage, pkgType string, frontend string) *Package { +func buildBazaarPackageWithMetadata(repo *StageRepo, bazaarStats map[string]*bazaarStats, pkgType string, frontend string) *Package { if nil == repo || nil == repo.Package { return nil } @@ -85,8 +85,8 @@ func buildBazaarPackageWithMetadata(repo *StageRepo, bazaarIndex map[string]*baz pkg.HSize = humanize.BytesCustomCeil(uint64(pkg.Size), 2) pkg.InstallSize = repo.InstallSize pkg.HInstallSize = humanize.BytesCustomCeil(uint64(pkg.InstallSize), 2) - if bp := bazaarIndex[repoURLHash[0]]; nil != bp { - pkg.Downloads = bp.Downloads + if stats := bazaarStats[repoURLHash[0]]; nil != stats { // 通过 bazaarStats[owner/repo] 获取单个包的统计数据 + pkg.Downloads = stats.Downloads } packageInstallSizeCache.SetDefault(pkg.RepoURL, pkg.InstallSize) return &pkg diff --git a/kernel/bazaar/package.go b/kernel/bazaar/package.go index 78efb8f65..295359a27 100644 --- a/kernel/bazaar/package.go +++ b/kernel/bazaar/package.go @@ -40,7 +40,7 @@ type Funding struct { } // Package 描述了集市包元数据和传递给前端的其他信息。 -// - 集市包新增元数据字段需要同步修改 bazaar 的工作流，参考 https://github.com/siyuan-note/bazaar/commit/aa36d0003139c52d8e767c6e18a635be006323e2 +// - 集市包新增元数据字段需要同步修改 bazaar 的工作流，参考 https://github.com/siyuan-note/bazaar/commit/aa36d0003139c52d8e767c6e18a635be006323e2 type Package struct { Author string `json:"author"` URL string `json:"url"` @@ -104,7 +104,7 @@ type StageRepo struct { type StageIndex struct { Repos []*StageRepo `json:"repos"` - reposByURL map[string]*StageRepo // 不序列化，首次按 URL 查找时懒构建 + reposByURL map[string]*StageRepo // 不序列化，首次按 URL 查找时懒构建，随整份索引一起过期 reposOnce sync.Once } diff --git a/kernel/bazaar/readme.go b/kernel/bazaar/readme.go index 82a709ffb..e69821d00 100644 --- a/kernel/bazaar/readme.go +++ b/kernel/bazaar/readme.go @@ -45,28 +45,10 @@ func getReadmeFileCandidates(readme LocaleStrings) []string { } // GetBazaarPackageREADME 获取集市包的在线 README。 -func GetBazaarPackageREADME(ctx context.Context, repoURL, repoHash, packageType string) (ret string) { +func GetBazaarPackageREADME(ctx context.Context, repoURL, repoHash, pkgType string) (ret string) { repoURLHash := repoURL + "@" + repoHash - - stageIndexLock.RLock() - stageIndex := cachedStageIndex[packageType] - stageIndexLock.RUnlock() - if stageIndex == nil { - var err error - stageIndex, err = getStageIndex(ctx, packageType) - if err != nil { - return - } - } - url := strings.TrimPrefix(repoURLHash, "https://github.com/") - var repo *StageRepo - for _, r := range stageIndex.Repos { - if r.URL == url { - repo = r - break - } - } + repo := getStageRepoByURL(ctx, pkgType, url) if repo == nil || repo.Package == nil { return } @@ -87,6 +69,7 @@ func GetBazaarPackageREADME(ctx context.Context, repoURL, repoHash, packageType return } + // 解码 UTF-16 BOM if len(data) > 2 { var decoded []byte var err error diff --git a/kernel/bazaar/stage.go b/kernel/bazaar/stage.go index 2f66b8302..0477dcbbe 100644 --- a/kernel/bazaar/stage.go +++ b/kernel/bazaar/stage.go @@ -22,25 +22,25 @@ import ( "sync" "time" + gcache "github.com/patrickmn/go-cache" "github.com/siyuan-note/httpclient" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" "golang.org/x/sync/singleflight" ) -var cachedStageIndex = map[string]*StageIndex{} -var stageIndexCacheTime int64 -var stageIndexLock = sync.RWMutex{} +// cachedStageIndex 缓存 stage 索引 +var cachedStageIndex = gcache.New(time.Duration(util.RhyCacheDuration)*time.Second, time.Duration(util.RhyCacheDuration)*time.Second/6) type StageBazaarResult struct { - StageIndex *StageIndex // stage 索引 - BazaarIndex map[string]*bazaarPackage // bazaar 索引 - Online bool // online 状态 - StageErr error // stage 错误 + StageIndex *StageIndex // stage 索引 + BazaarStats map[string]*bazaarStats // 统计信息 + Online bool // online 状态 + StageErr error // stage 错误 } var stageBazaarFlight singleflight.Group -var onlineCheckFlight singleflight.Group +var bazaarStatsFlight singleflight.Group // getStageAndBazaar 获取 stage 索引和 bazaar 索引，相同 pkgType 的并发调用会合并为一次实际请求 (single-flight) func getStageAndBazaar(pkgType string) (result StageBazaarResult) { @@ -58,12 +58,12 @@ func getStageAndBazaar(pkgType string) (result StageBazaarResult) { // getStageAndBazaar0 执行一次 stage 和 bazaar 索引拉取 func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { stageIndex, stageErr := getStageIndexFromCache(pkgType) - bazaarIndex := getBazaarIndexFromCache() - if nil != stageIndex && nil != bazaarIndex { + bazaarStats := getBazaarStatsFromCache() + if nil != stageIndex && nil != bazaarStats { // 两者都从缓存返回，不需要 online 检查 return StageBazaarResult{ StageIndex: stageIndex, - BazaarIndex: bazaarIndex, + BazaarStats: bazaarStats, Online: true, StageErr: stageErr, } @@ -77,7 +77,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { wg.Add(3) go func() { defer wg.Done() - onlineResult = isBazzarOnline() + onlineResult = isBazaarOnline() onlineDone <- true }() go func() { @@ -86,7 +86,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { }() go func() { defer wg.Done() - bazaarIndex = getBazaarIndex(ctx) + bazaarStats = getBazaarStats(ctx) }() <-onlineDone @@ -95,7 +95,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { cancel() return StageBazaarResult{ StageIndex: stageIndex, - BazaarIndex: bazaarIndex, + BazaarStats: bazaarStats, Online: false, StageErr: stageErr, } @@ -106,7 +106,7 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { return StageBazaarResult{ StageIndex: stageIndex, - BazaarIndex: bazaarIndex, + BazaarStats: bazaarStats, Online: onlineResult, StageErr: stageErr, } @@ -114,12 +114,8 @@ func getStageAndBazaar0(pkgType string) (result StageBazaarResult) { // getStageIndexFromCache 仅从缓存获取 stage 索引，过期或无缓存时返回 nil func getStageIndexFromCache(pkgType string) (ret *StageIndex, err error) { - stageIndexLock.RLock() - cacheTime := stageIndexCacheTime - cached := cachedStageIndex[pkgType] - stageIndexLock.RUnlock() - if util.RhyCacheDuration >= time.Now().Unix()-cacheTime && nil != cached { - ret = cached + if val, found := cachedStageIndex.Get(pkgType); found { + ret = val.(*StageIndex) } return } @@ -138,13 +134,10 @@ func getStageIndex(ctx context.Context, pkgType string) (ret *StageIndex, err er return } - stageIndexLock.Lock() - defer stageIndexLock.Unlock() - bazaarHash := rhyRet["bazaar"].(string) ret = &StageIndex{} request := httpclient.NewBrowserRequest() - u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/" + pkgType + ".json" + u := util.BazaarOSSServer + "/bazaar@" + bazaarHash + "/stage/" + pkgType + ".json" // pkgType 单词为复数形式 resp, reqErr := request.SetContext(ctx).SetSuccessResult(ret).Get(u) if nil != reqErr { logging.LogErrorf("get community stage index [%s] failed: %s", u, reqErr) @@ -157,22 +150,26 @@ func getStageIndex(ctx context.Context, pkgType string) (ret *StageIndex, err er return } - stageIndexCacheTime = time.Now().Unix() - cachedStageIndex[pkgType] = ret + cachedStageIndex.SetDefault(pkgType, ret) return } -func isBazzarOnline() bool { - v, err, _ := onlineCheckFlight.Do("bazaarOnline", func() (interface{}, error) { - return isBazzarOnline0(), nil - }) - if err != nil { - return false +// getStageRepoByURL 根据 pkgType 与 url（owner/repo@hash）获取 StageRepo +func getStageRepoByURL(ctx context.Context, pkgType, url string) *StageRepo { + stageIndex, _ := getStageIndex(ctx, pkgType) + if nil == stageIndex { + return nil } - return v.(bool) + stageIndex.reposOnce.Do(func() { + stageIndex.reposByURL = make(map[string]*StageRepo, len(stageIndex.Repos)) + for _, r := range stageIndex.Repos { + stageIndex.reposByURL[r.URL] = r + } + }) + return stageIndex.reposByURL[url] } -func isBazzarOnline0() (ret bool) { +func isBazaarOnline() (ret bool) { // Improve marketplace loading when offline https://github.com/siyuan-note/siyuan/issues/12050 ret = util.IsOnline(util.BazaarOSSServer+"/204", true, 3000) if !ret { @@ -181,50 +178,47 @@ func isBazzarOnline0() (ret bool) { return } -type bazaarPackage struct { - Name string `json:"name"` - Downloads int `json:"downloads"` +// bazaarStats 集市包统计信息 +type bazaarStats struct { + Downloads int `json:"downloads"` // 下载次数 } -var cachedBazaarIndex = map[string]*bazaarPackage{} -var bazaarIndexCacheTime int64 -var bazaarIndexLock = sync.RWMutex{} +// cachedBazaarStats 缓存集市包统计信息 +var cachedBazaarStats = gcache.New(time.Duration(util.RhyCacheDuration)*time.Second, time.Duration(util.RhyCacheDuration)*time.Second/6) -// getBazaarIndexFromCache 仅从缓存获取 bazaar 索引，过期或无缓存时返回 nil -func getBazaarIndexFromCache() (ret map[string]*bazaarPackage) { - bazaarIndexLock.RLock() - cacheTime := bazaarIndexCacheTime - cached := cachedBazaarIndex - hasData := 0 < len(cached) - bazaarIndexLock.RUnlock() - if util.RhyCacheDuration >= time.Now().Unix()-cacheTime && hasData { - ret = cached - } else { - ret = nil +// getBazaarStatsFromCache 仅从缓存获取集市包统计信息，过期或无缓存时返回 nil +func getBazaarStatsFromCache() (ret map[string]*bazaarStats) { + if val, found := cachedBazaarStats.Get("index"); found { + ret = val.(map[string]*bazaarStats) } return } -// getBazaarIndex 获取 bazaar 索引 -func getBazaarIndex(ctx context.Context) map[string]*bazaarPackage { - if cached := getBazaarIndexFromCache(); nil != cached { +// getBazaarStats 获取集市包统计信息 +func getBazaarStats(ctx context.Context) map[string]*bazaarStats { + if cached := getBazaarStatsFromCache(); nil != cached { return cached } - bazaarIndexLock.Lock() - defer bazaarIndexLock.Unlock() + v, _, _ := bazaarStatsFlight.Do("bazaarStats", func() (interface{}, error) { + return getBazaarStats0(ctx), nil + }) + return v.(map[string]*bazaarStats) +} +func getBazaarStats0(ctx context.Context) map[string]*bazaarStats { + var result map[string]*bazaarStats request := httpclient.NewBrowserRequest() u := util.BazaarStatServer + "/bazaar/index.json" - resp, reqErr := request.SetContext(ctx).SetSuccessResult(&cachedBazaarIndex).Get(u) + resp, reqErr := request.SetContext(ctx).SetSuccessResult(&result).Get(u) if nil != reqErr { - logging.LogErrorf("get bazaar index [%s] failed: %s", u, reqErr) - return cachedBazaarIndex + logging.LogErrorf("get bazaar stats [%s] failed: %s", u, reqErr) + return result } if 200 != resp.StatusCode { - logging.LogErrorf("get bazaar index [%s] failed: %d", u, resp.StatusCode) - return cachedBazaarIndex + logging.LogErrorf("get bazaar stats [%s] failed: %d", u, resp.StatusCode) + return result } - bazaarIndexCacheTime = time.Now().Unix() - return cachedBazaarIndex + cachedBazaarStats.SetDefault("index", result) + return result }