From c852f6f51a7337d5816db2f3a8e0e523411fcd36 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Wed, 9 Jul 2025 16:18:53 +0800 Subject: [PATCH] :zap: Improve the image loading performance in the database https://github.com/siyuan-note/siyuan/issues/15245 --- kernel/go.mod | 1 + kernel/go.sum | 3 ++ kernel/model/asset_content.go | 4 +- kernel/model/assets.go | 69 +++++++++++++++++++++++++++ kernel/model/assets_watcher.go | 8 ++-- kernel/model/assets_watcher_darwin.go | 4 +- kernel/server/serve.go | 26 ++++++++++ kernel/util/file.go | 6 +++ 8 files changed, 113 insertions(+), 8 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 4a6988d8c..0923b520a 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -22,6 +22,7 @@ require ( github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be github.com/denisbrodbeck/machineid v1.0.1 github.com/dgraph-io/ristretto v0.2.0 + github.com/disintegration/imaging v1.6.2 github.com/djherbis/times v1.6.0 github.com/emersion/go-ical v0.0.0-20250609112844-439c63cef608 github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff diff --git a/kernel/go.sum b/kernel/go.sum index 882beb80b..23dfd6203 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -117,6 +117,8 @@ github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dr github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= @@ -460,6 +462,7 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE= golang.org/x/image v0.28.0/go.mod h1:GUJYXtnGKEUgggyzh+Vxt+AviiCcyiwpsl8iQ8MvwGY= diff --git a/kernel/model/asset_content.go b/kernel/model/asset_content.go index b6ad30f5c..bfaf519bf 100644 --- a/kernel/model/asset_content.go +++ b/kernel/model/asset_content.go @@ -292,7 +292,7 @@ func buildAssetContentOrderBy(orderBy int) string { var assetContentSearcher = NewAssetsSearcher() -func RemoveIndexAssetContent(absPath string) { +func removeIndexAssetContent(absPath string) { defer logging.Recover() assetsDir := util.GetDataAssetsAbsPath() @@ -300,7 +300,7 @@ func RemoveIndexAssetContent(absPath string) { sql.DeleteAssetContentsByPathQueue(p) } -func IndexAssetContent(absPath string) { +func indexAssetContent(absPath string) { defer logging.Recover() ext := filepath.Ext(absPath) diff --git a/kernel/model/assets.go b/kernel/model/assets.go index d65e192ba..f455cb902 100644 --- a/kernel/model/assets.go +++ b/kernel/model/assets.go @@ -37,6 +37,7 @@ import ( "github.com/88250/lute/editor" "github.com/88250/lute/html" "github.com/88250/lute/parse" + "github.com/disintegration/imaging" "github.com/gabriel-vasile/mimetype" "github.com/siyuan-note/filelock" "github.com/siyuan-note/httpclient" @@ -50,6 +51,74 @@ import ( "github.com/siyuan-note/siyuan/kernel/util" ) +func HandleAssetsRemoveEvent(assetAbsPath string) { + removeIndexAssetContent(assetAbsPath) + removeAssetThumbnail(assetAbsPath) +} + +func HandleAssetsChangeEvent(assetAbsPath string) { + indexAssetContent(assetAbsPath) + removeAssetThumbnail(assetAbsPath) +} + +func removeAssetThumbnail(assetAbsPath string) { + if util.IsCompressibleAssetImage(assetAbsPath) { + p := filepath.ToSlash(assetAbsPath) + idx := strings.Index(p, "assets/") + if -1 == idx { + return + } + thumbnailPath := filepath.Join(util.TempDir, "thumbnails", "assets", p[idx+7:]) + os.RemoveAll(thumbnailPath) + } +} + +func NeedGenerateAssetsThumbnail(sourceImgPath string) bool { + info, err := os.Stat(sourceImgPath) + if err != nil { + return false + } + if info.IsDir() { + return false + } + return info.Size() > 1024*10 +} + +func GenerateAssetsThumbnail(sourceImgPath, resizedImgPath string) (err error) { + start := time.Now() + img, err := imaging.Open(sourceImgPath) + if err != nil { + return + } + + // 获取原图宽高 + originalWidth := img.Bounds().Dx() + originalHeight := img.Bounds().Dy() + + // 固定最大宽度为 520，计算缩放比例 + maxWidth := 520 + scale := float64(maxWidth) / float64(originalWidth) + + // 按比例计算新的宽高 + newWidth := maxWidth + newHeight := int(float64(originalHeight) * scale) + + // 缩放图片 + resizedImg := imaging.Resize(img, newWidth, newHeight, imaging.Lanczos) + + // 保存缩放后的图片 + err = os.MkdirAll(filepath.Dir(resizedImgPath), 0755) + if err != nil { + return + } + err = imaging.Save(resizedImg, resizedImgPath) + if err != nil { + return + } + logging.LogDebugf("generated thumbnail image [%s] to [%s], cost [%d]ms", sourceImgPath, resizedImgPath, time.Since(start).Milliseconds()) + return +} + func DocImageAssets(rootID string) (ret []string, err error) { tree, err := LoadTreeByBlockID(rootID) if err != nil { diff --git a/kernel/model/assets_watcher.go b/kernel/model/assets_watcher.go index 61191afe6..5a9953d2a 100644 --- a/kernel/model/assets_watcher.go +++ b/kernel/model/assets_watcher.go @@ -75,9 +75,9 @@ func watchAssets() { timer.Reset(time.Millisecond * 100) if lastEvent.Op&fsnotify.Rename == fsnotify.Rename || lastEvent.Op&fsnotify.Write == fsnotify.Write { - IndexAssetContent(lastEvent.Name) + HandleAssetsChangeEvent(lastEvent.Name) } else if lastEvent.Op&fsnotify.Remove == fsnotify.Remove { - RemoveIndexAssetContent(lastEvent.Name) + HandleAssetsRemoveEvent(lastEvent.Name) } case err, ok := <-assetsWatcher.Errors: if !ok { @@ -94,9 +94,9 @@ func watchAssets() { go cache.LoadAssets() if lastEvent.Op&fsnotify.Remove == fsnotify.Remove { - RemoveIndexAssetContent(lastEvent.Name) + HandleAssetsRemoveEvent(lastEvent.Name) } else { - IndexAssetContent(lastEvent.Name) + HandleAssetsChangeEvent(lastEvent.Name) } } } diff --git a/kernel/model/assets_watcher_darwin.go b/kernel/model/assets_watcher_darwin.go index f5d39e378..32e620e22 100644 --- a/kernel/model/assets_watcher_darwin.go +++ b/kernel/model/assets_watcher_darwin.go @@ -61,9 +61,9 @@ func watchAssets() { go cache.LoadAssets() if watcher.Remove == event.Op { - RemoveIndexAssetContent(event.Path) + HandleAssetsRemoveEvent(event.Path) } else { - IndexAssetContent(event.Path) + HandleAssetsChangeEvent(event.Path) } case err, ok := <-assetsWatcher.Error: if !ok { diff --git a/kernel/server/serve.go b/kernel/server/serve.go index 8de6fc91a..a1eb07580 100644 --- a/kernel/server/serve.go +++ b/kernel/server/serve.go @@ -491,9 +491,17 @@ func serveAssets(ginServer *gin.Engine) { return } } + + if serveThumbnail(context, p, requestPath) { + // 如果请求缩略图服务成功则返回 + return + } + + // 返回原始文件 http.ServeFile(context.Writer, context.Request, p) return }) + ginServer.GET("/history/*path", model.CheckAuth, model.CheckAdminRole, func(context *gin.Context) { p := filepath.Join(util.HistoryDir, context.Param("path")) http.ServeFile(context.Writer, context.Request, p) @@ -501,6 +509,24 @@ func serveAssets(ginServer *gin.Engine) { }) } +func serveThumbnail(context *gin.Context, assetAbsPath, requestPath string) bool { + if style := context.Query("style"); style == "thumb" && model.NeedGenerateAssetsThumbnail(assetAbsPath) { // 请求缩略图 + thumbnailPath := filepath.Join(util.TempDir, "thumbnails", "assets", requestPath) + if !gulu.File.IsExist(thumbnailPath) { + // 如果缩略图不存在，则生成缩略图 + err := model.GenerateAssetsThumbnail(assetAbsPath, thumbnailPath) + if err != nil { + logging.LogErrorf("generate thumbnail failed: %s", err) + return false + } + } + + http.ServeFile(context.Writer, context.Request, thumbnailPath) + return true + } + return false +} + func serveRepoDiff(ginServer *gin.Engine) { ginServer.GET("/repo/diff/*path", model.CheckAuth, model.CheckAdminRole, func(context *gin.Context) { requestPath := context.Param("path") diff --git a/kernel/util/file.go b/kernel/util/file.go index cd6d08102..1193700aa 100644 --- a/kernel/util/file.go +++ b/kernel/util/file.go @@ -300,6 +300,12 @@ func IsSubPath(absPath, toCheckPath string) bool { return false } +func IsCompressibleAssetImage(p string) bool { + lowerName := strings.ToLower(p) + return strings.HasPrefix(lowerName, "assets/") && + (strings.HasSuffix(lowerName, ".png") || strings.HasSuffix(lowerName, ".jpg") || strings.HasSuffix(lowerName, ".jpeg")) +} + func SizeOfDirectory(path string) (size int64, err error) { err = filelock.Walk(path, func(path string, d fs.DirEntry, err error) error { if err != nil {