// SiYuan - Refactor your thinking
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
package model
import (
"bytes"
"errors"
"fmt"
"io/fs"
"mime"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"sort"
"strings"
"time"
"github.com/88250/go-humanize"
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/88250/lute/editor"
"github.com/88250/lute/html"
"github.com/88250/lute/parse"
"github.com/disintegration/imaging"
"github.com/gabriel-vasile/mimetype"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/av"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/search"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
func HandleAssetsRemoveEvent(assetAbsPath string) {
removeIndexAssetContent(assetAbsPath)
removeAssetThumbnail(assetAbsPath)
}
func HandleAssetsChangeEvent(assetAbsPath string) {
indexAssetContent(assetAbsPath)
removeAssetThumbnail(assetAbsPath)
}
func removeAssetThumbnail(assetAbsPath string) {
if util.IsCompressibleAssetImage(assetAbsPath) {
p := filepath.ToSlash(assetAbsPath)
idx := strings.Index(p, "assets/")
if -1 == idx {
return
}
thumbnailPath := filepath.Join(util.TempDir, "thumbnails", "assets", p[idx+7:])
os.RemoveAll(thumbnailPath)
}
}
func NeedGenerateAssetsThumbnail(sourceImgPath string) bool {
info, err := os.Stat(sourceImgPath)
if err != nil {
return false
}
if info.IsDir() {
return false
}
return info.Size() > 1024*1024
}
func GenerateAssetsThumbnail(sourceImgPath, resizedImgPath string) (err error) {
start := time.Now()
img, err := imaging.Open(sourceImgPath)
if err != nil {
return
}
// 获取原图宽高
originalWidth := img.Bounds().Dx()
originalHeight := img.Bounds().Dy()
// 固定最大宽度为 520,计算缩放比例
maxWidth := 520
scale := float64(maxWidth) / float64(originalWidth)
// 按比例计算新的宽高
newWidth := maxWidth
newHeight := int(float64(originalHeight) * scale)
// 缩放图片
resizedImg := imaging.Resize(img, newWidth, newHeight, imaging.Lanczos)
// 保存缩放后的图片
err = os.MkdirAll(filepath.Dir(resizedImgPath), 0755)
if err != nil {
return
}
err = imaging.Save(resizedImg, resizedImgPath)
if err != nil {
return
}
logging.LogDebugf("generated thumbnail image [%s] to [%s], cost [%d]ms", sourceImgPath, resizedImgPath, time.Since(start).Milliseconds())
return
}
func DocImageAssets(rootID string) (ret []string, err error) {
tree, err := LoadTreeByBlockID(rootID)
if err != nil {
return
}
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeImage == n.Type {
linkDest := n.ChildByType(ast.NodeLinkDest)
dest := linkDest.Tokens
if 1 > len(dest) { // 双击打开图片不对 https://github.com/siyuan-note/siyuan/issues/5876
return ast.WalkContinue
}
ret = append(ret, gulu.Str.FromBytes(dest))
}
return ast.WalkContinue
})
return
}
func DocAssets(rootID string) (ret []string, err error) {
tree, err := LoadTreeByBlockID(rootID)
if err != nil {
return
}
ret = assetsLinkDestsInTree(tree)
return
}
func NetAssets2LocalAssets(rootID string, onlyImg bool, originalURL string) (err error) {
tree, err := LoadTreeByBlockID(rootID)
if err != nil {
return
}
var files int
msgId := gulu.Rand.String(7)
docDirLocalPath := filepath.Join(util.DataDir, tree.Box, path.Dir(tree.Path))
assetsDirPath := getAssetsDir(filepath.Join(util.DataDir, tree.Box), docDirLocalPath)
if !gulu.File.IsExist(assetsDirPath) {
if err = os.MkdirAll(assetsDirPath, 0755); err != nil {
return
}
}
browserClient := util.NewCustomReqClient() // 自定义了 TLS 指纹,增加下载成功率
forbiddenCount := 0
destNodes := getRemoteAssetsLinkDestsInTree(tree, onlyImg)
for _, destNode := range destNodes {
dests := getRemoteAssetsLinkDests(destNode, onlyImg)
if 1 > len(dests) {
continue
}
for _, dest := range dests {
if strings.HasPrefix(strings.ToLower(dest), "file://") { // 处理本地文件链接
u := dest[7:]
unescaped, _ := url.PathUnescape(u)
if unescaped != u {
// `Convert network images/assets to local` supports URL-encoded local file names https://github.com/siyuan-note/siyuan/issues/9929
u = unescaped
}
if strings.Contains(u, ":") {
u = strings.TrimPrefix(u, "/")
}
if strings.Contains(u, "?") {
u = u[:strings.Index(u, "?")]
}
if !gulu.File.IsExist(u) || gulu.File.IsDir(u) {
continue
}
name := filepath.Base(u)
name = util.FilterUploadFileName(name)
name = "network-asset-" + name
name = util.AssetName(name)
writePath := filepath.Join(assetsDirPath, name)
if err = filelock.Copy(u, writePath); err != nil {
logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err)
continue
}
setAssetsLinkDest(destNode, dest, "assets/"+name)
files++
continue
}
if strings.HasPrefix(strings.ToLower(dest), "https://") || strings.HasPrefix(strings.ToLower(dest), "http://") || strings.HasPrefix(dest, "//") {
if strings.HasPrefix(dest, "//") {
// `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598
dest = "https:" + dest
}
u := dest
if strings.Contains(u, "qpic.cn") {
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
if strings.Contains(u, "http://") {
u = strings.Replace(u, "http://", "https://", 1)
}
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431
// 下面这部分需要注释掉,否则会导致响应 400
//if strings.HasSuffix(u, "/0") {
// u = strings.Replace(u, "/0", "/640", 1)
//} else if strings.Contains(u, "/0?") {
// u = strings.Replace(u, "/0?", "/640?", 1)
//}
}
displayU := u
if 64 < len(displayU) {
displayU = displayU[:64] + "..."
}
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), displayU), 15000)
request := browserClient.R()
request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second)
if "" != originalURL {
request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464
}
resp, reqErr := request.Get(u)
if nil != reqErr {
logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr)
continue
}
if http.StatusForbidden == resp.StatusCode || http.StatusUnauthorized == resp.StatusCode {
forbiddenCount++
}
if strings.Contains(strings.ToLower(resp.GetContentType()), "text/html") {
// 忽略超链接网页 `Convert network assets to local` no longer process webpage https://github.com/siyuan-note/siyuan/issues/9965
continue
}
if 200 != resp.StatusCode {
logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode)
continue
}
if 1024*1024*96 < resp.ContentLength {
logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength)))
continue
}
data, repErr := resp.ToBytes()
if nil != repErr {
logging.LogErrorf("download network asset [%s] failed: %s", u, repErr)
continue
}
var name string
if strings.Contains(u, "?") {
name = u[:strings.Index(u, "?")]
name = path.Base(name)
} else {
name = path.Base(u)
}
if strings.Contains(name, "#") {
name = name[:strings.Index(name, "#")]
}
name, _ = url.PathUnescape(name)
name = util.FilterUploadFileName(name)
ext := util.Ext(name)
if !util.IsCommonExt(ext) {
if mtype := mimetype.Detect(data); nil != mtype {
ext = mtype.Extension()
name += ext
}
}
if "" == ext && bytes.HasPrefix(data, []byte("