Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2026-02-11 09:44:21 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

🎨 插入较大的资源文件时内存占用较大 https://github.com/siyuan-note/siyuan/issues/5023

Browse source
This commit is contained in:
Liang Ding 2022-05-27 12:56:23 +08:00
parent dc46b478bc
commit 49b9f7bc92
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
7 changed files with 137 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

67
kernel/model/osssync.go
View file

@ -17,13 +17,9 @@
package model
import (
"bytes"
"context"
"crypto/sha1"
"encoding/base64"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
@ -662,7 +658,7 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo
return nil
}
localHash, hashErr := GetEtag(path)
localHash, hashErr := util.GetEtag(path)
if nil != hashErr {
util.LogErrorf("get local file [%s] etag failed: %s", path, hashErr)
return nil
@ -696,7 +692,7 @@ func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo
continue
}
localHash, hashErr := GetEtag(localCheckPath)
localHash, hashErr := util.GetEtag(localCheckPath)
if nil != hashErr {
util.LogErrorf("get local file [%s] hash failed: %s", localCheckPath, hashErr)
err = hashErr
@ -760,62 +756,3 @@ func putFileToCloud(filePath, key, upToken string) (err error) {
}
return
}
// 以下是七牛云 Hash 算法实现 https://github.com/qiniu/qetag/blob/master/qetag.go
func GetEtag(filename string) (etag string, err error) {
f, err := os.Open(filename)
if err != nil {
return
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return
}
fsize := fi.Size()
blockCnt := BlockCount(fsize)
sha1Buf := make([]byte, 0, 21)
if blockCnt <= 1 { // file size <= 4M
sha1Buf = append(sha1Buf, 0x16)
sha1Buf, err = CalSha1(sha1Buf, f)
if err != nil {
return
}
} else { // file size > 4M
sha1Buf = append(sha1Buf, 0x96)
sha1BlockBuf := make([]byte, 0, blockCnt*20)
for i := 0; i < blockCnt; i++ {
body := io.LimitReader(f, BLOCK_SIZE)
sha1BlockBuf, err = CalSha1(sha1BlockBuf, body)
if err != nil {
return
}
}
sha1Buf, _ = CalSha1(sha1Buf, bytes.NewReader(sha1BlockBuf))
}
etag = base64.URLEncoding.EncodeToString(sha1Buf)
return
}
const (
BLOCK_BITS = 22 // Indicate that the blocksize is 4M
BLOCK_SIZE = 1 << BLOCK_BITS
)
func BlockCount(fsize int64) int {
return int((fsize + (BLOCK_SIZE - 1)) >> BLOCK_BITS)
}
func CalSha1(b []byte, r io.Reader) ([]byte, error) {
h := sha1.New()
_, err := io.Copy(h, r)
if err != nil {
return nil, err
}
return h.Sum(b), nil
}

2
kernel/model/sync.go
View file

@ -571,7 +571,7 @@ func genCloudIndex(localDirPath string, excludes map[string]bool) (err error) {
return nil
}
hash, hashErr := GetEtag(path)
hash, hashErr := util.GetEtag(path)
if nil != hashErr {
util.LogErrorf("get file [%s] hash failed: %s", path, hashErr)
return hashErr

57
kernel/model/upload.go
View file

@ -17,9 +17,7 @@
package model
import (
"crypto/sha256"
"errors"
"fmt"
"io"
"os"
"path"
@ -58,20 +56,22 @@ func InsertLocalAssets(id string, assetPaths []string) (succMap map[string]inter
continue
}
var f *os.File
f, err = os.Open(p)
if nil != err {
fi, statErr := os.Stat(p)
if nil != statErr {
err = statErr
return
}
f, openErr := os.Open(p)
if nil != openErr {
err = openErr
return
}
hash, hashErr := util.GetEtagByHandle(f, fi.Size())
if nil != hashErr {
f.Close()
return
}
var data []byte
data, err = io.ReadAll(f)
f.Close()
if nil != err {
return
}
hash := fmt.Sprintf("%x", sha256.Sum256(data))
if existAsset := sql.QueryAssetByHash(hash); nil != existAsset {
// 已经存在同样数据的资源文件的话不重复保存
succMap[baseName] = existAsset.Path
@ -80,9 +80,15 @@ func InsertLocalAssets(id string, assetPaths []string) (succMap map[string]inter
fName = fName[0 : len(fName)-len(ext)]
fName = fName + "-" + ast.NewNodeID() + ext
writePath := filepath.Join(assets, fName)
if err = gulu.File.WriteFileSafer(writePath, data, 0644); nil != err {
if _, err = f.Seek(0, io.SeekStart); nil != err {
f.Close()
return
}
if err = gulu.File.WriteFileSaferByReader(writePath, f, 0644); nil != err {
f.Close()
return
}
f.Close()
succMap[baseName] = "assets/" + fName
}
}
@ -134,22 +140,21 @@ func Upload(c *gin.Context) {
ext = strings.ToLower(ext)
fName += ext
baseName := fName
f, err := file.Open()
if nil != err {
f, openErr := file.Open()
if nil != openErr {
errFiles = append(errFiles, fName)
ret.Msg = err.Error()
ret.Msg = openErr.Error()
break
}
data, err := io.ReadAll(f)
if nil != err {
hash, hashErr := util.GetEtagByHandle(f, file.Size)
if nil != hashErr {
errFiles = append(errFiles, fName)
ret.Msg = err.Error()
f.Close()
break
}
f.Close()
hash := fmt.Sprintf("%x", sha256.Sum256(data))
if existAsset := sql.QueryAssetByHash(hash); nil != existAsset {
// 已经存在同样数据的资源文件的话不重复保存
succMap[baseName] = existAsset.Path
@ -168,11 +173,19 @@ func Upload(c *gin.Context) {
}
}
writePath := filepath.Join(assetsDirPath, fName)
if err = gulu.File.WriteFileSafer(writePath, data, 0644); nil != err {
if _, err = f.Seek(0, io.SeekStart); nil != err {
errFiles = append(errFiles, fName)
ret.Msg = err.Error()
f.Close()
break
}
if err = gulu.File.WriteFileSaferByReader(writePath, f, 0644); nil != err {
errFiles = append(errFiles, fName)
ret.Msg = err.Error()
f.Close()
break
}
f.Close()
succMap[baseName] = "assets/" + fName
}
}