Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2025-12-24 02:20:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

🎨 插入较大的资源文件时内存占用较大 https://github.com/siyuan-note/siyuan/issues/5023

Browse source
This commit is contained in:
Liang Ding 2022-05-27 12:56:23 +08:00
parent dc46b478bc
commit 49b9f7bc92
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
7 changed files with 137 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

67
kernel/model/osssync.go
View file

@ -17,13 +17,9 @@
package model
import (
"bytes"
"context"
"crypto/sha1"
"encoding/base64"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
@ -662,7 +658,7 @@ func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo
return nil
}
localHash, hashErr := GetEtag(path)
localHash, hashErr := util.GetEtag(path)
if nil != hashErr {
util.LogErrorf("get local file [%s] etag failed: %s", path, hashErr)
return nil
@ -696,7 +692,7 @@ func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*Clo
continue
}
localHash, hashErr := GetEtag(localCheckPath)
localHash, hashErr := util.GetEtag(localCheckPath)
if nil != hashErr {
util.LogErrorf("get local file [%s] hash failed: %s", localCheckPath, hashErr)
err = hashErr
@ -760,62 +756,3 @@ func putFileToCloud(filePath, key, upToken string) (err error) {
}
return
}
// 以下是七牛云 Hash 算法实现 https://github.com/qiniu/qetag/blob/master/qetag.go
func GetEtag(filename string) (etag string, err error) {
f, err := os.Open(filename)
if err != nil {
return
}
defer f.Close()
fi, err := f.Stat()
if err != nil {
return
}
fsize := fi.Size()
blockCnt := BlockCount(fsize)
sha1Buf := make([]byte, 0, 21)
if blockCnt <= 1 { // file size <= 4M
sha1Buf = append(sha1Buf, 0x16)
sha1Buf, err = CalSha1(sha1Buf, f)
if err != nil {
return
}
} else { // file size > 4M
sha1Buf = append(sha1Buf, 0x96)
sha1BlockBuf := make([]byte, 0, blockCnt*20)
for i := 0; i < blockCnt; i++ {
body := io.LimitReader(f, BLOCK_SIZE)
sha1BlockBuf, err = CalSha1(sha1BlockBuf, body)
if err != nil {
return
}
}
sha1Buf, _ = CalSha1(sha1Buf, bytes.NewReader(sha1BlockBuf))
}
etag = base64.URLEncoding.EncodeToString(sha1Buf)
return
}
const (
BLOCK_BITS = 22 // Indicate that the blocksize is 4M
BLOCK_SIZE = 1 << BLOCK_BITS
)
func BlockCount(fsize int64) int {
return int((fsize + (BLOCK_SIZE - 1)) >> BLOCK_BITS)
}
func CalSha1(b []byte, r io.Reader) ([]byte, error) {
h := sha1.New()
_, err := io.Copy(h, r)
if err != nil {
return nil, err
}
return h.Sum(b), nil
}