🎨 插入较大的资源文件时内存占用较大 https://github.com/siyuan-note/siyuan/issues/5023

kernel/util/etag.go

@@ -0,0 +1,88 @@
+// SiYuan - Build Your Eternal Digital Garden
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package util
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding/base64"
+	"io"
+	"os"
+)
+
+// 以下是七牛云 Hash 算法实现 https://github.com/qiniu/qetag/blob/master/qetag.go
+
+func GetEtagByHandle(f io.Reader, size int64) (etag string, err error) {
+	blockCnt := BlockCount(size)
+	sha1Buf := make([]byte, 0, 21)
+
+	if blockCnt <= 1 { // file size <= 4M
+		sha1Buf = append(sha1Buf, 0x16)
+		sha1Buf, err = CalSha1(sha1Buf, f)
+		if err != nil {
+			return
+		}
+	} else { // file size > 4M
+		sha1Buf = append(sha1Buf, 0x96)
+		sha1BlockBuf := make([]byte, 0, blockCnt*20)
+		for i := 0; i < blockCnt; i++ {
+			body := io.LimitReader(f, BLOCK_SIZE)
+			sha1BlockBuf, err = CalSha1(sha1BlockBuf, body)
+			if err != nil {
+				return
+			}
+		}
+		sha1Buf, _ = CalSha1(sha1Buf, bytes.NewReader(sha1BlockBuf))
+	}
+	etag = base64.URLEncoding.EncodeToString(sha1Buf)
+	return
+}
+
+func GetEtag(filename string) (etag string, err error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		return
+	}
+
+	etag, err = GetEtagByHandle(f, fi.Size())
+	return
+}
+
+const (
+	BLOCK_BITS = 22 // Indicate that the blocksize is 4M
+	BLOCK_SIZE = 1 << BLOCK_BITS
+)
+
+func BlockCount(fsize int64) int {
+	return int((fsize + (BLOCK_SIZE - 1)) >> BLOCK_BITS)
+}
+
+func CalSha1(b []byte, r io.Reader) ([]byte, error) {
+
+	h := sha1.New()
+	_, err := io.Copy(h, r)
+	if err != nil {
+		return nil, err
+	}
+	return h.Sum(b), nil
+}