🎨 Automatically download network assets when the cloud inbox is moved to docs https://github.com/siyuan-note/siyuan/issues/9775

kernel/model/assets.go

@@ -220,6 +220,164 @@ func NetImg2LocalAssets(rootID, originalURL string) (err error) {
 	return
 }
 
+func NetAssets2LocalAssets(rootID string) (err error) {
+	tree, err := loadTreeByBlockID(rootID)
+	if nil != err {
+		return
+	}
+
+	var files int
+	msgId := gulu.Rand.String(7)
+
+	docDirLocalPath := filepath.Join(util.DataDir, tree.Box, path.Dir(tree.Path))
+	assetsDirPath := getAssetsDir(filepath.Join(util.DataDir, tree.Box), docDirLocalPath)
+	if !gulu.File.IsExist(assetsDirPath) {
+		if err = os.MkdirAll(assetsDirPath, 0755); nil != err {
+			return
+		}
+	}
+
+	browserClient := req.C().
+		SetUserAgent(util.UserAgent).
+		SetTimeout(30 * time.Second).
+		EnableInsecureSkipVerify().
+		SetProxy(httpclient.ProxyFromEnvironment)
+
+	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
+		if !entering || (ast.NodeLinkDest != n.Type && !n.IsTextMarkType("a")) {
+			return ast.WalkContinue
+		}
+
+		var dest []byte
+		if ast.NodeLinkDest == n.Type {
+			dest = n.Tokens
+		} else {
+			dest = []byte(n.TextMarkAHref)
+		}
+
+		if util.IsAssetLinkDest(dest) {
+			return ast.WalkContinue
+		}
+
+		if bytes.HasPrefix(bytes.ToLower(dest), []byte("file://")) { // 处理本地文件链接
+			u := string(dest)[7:]
+			if !gulu.File.IsExist(u) || gulu.File.IsDir(u) {
+				return ast.WalkContinue
+			}
+
+			name := filepath.Base(u)
+			name = util.FilterFileName(name)
+			name = util.TruncateLenFileName(name)
+			name = "network-asset-" + name
+			name = util.AssetName(name)
+			writePath := filepath.Join(assetsDirPath, name)
+			if err = filelock.Copy(u, writePath); nil != err {
+				logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err)
+				return ast.WalkContinue
+			}
+
+			if ast.NodeLinkDest == n.Type {
+				n.Tokens = []byte("assets/" + name)
+			} else {
+				n.TextMarkAHref = "assets/" + name
+			}
+			files++
+			return ast.WalkContinue
+		}
+
+		if bytes.HasPrefix(bytes.ToLower(dest), []byte("https://")) || bytes.HasPrefix(bytes.ToLower(dest), []byte("http://")) {
+			u := string(dest)
+			if strings.Contains(u, "qpic.cn") {
+				// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
+				if strings.Contains(u, "http://") {
+					u = strings.Replace(u, "http://", "https://", 1)
+				}
+
+				// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431
+				// 下面这部分需要注释掉，否则会导致响应 400
+				//if strings.HasSuffix(u, "/0") {
+				//	u = strings.Replace(u, "/0", "/640", 1)
+				//} else if strings.Contains(u, "/0?") {
+				//	u = strings.Replace(u, "/0?", "/640?", 1)
+				//}
+			}
+			util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), u), 15000)
+			request := browserClient.R()
+			request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second)
+			resp, reqErr := request.Get(u)
+			if nil != reqErr {
+				logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr)
+				return ast.WalkContinue
+			}
+			if 200 != resp.StatusCode {
+				logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode)
+				return ast.WalkContinue
+			}
+
+			if 1024*1024*96 < resp.ContentLength {
+				logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength)))
+				return ast.WalkContinue
+			}
+
+			data, repErr := resp.ToBytes()
+			if nil != repErr {
+				logging.LogErrorf("download network asset [%s] failed: %s", u, repErr)
+				return ast.WalkContinue
+			}
+			var name string
+			if strings.Contains(u, "?") {
+				name = u[:strings.Index(u, "?")]
+				name = path.Base(name)
+			} else {
+				name = path.Base(u)
+			}
+			if strings.Contains(name, "#") {
+				name = name[:strings.Index(name, "#")]
+			}
+			name, _ = url.PathUnescape(name)
+			ext := path.Ext(name)
+			if "" == ext {
+				if mtype := mimetype.Detect(data); nil != mtype {
+					ext = mtype.Extension()
+				}
+			}
+			if "" == ext {
+				contentType := resp.Header.Get("Content-Type")
+				exts, _ := mime.ExtensionsByType(contentType)
+				if 0 < len(exts) {
+					ext = exts[0]
+				}
+			}
+			name = strings.TrimSuffix(name, ext)
+			name = util.FilterFileName(name)
+			name = util.TruncateLenFileName(name)
+			name = "network-asset-" + name + "-" + ast.NewNodeID() + ext
+			writePath := filepath.Join(assetsDirPath, name)
+			if err = filelock.WriteFile(writePath, data); nil != err {
+				logging.LogErrorf("write downloaded network asset [%s] to local asset [%s] failed: %s", u, writePath, err)
+				return ast.WalkContinue
+			}
+
+			if ast.NodeLinkDest == n.Type {
+				n.Tokens = []byte("assets/" + name)
+			} else {
+				n.TextMarkAHref = "assets/" + name
+			}
+			files++
+		}
+		return ast.WalkContinue
+	})
+
+	if 0 < files {
+		util.PushUpdateMsg(msgId, Conf.Language(113), 7000)
+		if err = writeJSONQueue(tree); nil != err {
+			return
+		}
+		util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files), 5000)
+	}
+	return
+}
+
 func SearchAssetsByName(keyword string, exts []string) (ret []*cache.Asset) {
 	ret = []*cache.Asset{}