From 8b2c08439f42dc2e3b8aab9212a2b30e80c76c6b Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Wed, 11 Sep 2024 17:22:16 +0800 Subject: [PATCH 1/3] :art: Refresh associated blocks after find-replacing and rolling back doc https://github.com/siyuan-note/siyuan/issues/12439 https://github.com/siyuan-note/siyuan/issues/12438 --- kernel/api/block.go | 2 +- kernel/model/attribute_view.go | 10 -- kernel/model/blockinfo.go | 10 +- kernel/model/format.go | 2 +- kernel/model/history.go | 5 +- kernel/model/push_reload.go | 215 +++++++++++++++++++++++++++++++++ kernel/model/search.go | 2 +- kernel/model/transaction.go | 152 ++--------------------- kernel/task/queue.go | 2 + kernel/util/websocket.go | 2 +- 10 files changed, 235 insertions(+), 167 deletions(-) create mode 100644 kernel/model/push_reload.go diff --git a/kernel/api/block.go b/kernel/api/block.go index d6d584a76..aae2f72e2 100644 --- a/kernel/api/block.go +++ b/kernel/api/block.go @@ -385,7 +385,7 @@ func getRefIDs(c *gin.Context) { } id := arg["id"].(string) - refIDs, refTexts, defIDs := model.GetBlockRefIDs(id) + refIDs, refTexts, defIDs := model.GetBlockRefs(id) ret.Data = map[string][]string{ "refIDs": refIDs, "refTexts": refTexts, diff --git a/kernel/model/attribute_view.go b/kernel/model/attribute_view.go index faf8fdcb2..524ddc6c6 100644 --- a/kernel/model/attribute_view.go +++ b/kernel/model/attribute_view.go @@ -19,7 +19,6 @@ package model import ( "bytes" "fmt" - "github.com/siyuan-note/siyuan/kernel/task" "os" "path/filepath" "slices" @@ -3580,12 +3579,3 @@ func updateBoundBlockAvsAttribute(avIDs []string) { av.BatchUpsertBlockRel(avNodes) } } - -func ReloadAttrView(avID string) { - task.AppendAsyncTaskWithDelay(task.ReloadAttributeView, 200*time.Millisecond, pushReloadAttrView, avID) - -} - -func pushReloadAttrView(avID string) { - util.BroadcastByType("protyle", "refreshAttributeView", 0, "", map[string]interface{}{"id": avID}) -} diff --git a/kernel/model/blockinfo.go b/kernel/model/blockinfo.go index b1231844e..8e368181c 100644 --- a/kernel/model/blockinfo.go +++ b/kernel/model/blockinfo.go @@ -215,21 +215,21 @@ func getNodeRefText0(node *ast.Node) string { return ret } -func GetBlockRefIDs(id string) (refIDs, refTexts, defIDs []string) { +func GetBlockRefs(defID string) (refIDs, refTexts, defIDs []string) { refIDs = []string{} refTexts = []string{} defIDs = []string{} - bt := treenode.GetBlockTree(id) + bt := treenode.GetBlockTree(defID) if nil == bt { return } isDoc := bt.ID == bt.RootID - refIDs, refTexts = sql.QueryRefIDsByDefID(id, isDoc) + refIDs, refTexts = sql.QueryRefIDsByDefID(defID, isDoc) if isDoc { - defIDs = sql.QueryChildDefIDsByRootDefID(id) + defIDs = sql.QueryChildDefIDsByRootDefID(defID) } else { - defIDs = append(defIDs, id) + defIDs = append(defIDs, defID) } return } diff --git a/kernel/model/format.go b/kernel/model/format.go index 856ca9242..e08332e01 100644 --- a/kernel/model/format.go +++ b/kernel/model/format.go @@ -31,7 +31,7 @@ func AutoSpace(rootID string) (err error) { logging.LogInfof("formatting tree [%s]...", rootID) util.PushProtyleLoading(rootID, Conf.Language(116)) - defer util.PushProtyleReload(rootID) + defer util.PushReloadProtyle(rootID) WaitForWritingFiles() diff --git a/kernel/model/history.go b/kernel/model/history.go index f2dd72478..44b270213 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -284,7 +284,7 @@ func RollbackDocHistory(boxID, historyPath string) (err error) { sql.RemoveTreeQueue(id) sql.IndexTreeQueue(tree) util.PushReloadFiletree() - util.PushProtyleReload(id) + util.PushReloadProtyle(id) util.PushMsg(Conf.Language(102), 3000) IncSync() @@ -302,8 +302,7 @@ func RollbackDocHistory(boxID, historyPath string) (err error) { return } - // 刷新关联的动态锚文本 https://github.com/siyuan-note/siyuan/issues/11575 - refreshDynamicRefText(tree.Root, tree) + refreshProtyle(id) // 刷新页签名 refText := getNodeRefText(tree.Root) diff --git a/kernel/model/push_reload.go b/kernel/model/push_reload.go new file mode 100644 index 000000000..6b015b758 --- /dev/null +++ b/kernel/model/push_reload.go @@ -0,0 +1,215 @@ +// SiYuan - Refactor your thinking +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package model + +import ( + "strings" + "time" + + "github.com/88250/gulu" + "github.com/88250/lute/ast" + "github.com/88250/lute/parse" + "github.com/emirpasic/gods/sets/hashset" + "github.com/siyuan-note/siyuan/kernel/av" + "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" + "github.com/siyuan-note/siyuan/kernel/treenode" + "github.com/siyuan-note/siyuan/kernel/util" +) + +func refreshProtyle(rootID string) { + // 刷新关联的引用 + defTree, _ := LoadTreeByBlockID(rootID) + if nil != defTree { + defIDs := sql.QueryChildDefIDsByRootDefID(rootID) + + var defNodes []*ast.Node + ast.Walk(defTree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering || !n.IsBlock() { + return ast.WalkContinue + } + + if gulu.Str.Contains(n.ID, defIDs) { + defNodes = append(defNodes, n) + } + return ast.WalkContinue + }) + + for _, def := range defNodes { + refreshDynamicRefText(def, defTree) + } + } + + // 刷新关联的嵌入块 + refIDs, _ := sql.QueryRefIDsByDefID(rootID, true) + var rootIDs []string + bts := treenode.GetBlockTrees(refIDs) + for _, bt := range bts { + rootIDs = append(rootIDs, bt.RootID) + } + rootIDs = gulu.Str.RemoveDuplicatedElem(rootIDs) + for _, id := range rootIDs { + task.AppendAsyncTaskWithDelay(task.ReloadProtyle, 200*time.Millisecond, util.PushReloadProtyle, id) + } +} + +// refreshRefCount 用于刷新定义块处的引用计数。 +func refreshRefCount(rootID, blockID string) { + sql.WaitForWritingDatabase() + + bt := treenode.GetBlockTree(blockID) + if nil == bt { + return + } + + refCounts := sql.QueryRootChildrenRefCount(bt.RootID) + refCount := refCounts[blockID] + var rootRefCount int + for _, count := range refCounts { + rootRefCount += count + } + refIDs, _, _ := GetBlockRefs(blockID) + util.PushSetDefRefCount(rootID, blockID, refIDs, refCount, rootRefCount) +} + +// refreshDynamicRefText 用于刷新块引用的动态锚文本。 +// 该实现依赖了数据库缓存，导致外部调用时可能需要阻塞等待数据库写入后才能获取到 refs +func refreshDynamicRefText(updatedDefNode *ast.Node, updatedTree *parse.Tree) { + changedDefs := map[string]*ast.Node{updatedDefNode.ID: updatedDefNode} + changedTrees := map[string]*parse.Tree{updatedTree.ID: updatedTree} + refreshDynamicRefTexts(changedDefs, changedTrees) +} + +// refreshDynamicRefTexts 用于批量刷新块引用的动态锚文本。 +// 该实现依赖了数据库缓存，导致外部调用时可能需要阻塞等待数据库写入后才能获取到 refs +func refreshDynamicRefTexts(updatedDefNodes map[string]*ast.Node, updatedTrees map[string]*parse.Tree) { + // 1. 更新引用的动态锚文本 + treeRefNodeIDs := map[string]*hashset.Set{} + var changedParentNodes []*ast.Node + for _, updateNode := range updatedDefNodes { + refs, parentNodes := getRefsCacheByDefNode(updateNode) + for _, ref := range refs { + if refIDs, ok := treeRefNodeIDs[ref.RootID]; !ok { + refIDs = hashset.New() + refIDs.Add(ref.BlockID) + treeRefNodeIDs[ref.RootID] = refIDs + } else { + refIDs.Add(ref.BlockID) + } + } + if 0 < len(parentNodes) { + changedParentNodes = append(changedParentNodes, parentNodes...) + } + } + if 0 < len(changedParentNodes) { + for _, parent := range changedParentNodes { + updatedDefNodes[parent.ID] = parent + } + } + + changedRefTree := map[string]*parse.Tree{} + + for refTreeID, refNodeIDs := range treeRefNodeIDs { + refTree, ok := updatedTrees[refTreeID] + if !ok { + var err error + refTree, err = LoadTreeByBlockID(refTreeID) + if err != nil { + continue + } + } + + var refTreeChanged bool + ast.Walk(refTree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering { + return ast.WalkContinue + } + + if n.IsBlock() && refNodeIDs.Contains(n.ID) { + changed, changedDefNodes := updateRefText(n, updatedDefNodes) + if !refTreeChanged && changed { + refTreeChanged = true + } + + // 推送动态锚文本节点刷新 + for _, defNode := range changedDefNodes { + if "ref-d" == defNode.refType { + task.AppendAsyncTaskWithDelay(task.SetRefDynamicText, 200*time.Millisecond, util.PushSetRefDynamicText, refTreeID, n.ID, defNode.id, defNode.refText) + } + } + return ast.WalkContinue + } + return ast.WalkContinue + }) + + if refTreeChanged { + changedRefTree[refTreeID] = refTree + sql.UpdateRefsTreeQueue(refTree) + } + } + + // 2. 更新属性视图主键内容 + for _, updatedDefNode := range updatedDefNodes { + avs := updatedDefNode.IALAttr(av.NodeAttrNameAvs) + if "" == avs { + continue + } + + avIDs := strings.Split(avs, ",") + for _, avID := range avIDs { + attrView, parseErr := av.ParseAttributeView(avID) + if nil != parseErr { + continue + } + + changedAv := false + blockValues := attrView.GetBlockKeyValues() + if nil == blockValues { + continue + } + + for _, blockValue := range blockValues.Values { + if blockValue.Block.ID == updatedDefNode.ID { + newContent := getNodeRefText(updatedDefNode) + if newContent != blockValue.Block.Content { + blockValue.Block.Content = newContent + changedAv = true + } + break + } + } + if changedAv { + av.SaveAttributeView(attrView) + ReloadAttrView(avID) + } + } + } + + // 3. 保存变更 + for _, tree := range changedRefTree { + indexWriteTreeUpsertQueue(tree) + } +} + +// ReloadAttrView 用于重新加载属性视图。 +func ReloadAttrView(avID string) { + task.AppendAsyncTaskWithDelay(task.ReloadAttributeView, 200*time.Millisecond, pushReloadAttrView, avID) +} + +func pushReloadAttrView(avID string) { + util.BroadcastByType("protyle", "refreshAttributeView", 0, "", map[string]interface{}{"id": avID}) +} diff --git a/kernel/model/search.go b/kernel/model/search.go index a38c5cbbb..cb8fcb1d2 100644 --- a/kernel/model/search.go +++ b/kernel/model/search.go @@ -795,7 +795,7 @@ func FindReplace(keyword, replacement string, replaceTypes map[string]bool, ids reloadTreeIDs = gulu.Str.RemoveDuplicatedElem(reloadTreeIDs) for _, id := range reloadTreeIDs { - util.PushProtyleReload(id) + refreshProtyle(id) } util.PushClearProgress() diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 322c2a85e..6e93797f5 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -34,7 +34,6 @@ import ( "github.com/88250/lute/editor" "github.com/88250/lute/lex" "github.com/88250/lute/parse" - "github.com/emirpasic/gods/sets/hashset" "github.com/siyuan-note/filelock" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/av" @@ -384,8 +383,8 @@ func (tx *Transaction) doMove(operation *Operation) (ret *TxErr) { if err = tx.writeTree(targetTree); err != nil { return } - task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, pushSetDefRefCount, srcTree.ID, srcTree.ID) - task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, pushSetDefRefCount, targetTree.ID, srcNode.ID) + task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, refreshRefCount, srcTree.ID, srcTree.ID) + task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, refreshRefCount, targetTree.ID, srcNode.ID) } return } @@ -465,8 +464,8 @@ func (tx *Transaction) doMove(operation *Operation) (ret *TxErr) { if err = tx.writeTree(targetTree); err != nil { return &TxErr{code: TxErrCodeWriteTree, msg: err.Error(), id: id} } - task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, pushSetDefRefCount, srcTree.ID, srcTree.ID) - task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, pushSetDefRefCount, targetTree.ID, srcNode.ID) + task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, refreshRefCount, srcTree.ID, srcTree.ID) + task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, refreshRefCount, targetTree.ID, srcNode.ID) } return } @@ -770,7 +769,7 @@ func (tx *Transaction) doDelete(operation *Operation) (ret *TxErr) { if nil != defTree { defNode := treenode.GetNodeInTree(defTree, defID) if nil != defNode { - task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, pushSetDefRefCount, defTree.ID, defNode.ID) + task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, refreshRefCount, defTree.ID, defNode.ID) } } } @@ -1089,7 +1088,7 @@ func (tx *Transaction) doInsert(operation *Operation) (ret *TxErr) { if nil != defTree { defNode := treenode.GetNodeInTree(defTree, defID) if nil != defNode { - task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, pushSetDefRefCount, defTree.ID, defNode.ID) + task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, refreshRefCount, defTree.ID, defNode.ID) } } } @@ -1187,7 +1186,7 @@ func (tx *Transaction) doUpdate(operation *Operation) (ret *TxErr) { if nil != defTree { defNode := treenode.GetNodeInTree(defTree, defID) if nil != defNode { - task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, pushSetDefRefCount, defTree.ID, defNode.ID) + task.AppendAsyncTaskWithDelay(task.SetDefRefCount, 1*time.Second, refreshRefCount, defTree.ID, defNode.ID) } } } @@ -1240,24 +1239,6 @@ func getRefDefIDs(node *ast.Node) (refDefIDs []string) { return } -func pushSetDefRefCount(rootID, blockID string) { - sql.WaitForWritingDatabase() - - bt := treenode.GetBlockTree(blockID) - if nil == bt { - return - } - - refCounts := sql.QueryRootChildrenRefCount(bt.RootID) - refCount := refCounts[blockID] - var rootRefCount int - for _, count := range refCounts { - rootRefCount += count - } - refIDs, _, _ := GetBlockRefIDs(blockID) - util.PushSetDefRefCount(rootID, blockID, refIDs, refCount, rootRefCount) -} - func upsertAvBlockRel(node *ast.Node) { var avIDs []string ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus { @@ -1514,125 +1495,6 @@ func (tx *Transaction) writeTree(tree *parse.Tree) (err error) { return } -// refreshDynamicRefText 用于刷新块引用的动态锚文本。 -// 该实现依赖了数据库缓存，导致外部调用时可能需要阻塞等待数据库写入后才能获取到 refs -func refreshDynamicRefText(updatedDefNode *ast.Node, updatedTree *parse.Tree) { - changedDefs := map[string]*ast.Node{updatedDefNode.ID: updatedDefNode} - changedTrees := map[string]*parse.Tree{updatedTree.ID: updatedTree} - refreshDynamicRefTexts(changedDefs, changedTrees) -} - -// refreshDynamicRefTexts 用于批量刷新块引用的动态锚文本。 -// 该实现依赖了数据库缓存，导致外部调用时可能需要阻塞等待数据库写入后才能获取到 refs -func refreshDynamicRefTexts(updatedDefNodes map[string]*ast.Node, updatedTrees map[string]*parse.Tree) { - // 1. 更新引用的动态锚文本 - treeRefNodeIDs := map[string]*hashset.Set{} - var changedParentNodes []*ast.Node - for _, updateNode := range updatedDefNodes { - refs, parentNodes := getRefsCacheByDefNode(updateNode) - for _, ref := range refs { - if refIDs, ok := treeRefNodeIDs[ref.RootID]; !ok { - refIDs = hashset.New() - refIDs.Add(ref.BlockID) - treeRefNodeIDs[ref.RootID] = refIDs - } else { - refIDs.Add(ref.BlockID) - } - } - if 0 < len(parentNodes) { - changedParentNodes = append(changedParentNodes, parentNodes...) - } - } - if 0 < len(changedParentNodes) { - for _, parent := range changedParentNodes { - updatedDefNodes[parent.ID] = parent - } - } - - changedRefTree := map[string]*parse.Tree{} - - for refTreeID, refNodeIDs := range treeRefNodeIDs { - refTree, ok := updatedTrees[refTreeID] - if !ok { - var err error - refTree, err = LoadTreeByBlockID(refTreeID) - if err != nil { - continue - } - } - - var refTreeChanged bool - ast.Walk(refTree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering { - return ast.WalkContinue - } - - if n.IsBlock() && refNodeIDs.Contains(n.ID) { - changed, changedDefNodes := updateRefText(n, updatedDefNodes) - if !refTreeChanged && changed { - refTreeChanged = true - } - - // 推送动态锚文本节点刷新 - for _, defNode := range changedDefNodes { - if "ref-d" == defNode.refType { - task.AppendAsyncTaskWithDelay(task.SetRefDynamicText, 200*time.Millisecond, util.PushSetRefDynamicText, refTreeID, n.ID, defNode.id, defNode.refText) - } - } - return ast.WalkContinue - } - return ast.WalkContinue - }) - - if refTreeChanged { - changedRefTree[refTreeID] = refTree - sql.UpdateRefsTreeQueue(refTree) - } - } - - // 2. 更新属性视图主键内容 - for _, updatedDefNode := range updatedDefNodes { - avs := updatedDefNode.IALAttr(av.NodeAttrNameAvs) - if "" == avs { - continue - } - - avIDs := strings.Split(avs, ",") - for _, avID := range avIDs { - attrView, parseErr := av.ParseAttributeView(avID) - if nil != parseErr { - continue - } - - changedAv := false - blockValues := attrView.GetBlockKeyValues() - if nil == blockValues { - continue - } - - for _, blockValue := range blockValues.Values { - if blockValue.Block.ID == updatedDefNode.ID { - newContent := getNodeRefText(updatedDefNode) - if newContent != blockValue.Block.Content { - blockValue.Block.Content = newContent - changedAv = true - } - break - } - } - if changedAv { - av.SaveAttributeView(attrView) - ReloadAttrView(avID) - } - } - } - - // 3. 保存变更 - for _, tree := range changedRefTree { - indexWriteTreeUpsertQueue(tree) - } -} - func getRefsCacheByDefNode(updateNode *ast.Node) (ret []*sql.Ref, changedParentNodes []*ast.Node) { ret = sql.GetRefsCacheByDefID(updateNode.ID) if nil != updateNode.Parent && ast.NodeDocument != updateNode.Parent.Type && diff --git a/kernel/task/queue.go b/kernel/task/queue.go index b9f5099fa..93a792ee3 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -134,6 +134,7 @@ const ( AssetContentDatabaseIndexCommit = "task.asset.database.index.commit" // 资源文件数据库索引提交 CacheVirtualBlockRef = "task.cache.virtualBlockRef" // 缓存虚拟块引用 ReloadAttributeView = "task.reload.attributeView" // 重新加载属性视图 + ReloadProtyle = "task.reload.protyle" // 重新加载编辑器 SetRefDynamicText = "task.ref.setDynamicText" // 设置引用的动态锚文本 SetDefRefCount = "task.def.setRefCount" // 设置定义的引用计数 PushMsg = "task.push.msg" // 推送消息 @@ -151,6 +152,7 @@ var uniqueActions = []string{ AssetContentDatabaseIndexFull, AssetContentDatabaseIndexCommit, ReloadAttributeView, + ReloadProtyle, SetRefDynamicText, SetDefRefCount, } diff --git a/kernel/util/websocket.go b/kernel/util/websocket.go index 8ac67673e..fcb9443fc 100644 --- a/kernel/util/websocket.go +++ b/kernel/util/websocket.go @@ -254,7 +254,7 @@ func PushSaveDoc(rootID, typ string, sources interface{}) { PushEvent(evt) } -func PushProtyleReload(rootID string) { +func PushReloadProtyle(rootID string) { BroadcastByType("protyle", "reload", 0, "", rootID) } From b182d9699ed956f99a3159c69c684b9f3b1e7fa3 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Wed, 11 Sep 2024 17:28:27 +0800 Subject: [PATCH 2/3] :arrow_up: Upgrade kernel deps --- kernel/go.mod | 8 ++++---- kernel/go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 031597d0c..3ae978289 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -38,7 +38,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.2.1 github.com/gorilla/css v1.0.1 github.com/gorilla/websocket v1.5.3 - github.com/imroc/req/v3 v3.45.0 + github.com/imroc/req/v3 v3.46.0 github.com/jinzhu/copier v0.4.0 github.com/json-iterator/go v1.1.12 github.com/klippa-app/go-pdfium v1.12.2 @@ -54,11 +54,11 @@ require ( github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/sashabaranov/go-openai v1.29.1 github.com/shirou/gopsutil/v3 v3.24.5 - github.com/siyuan-note/dejavu v0.0.0-20240910021354-c84adc417b44 + github.com/siyuan-note/dejavu v0.0.0-20240911092641-ad980102c9c4 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04 - github.com/siyuan-note/httpclient v0.0.0-20240910021232-ab6f84db3b8b + github.com/siyuan-note/httpclient v0.0.0-20240911092543-5e2322472fde github.com/siyuan-note/logging v0.0.0-20240505035402-6430d57006a2 github.com/siyuan-note/riff v0.0.0-20240911034015-5fb2819a3ad3 github.com/spf13/cast v1.7.0 @@ -107,7 +107,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/glog v1.2.2 // indirect - github.com/google/pprof v0.0.0-20240903155634-a8630aee4ab9 // indirect + github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/gorilla/context v1.1.2 // indirect diff --git a/kernel/go.sum b/kernel/go.sum index 8fc3d4b40..61eedd458 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -164,8 +164,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240903155634-a8630aee4ab9 h1:q5g0N9eal4bmJwXHC5z0QCKs8qhS35hFfq0BAYsIwZI= -github.com/google/pprof v0.0.0-20240903155634-a8630aee4ab9/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 h1:c5FlPPgxOn7kJz3VoPLkQYQXGBS3EklQ4Zfi57uOuqQ= +github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -192,8 +192,8 @@ github.com/hhrutter/tiff v1.0.1/go.mod h1:zU/dNgDm0cMIa8y8YwcYBeuEEveI4B0owqHyiP github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= -github.com/imroc/req/v3 v3.45.0 h1:cEpleKeyXXECAYuD33ZFdU1ghjqlfhtmQHwO8+Nk96Q= -github.com/imroc/req/v3 v3.45.0/go.mod h1:bhMZLgMY+g46mKmNUKFfAIY1JICvqj3kpQpSDDp45Zg= +github.com/imroc/req/v3 v3.46.0 h1:18WVbx5NdlQFpZKCZkYwO/AglIQKAx/UPC/w+rrPWv8= +github.com/imroc/req/v3 v3.46.0/go.mod h1:weam9gmyb00QnOtu6HXSnk44dNFkIUQb5QdMx13FeUU= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA= github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= @@ -337,16 +337,16 @@ github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+D github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA= github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= -github.com/siyuan-note/dejavu v0.0.0-20240910021354-c84adc417b44 h1:MPoDqmVYRIHGMBMInew7kHSPhrNR7aqbaOZHNXtNaVc= -github.com/siyuan-note/dejavu v0.0.0-20240910021354-c84adc417b44/go.mod h1:KA3/voC614vTOUjaoLXG7XeTVBoptMtzYIoA1KAroRw= +github.com/siyuan-note/dejavu v0.0.0-20240911092641-ad980102c9c4 h1:w9AW7XW4/u1yKeHHnCpU8WIm0R2+9Z+Z1zq09wHs5gk= +github.com/siyuan-note/dejavu v0.0.0-20240911092641-ad980102c9c4/go.mod h1:VjX0WMQO2HTjUdVTz6BgAEVQ37IX0uImOYbTfoxcxzk= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis= github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8= github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97/go.mod h1:1/nGgthl89FPA7GzAcEWKl6zRRnfgyTjzLZj9bW7kuw= github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04 h1:aoXvEO6BMqm6L0EnTjRhB4ynQIyJvHpqz+Ue3g0D3a0= github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04/go.mod h1:iqFhf4EDKy4MjQgT6RQJ6Z6NSW662NS0PR40K1qdSpE= -github.com/siyuan-note/httpclient v0.0.0-20240910021232-ab6f84db3b8b h1:gl+FZ3mo6ybClz2ePCJ0yo58abr1ByUDCbffqrdDxFE= -github.com/siyuan-note/httpclient v0.0.0-20240910021232-ab6f84db3b8b/go.mod h1:szsySkVwuZ1INYAbPE/afcmTjJW+IrDMBMK1tMw36oA= +github.com/siyuan-note/httpclient v0.0.0-20240911092543-5e2322472fde h1:TS8I7yDwJuEOXZcEdrup8LmMPukwSRLU8BV9WgZIEbs= +github.com/siyuan-note/httpclient v0.0.0-20240911092543-5e2322472fde/go.mod h1:IzBFIxpGyTdUhgC28wF/1LbkqS6U/VsX33Fa4V0LNSA= github.com/siyuan-note/logging v0.0.0-20240505035402-6430d57006a2 h1:/2+tlOThVB86RxSLeW0JFw2ISUrH2ZFRg15ULGAUGAE= github.com/siyuan-note/logging v0.0.0-20240505035402-6430d57006a2/go.mod h1:3Osd2/nwzXZFl6ZcDE4hA0HD83Wyv1fds47nVuapyOM= github.com/siyuan-note/riff v0.0.0-20240911034015-5fb2819a3ad3 h1:8rFXYq638/CXQ8W7xd2MFkoI7ArUeVzeWP8gEcr0W0o= From 91f74a85460e725be9b7d0130297cf9c3a6f5968 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Wed, 11 Sep 2024 18:07:29 +0800 Subject: [PATCH 3/3] :art: Parsing YAML Front Matter as document custom attributes when importing Markdown files https://github.com/siyuan-note/siyuan/issues/10878 --- kernel/model/box.go | 41 ++++++++++++++++++++++++++++++++++++++++- kernel/model/import.go | 1 + 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/kernel/model/box.go b/kernel/model/box.go index 1c0b4fa1f..20f1304bb 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -32,6 +32,7 @@ import ( "github.com/88250/gulu" "github.com/88250/lute/ast" "github.com/88250/lute/html" + "github.com/88250/lute/lex" "github.com/88250/lute/parse" "github.com/facette/natsort" "github.com/siyuan-note/filelock" @@ -43,6 +44,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" + "gopkg.in/yaml.v3" ) // Box 笔记本。 @@ -435,6 +437,7 @@ func normalizeTree(tree *parse.Tree) { tree.Root.AppendChild(treenode.NewParagraph()) } + var unlinks []*ast.Node ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { if !entering { return ast.WalkContinue @@ -496,9 +499,45 @@ func normalizeTree(tree *parse.Tree) { n.Tokens = html.UnescapeBytes(n.Tokens) } + if ast.NodeYamlFrontMatterContent == n.Type { + // Parsing YAML Front Matter as document custom attributes when importing Markdown files https://github.com/siyuan-note/siyuan/issues/10878 + attrs := map[string]interface{}{} + parseErr := yaml.Unmarshal(n.Tokens, &attrs) + if parseErr != nil { + logging.LogWarnf("parse YAML front matter [%s] failed: %s", n.Tokens, parseErr) + } else { + for attrK, attrV := range attrs { + validKeyName := true + for i := 0; i < len(attrK); i++ { + if !lex.IsASCIILetterNumHyphen(attrK[i]) { + validKeyName = false + break + } + } + if !validKeyName { + logging.LogWarnf("invalid YAML key [%s] in [%s]", attrK, n.ID) + continue + } + + tree.Root.SetIALAttr("custom-"+attrK, fmt.Sprint(attrV)) + } + } + } + + if ast.NodeYamlFrontMatter == n.Type { + unlinks = append(unlinks, n) + } + return ast.WalkContinue }) - tree.Root.KramdownIAL = parse.Tokens2IAL(tree.Root.LastChild.Tokens) + for _, n := range unlinks { + n.Unlink() + } + + rootIAL := parse.Tokens2IAL(tree.Root.LastChild.Tokens) + for _, kv := range rootIAL { + tree.Root.SetIALAttr(kv[0], kv[1]) + } return } diff --git a/kernel/model/import.go b/kernel/model/import.go index cdfc183a6..e88c62e24 100644 --- a/kernel/model/import.go +++ b/kernel/model/import.go @@ -979,6 +979,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) { func parseStdMd(markdown []byte) (ret *parse.Tree) { luteEngine := util.NewStdLute() + luteEngine.SetYamlFrontMatter(true) // 解析 YAML Front Matter https://github.com/siyuan-note/siyuan/issues/10878 ret = parse.Parse("", markdown, luteEngine.ParseOptions) if nil == ret { return