Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2026-01-07 09:18:49 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

❤️ 完整开源界面和内核 https://github.com/siyuan-note/siyuan/issues/5013

Browse source
This commit is contained in:
Liang Ding 2022-05-26 15:18:53 +08:00
parent e650b8100c
commit f40ed985e1
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
1214 changed files with 345766 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

169
kernel/sql/aseet.go Normal file
View file

@ -0,0 +1,169 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"crypto/sha256"
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
type Asset struct {
ID string
BlockID string
RootID string
Box string
DocPath string
Path string
Name string
Title string
Hash string
}
func docTagSpans(n *ast.Node) (ret []*Span) {
if tagsVal := n.IALAttr("tags"); "" != tagsVal {
tags := strings.Split(tagsVal, ",")
for _, tag := range tags {
markdown := "#" + tag + "#"
span := &Span{
ID: ast.NewNodeID(),
BlockID: n.ID,
RootID: n.ID,
Box: n.Box,
Path: n.Path,
Content: tag,
Markdown: markdown,
Type: "tag",
IAL: "",
}
ret = append(ret, span)
}
}
return
}
func docTitleImgAsset(root *ast.Node) *Asset {
if p := treenode.GetDocTitleImgPath(root); "" != p {
if !IsAssetLinkDest([]byte(p)) {
return nil
}
var hash string
absPath := filepath.Join(util.DataDir, p)
if data, err := os.ReadFile(absPath); nil != err {
util.LogErrorf("read asset [%s] data failed: %s", absPath, err)
hash = fmt.Sprintf("%x", sha256.Sum256([]byte(gulu.Rand.String(7))))
} else {
hash = fmt.Sprintf("%x", sha256.Sum256(data))
}
name, _ := util.LastID(p)
asset := &Asset{
ID: ast.NewNodeID(),
BlockID: root.ID,
RootID: root.ID,
Box: root.Box,
DocPath: p,
Path: p,
Name: name,
Title: "title-img",
Hash: hash,
}
return asset
}
return nil
}
func QueryAssetsByName(name string) (ret []*Asset) {
ret = []*Asset{}
sqlStmt := "SELECT * FROM assets WHERE name LIKE ? GROUP BY id ORDER BY id DESC LIMIT 32"
rows, err := query(sqlStmt, "%"+name+"%")
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
asset := scanAssetRows(rows)
ret = append(ret, asset)
}
return
}
func QueryAssetByHash(hash string) (ret *Asset) {
sqlStmt := "SELECT * FROM assets WHERE hash = ?"
row := queryRow(sqlStmt, hash)
var asset Asset
if err := row.Scan(&asset.ID, &asset.BlockID, &asset.RootID, &asset.Box, &asset.DocPath, &asset.Path, &asset.Name, &asset.Title, &asset.Hash); nil != err {
if sql.ErrNoRows != err {
util.LogErrorf("query scan field failed: %s", err)
}
return
}
ret = &asset
return
}
func QueryRootBlockAssets(rootID string) (ret []*Asset) {
sqlStmt := "SELECT * FROM assets WHERE root_id = ?"
rows, err := query(sqlStmt, rootID)
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
asset := scanAssetRows(rows)
ret = append(ret, asset)
}
return
}
func scanAssetRows(rows *sql.Rows) (ret *Asset) {
var asset Asset
if err := rows.Scan(&asset.ID, &asset.BlockID, &asset.RootID, &asset.Box, &asset.DocPath, &asset.Path, &asset.Name, &asset.Title, &asset.Hash); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret = &asset
return
}
func assetLocalPath(linkDest, boxLocalPath, docDirLocalPath string) (ret string) {
ret = filepath.Join(docDirLocalPath, linkDest)
if gulu.File.IsExist(ret) {
return
}
ret = filepath.Join(boxLocalPath, linkDest)
if gulu.File.IsExist(ret) {
return
}
ret = filepath.Join(util.DataDir, linkDest)
if gulu.File.IsExist(ret) {
return
}
return ""
}

28
kernel/sql/attribute.go Normal file
View file

@ -0,0 +1,28 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
type Attribute struct {
ID string
Name string
Value string
Type string
BlockID string
RootID string
Box string
Path string
}

65
kernel/sql/block.go Normal file
View file

@ -0,0 +1,65 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
)
type Block struct {
ID string
ParentID string
RootID string
Hash string
Box string
Path string
HPath string
Name string
Alias string
Memo string
Tag string
Content string
FContent string
Markdown string
Length int
Type string
SubType string
IAL string
Sort int
Created string
Updated string
}
func updateRootContent(tx *sql.Tx, content, id string) {
stmt := "UPDATE blocks SET content = ?, fcontent = ? WHERE id = ?"
if err := execStmtTx(tx, stmt, content, content, id); nil != err {
return
}
stmt = "UPDATE blocks_fts SET content = ?, fcontent = ? WHERE id = ?"
if err := execStmtTx(tx, stmt, content, content, id); nil != err {
return
}
stmt = "UPDATE blocks_fts_case_insensitive SET content = ?, fcontent = ? WHERE id = ?"
if err := execStmtTx(tx, stmt, content, content, id); nil != err {
return
}
removeBlockCache(id)
}
func InsertBlock(tx *sql.Tx, block *Block) (err error) {
return insertBlocks(tx, []*Block{block})
}

670
kernel/sql/block_query.go Normal file
View file

@ -0,0 +1,670 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"bytes"
"database/sql"
"sort"
"strconv"
"strings"
"github.com/88250/lute/ast"
"github.com/88250/vitess-sqlparser/sqlparser"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
func queryBlockHashes(rootID string) (ret map[string]string) {
stmt := "SELECT id, hash FROM blocks WHERE root_id = ?"
rows, err := query(stmt, rootID)
if nil != err {
util.LogErrorf("sql query failed: %s", stmt, err)
return
}
defer rows.Close()
ret = map[string]string{}
for rows.Next() {
var id, hash string
if err = rows.Scan(&id, &hash); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret[id] = hash
}
return
}
func QueryRootBlockByCondition(condition string) (ret []*Block) {
sqlStmt := "SELECT *, length(hpath) - length(replace(hpath, '/', '')) AS lv FROM blocks WHERE type = 'd' AND " + condition + " ORDER BY box DESC,lv ASC LIMIT 128"
rows, err := query(sqlStmt)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var block Block
var sepCount int
if err = rows.Scan(&block.ID, &block.ParentID, &block.RootID, &block.Hash, &block.Box, &block.Path, &block.HPath, &block.Name, &block.Alias, &block.Memo, &block.Tag, &block.Content, &block.FContent, &block.Markdown, &block.Length, &block.Type, &block.SubType, &block.IAL, &block.Sort, &block.Created, &block.Updated, &sepCount); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret = append(ret, &block)
}
return
}
func (block *Block) IsContainerBlock() bool {
switch block.Type {
case "d", "b", "l", "i", "s":
return true
}
return false
}
func IsBlockFolded(id string) (ret bool) {
sqlStmt := "SELECT parent_id, ial FROM blocks WHERE id = ? AND type != 'd'"
for i := 0; i < 64; i++ {
row := queryRow(sqlStmt, id)
var pid, ial string
if err := row.Scan(&pid, &ial); nil != err {
if sql.ErrNoRows != err {
util.LogErrorf("query scan field failed: %s", err)
}
return
}
id = pid
if strings.Contains(ial, "fold=\"1\"") {
return true
}
}
return
}
func queryBlockChildrenIDs(id string) (ret []string) {
ret = append(ret, id)
childIDs := queryBlockIDByParentID(id)
for _, childID := range childIDs {
ret = append(ret, queryBlockChildrenIDs(childID)...)
}
return
}
func queryBlockIDByParentID(parentID string) (ret []string) {
sqlStmt := "SELECT id FROM blocks WHERE parent_id = ?"
rows, err := query(sqlStmt, parentID)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var id string
rows.Scan(&id)
ret = append(ret, id)
}
return
}
func QueryRecentUpdatedBlocks() (ret []*Block) {
sqlStmt := "SELECT * FROM blocks WHERE type = 'p' AND length > 1 ORDER BY updated DESC LIMIT 16"
if "ios" == util.Container || "android" == util.Container {
sqlStmt = "SELECT * FROM blocks WHERE type = 'd' ORDER BY updated DESC LIMIT 16"
}
rows, err := query(sqlStmt)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func QueryBlockByNameOrAlias(rootID, text string) (ret *Block) {
sqlStmt := "SELECT * FROM blocks WHERE root_id = ? AND (alias LIKE ? OR name = ?)"
row := queryRow(sqlStmt, rootID, "%"+text+"%", text)
ret = scanBlockRow(row)
return
}
func QueryBlockAliases(rootID string) (ret []string) {
sqlStmt := "SELECT alias FROM blocks WHERE root_id = ? AND alias != ''"
rows, err := query(sqlStmt, rootID)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
var aliasesRows []string
for rows.Next() {
var name string
rows.Scan(&name)
aliasesRows = append(aliasesRows, name)
}
for _, aliasStr := range aliasesRows {
aliases := strings.Split(aliasStr, ",")
for _, alias := range aliases {
var exist bool
for _, retAlias := range ret {
if retAlias == alias {
exist = true
}
}
if !exist {
ret = append(ret, alias)
}
}
}
return
}
func queryNames() (ret []string) {
ret = []string{}
sqlStmt := "SELECT name FROM blocks WHERE name != '' LIMIT ?"
rows, err := query(sqlStmt, 10240)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
var namesRows []string
for rows.Next() {
var name string
rows.Scan(&name)
namesRows = append(namesRows, name)
}
set := hashset.New()
for _, namesStr := range namesRows {
names := strings.Split(namesStr, ",")
for _, name := range names {
if "" == strings.TrimSpace(name) {
continue
}
set.Add(name)
}
}
for _, v := range set.Values() {
ret = append(ret, v.(string))
}
return
}
func queryAliases() (ret []string) {
ret = []string{}
sqlStmt := "SELECT alias FROM blocks WHERE alias != '' LIMIT ?"
rows, err := query(sqlStmt, 10240)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
var aliasesRows []string
for rows.Next() {
var alias string
rows.Scan(&alias)
aliasesRows = append(aliasesRows, alias)
}
set := hashset.New()
for _, aliasStr := range aliasesRows {
aliases := strings.Split(aliasStr, ",")
for _, alias := range aliases {
if "" == strings.TrimSpace(alias) {
continue
}
set.Add(alias)
}
}
for _, v := range set.Values() {
ret = append(ret, v.(string))
}
return
}
func queryDocIDsByTitle(title string, excludeIDs []string) (ret []string) {
ret = []string{}
notIn := "('" + strings.Join(excludeIDs, "','") + "')"
sqlStmt := "SELECT id FROM blocks WHERE type = 'd' AND content = ? AND id NOT IN " + notIn + " LIMIT ?"
rows, err := query(sqlStmt, title, 32)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
set := hashset.New()
for rows.Next() {
var id string
rows.Scan(&id)
set.Add(id)
}
for _, v := range set.Values() {
ret = append(ret, v.(string))
}
return
}
func queryDocTitles() (ret []string) {
ret = []string{}
sqlStmt := "SELECT content FROM blocks WHERE type = 'd' LIMIT ?"
rows, err := query(sqlStmt, 10240)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
var docNamesRows []string
for rows.Next() {
var name string
rows.Scan(&name)
docNamesRows = append(docNamesRows, name)
}
set := hashset.New()
for _, nameStr := range docNamesRows {
names := strings.Split(nameStr, ",")
for _, name := range names {
if "" == strings.TrimSpace(name) {
continue
}
set.Add(name)
}
}
for _, v := range set.Values() {
ret = append(ret, v.(string))
}
return
}
func QueryBlockNamesByRootID(rootID string) (ret []string) {
sqlStmt := "SELECT DISTINCT name FROM blocks WHERE root_id = ? AND name != ''"
rows, err := query(sqlStmt, rootID)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
var name string
rows.Scan(&name)
ret = append(ret, name)
}
return
}
func QueryBookmarkBlocksByKeyword(bookmark string) (ret []*Block) {
sqlStmt := "SELECT * FROM blocks WHERE ial LIKE ?"
rows, err := query(sqlStmt, "%bookmark=%")
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func QueryBookmarkBlocks() (ret []*Block) {
sqlStmt := "SELECT * FROM blocks WHERE ial LIKE ?"
rows, err := query(sqlStmt, "%bookmark=%")
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func QueryBookmarkLabels() (ret []string) {
ret = []string{}
sqlStmt := "SELECT * FROM blocks WHERE ial LIKE ?"
rows, err := query(sqlStmt, "%bookmark=%")
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
labels := map[string]bool{}
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
if v := ialAttr(block.IAL, "bookmark"); "" != v {
labels[v] = true
}
}
}
for label := range labels {
ret = append(ret, label)
}
sort.Strings(ret)
return
}
func Query(stmt string) (ret []map[string]interface{}, err error) {
ret = []map[string]interface{}{}
rows, err := query(stmt)
if nil != err {
util.LogWarnf("sql query [%s] failed: %s", stmt, err)
return
}
defer rows.Close()
cols, _ := rows.Columns()
if nil == cols {
return
}
for rows.Next() {
columns := make([]interface{}, len(cols))
columnPointers := make([]interface{}, len(cols))
for i := range columns {
columnPointers[i] = &columns[i]
}
if err = rows.Scan(columnPointers...); nil != err {
return
}
m := make(map[string]interface{})
for i, colName := range cols {
val := columnPointers[i].(*interface{})
m[colName] = *val
}
ret = append(ret, m)
}
return
}
func SelectBlocksRawStmtNoParse(stmt string, limit int) (ret []*Block) {
return selectBlocksRawStmt(stmt, limit)
}
func SelectBlocksRawStmt(stmt string, limit int) (ret []*Block) {
parsedStmt, err := sqlparser.Parse(stmt)
if nil != err {
return selectBlocksRawStmt(stmt, limit)
}
switch parsedStmt.(type) {
case *sqlparser.Select:
slct := parsedStmt.(*sqlparser.Select)
if nil == slct.Limit {
slct.Limit = &sqlparser.Limit{
Rowcount: &sqlparser.SQLVal{
Type: sqlparser.IntVal,
Val: []byte(strconv.Itoa(limit)),
},
}
}
stmt = sqlparser.String(slct)
default:
return
}
stmt = strings.ReplaceAll(stmt, "\\'", "''")
stmt = strings.ReplaceAll(stmt, "\\\"", "\"")
stmt = strings.ReplaceAll(stmt, "\\\\*", "\\*")
stmt = strings.ReplaceAll(stmt, "from dual", "")
rows, err := query(stmt)
if nil != err {
if strings.Contains(err.Error(), "syntax error") {
return
}
util.LogWarnf("sql query [%s] failed: %s", stmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func selectBlocksRawStmt(stmt string, limit int) (ret []*Block) {
rows, err := query(stmt)
if nil != err {
if strings.Contains(err.Error(), "syntax error") {
return
}
return
}
defer rows.Close()
confLimit := !strings.Contains(strings.ToLower(stmt), " limit ")
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
if confLimit && limit < len(ret) {
break
}
}
}
return
}
func scanBlockRows(rows *sql.Rows) (ret *Block) {
var block Block
if err := rows.Scan(&block.ID, &block.ParentID, &block.RootID, &block.Hash, &block.Box, &block.Path, &block.HPath, &block.Name, &block.Alias, &block.Memo, &block.Tag, &block.Content, &block.FContent, &block.Markdown, &block.Length, &block.Type, &block.SubType, &block.IAL, &block.Sort, &block.Created, &block.Updated); nil != err {
util.LogErrorf("query scan field failed: %s\n%s", err, util.ShortStack())
return
}
ret = &block
return
}
func scanBlockRow(row *sql.Row) (ret *Block) {
var block Block
if err := row.Scan(&block.ID, &block.ParentID, &block.RootID, &block.Hash, &block.Box, &block.Path, &block.HPath, &block.Name, &block.Alias, &block.Memo, &block.Tag, &block.Content, &block.FContent, &block.Markdown, &block.Length, &block.Type, &block.SubType, &block.IAL, &block.Sort, &block.Created, &block.Updated); nil != err {
if sql.ErrNoRows != err {
util.LogErrorf("query scan field failed: %s\n%s", err, util.ShortStack())
}
return
}
ret = &block
return
}
func GetChildBlocks(parentID, condition string) (ret []*Block) {
blockIDs := queryBlockChildrenIDs(parentID)
var params []string
for _, id := range blockIDs {
params = append(params, "\""+id+"\"")
}
ret = []*Block{}
sqlStmt := "SELECT * FROM blocks AS ref WHERE ref.id IN (" + strings.Join(params, ",") + ")"
if "" != condition {
sqlStmt += " AND " + condition
}
rows, err := query(sqlStmt)
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func GetAllChildBlocks(rootID, condition string) (ret []*Block) {
ret = []*Block{}
sqlStmt := "SELECT * FROM blocks AS ref WHERE ref.root_id = ?"
if "" != condition {
sqlStmt += " AND " + condition
}
rows, err := query(sqlStmt, rootID)
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func GetRefUnresolvedBlocks() (ret []*Block) {
stmt := "SELECT * FROM blocks WHERE content LIKE ?"
rows, err := query(stmt, "%ref resolve failed%")
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", stmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func GetRefExistedBlocks() (ret []*Block) {
stmt := "SELECT * FROM blocks WHERE markdown LIKE ? OR markdown LIKE ?"
rows, err := query(stmt, "%((20%", "%<<20%")
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", stmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func GetBlock(id string) (ret *Block) {
ret = getBlockCache(id)
if nil != ret {
return
}
row := queryRow("SELECT * FROM blocks WHERE id = ?", id)
ret = scanBlockRow(row)
if nil != ret {
putBlockCache(ret)
}
return
}
func GetAllRootBlocks() (ret []*Block) {
stmt := "SELECT * FROM blocks WHERE type = 'd'"
rows, err := query(stmt)
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", stmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func GetBlocks(ids []string) (ret []*Block) {
length := len(ids)
stmtBuilder := bytes.Buffer{}
stmtBuilder.WriteString("SELECT * FROM blocks WHERE id IN (")
var args []interface{}
for i, id := range ids {
args = append(args, id)
stmtBuilder.WriteByte('?')
if i < length-1 {
stmtBuilder.WriteByte(',')
}
}
stmtBuilder.WriteString(")")
sqlStmt := stmtBuilder.String()
rows, err := query(sqlStmt, args...)
if nil != err {
util.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
putBlockCache(block)
}
}
return
}
func GetContainerText(container *ast.Node) string {
buf := &bytes.Buffer{}
buf.Grow(4096)
leaf := treenode.FirstLeafBlock(container)
ast.Walk(leaf, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
switch n.Type {
case ast.NodeTagOpenMarker, ast.NodeTagCloseMarker:
buf.WriteByte('#')
case ast.NodeText, ast.NodeLinkText, ast.NodeFileAnnotationRefText, ast.NodeFootnotesRef,
ast.NodeCodeSpanContent, ast.NodeInlineMathContent, ast.NodeCodeBlockCode, ast.NodeMathBlockContent:
buf.Write(n.Tokens)
case ast.NodeBlockRef:
if anchor := n.ChildByType(ast.NodeBlockRefText); nil != anchor {
buf.WriteString(anchor.Text())
} else if anchor = n.ChildByType(ast.NodeBlockRefDynamicText); nil != anchor {
buf.WriteString(anchor.Text())
} else {
text := GetRefText(n.TokensStr())
buf.WriteString(text)
}
return ast.WalkSkipChildren
}
return ast.WalkContinue
})
return buf.String()
}

49
kernel/sql/block_ref.go Normal file
View file

@ -0,0 +1,49 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"github.com/88250/lute/parse"
)
type Ref struct {
ID string
DefBlockID string
DefBlockParentID string
DefBlockRootID string
DefBlockPath string
BlockID string
RootID string
Box string
Path string
Content string
Markdown string
Type string
}
func UpsertRefs(tx *sql.Tx, tree *parse.Tree) {
if err := deleteRefsByPath(tx, tree.Box, tree.Path); nil != err {
return
}
if err := deleteFileAnnotationRefsByPath(tx, tree.Box, tree.Path); nil != err {
return
}
insertRef(tx, tree)
return
}

419
kernel/sql/block_ref_query.go Normal file
View file

@ -0,0 +1,419 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"sort"
"strings"
"github.com/88250/lute/parse"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/siyuan/kernel/util"
)
func QueryVirtualRefKeywords(name, alias, anchor, doc bool) (ret []string) {
ret, ok := getVirtualRefKeywordsCache()
if ok {
return ret
}
if name {
ret = append(ret, queryNames()...)
}
if alias {
ret = append(ret, queryAliases()...)
}
if anchor {
ret = append(ret, queryRefTexts()...)
}
if doc {
ret = append(ret, queryDocTitles()...)
}
ret = util.RemoveDuplicatedElem(ret)
sort.SliceStable(ret, func(i, j int) bool {
return len(ret[i]) >= len(ret[j])
})
setVirtualRefKeywords(ret)
return
}
func queryRefTexts() (ret []string) {
ret = []string{}
sqlStmt := "SELECT DISTINCT content FROM refs LIMIT 1024"
rows, err := query(sqlStmt)
if nil != err {
util.LogErrorf("sql query failed: %s", sqlStmt, err)
return
}
defer rows.Close()
set := hashset.New()
for rows.Next() {
var refText string
rows.Scan(&refText)
if "" == strings.TrimSpace(refText) {
continue
}
set.Add(refText)
}
for _, refText := range set.Values() {
ret = append(ret, refText.(string))
}
return
}
func QueryRootChildrenRefCount(defRootID string) (ret map[string]int) {
ret = map[string]int{}
rows, err := query("SELECT def_block_id, COUNT(*) AS ref_cnt FROM refs WHERE def_block_root_id = ? GROUP BY def_block_id", defRootID)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var id string
var cnt int
if err = rows.Scan(&id, &cnt); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret[id] = cnt
}
return
}
func QueryRootBlockRefCount() (ret map[string]int) {
ret = map[string]int{}
rows, err := query("SELECT def_block_root_id, COUNT(*) AS ref_cnt FROM refs GROUP BY def_block_root_id")
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var id string
var cnt int
if err = rows.Scan(&id, &cnt); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret[id] = cnt
}
return
}
func QueryDefRootBlocksByRefRootID(refRootID string) (ret []*Block) {
rows, err := query("SELECT * FROM blocks WHERE id IN (SELECT DISTINCT def_block_root_id FROM refs WHERE root_id = ?)", refRootID)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func QueryRefRootBlocksByDefRootID(defRootID string) (ret []*Block) {
rows, err := query("SELECT * FROM blocks WHERE id IN (SELECT DISTINCT root_id FROM refs WHERE def_block_root_id = ?)", defRootID)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
if block := scanBlockRows(rows); nil != block {
ret = append(ret, block)
}
}
return
}
func GetRefText(defBlockID string) string {
block := GetBlock(defBlockID)
if nil == block {
if strings.HasPrefix(defBlockID, "assets") {
return defBlockID
}
return "block not found"
}
if "" != block.Name {
return block.Name
}
switch block.Type {
case "d":
return block.Content
case "query_embed":
return "Query Embed Block " + block.Markdown
case "iframe":
return "IFrame " + block.Markdown
case "tb":
return "Thematic Break"
case "video":
return "Video " + block.Markdown
case "audio":
return "Audio " + block.Markdown
}
if block.IsContainerBlock() {
subTree := parse.Parse("", []byte(block.Markdown), luteEngine.ParseOptions)
return GetContainerText(subTree.Root)
}
return block.Content
}
func QueryBlockDefIDsByRefText(refText string, excludeIDs []string) (ret []string) {
ret = queryDefIDsByDefText(refText, excludeIDs)
ret = append(ret, queryDefIDsByNameAlias(refText, excludeIDs)...)
ret = append(ret, queryDocIDsByTitle(refText, excludeIDs)...)
ret = util.RemoveDuplicatedElem(ret)
return
}
func queryDefIDsByDefText(keyword string, excludeIDs []string) (ret []string) {
ret = []string{}
notIn := "('" + strings.Join(excludeIDs, "','") + "')"
rows, err := query("SELECT DISTINCT(def_block_id) FROM refs WHERE content = ? AND def_block_id NOT IN "+notIn, keyword)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var id string
if err = rows.Scan(&id); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret = append(ret, id)
}
return
}
func queryDefIDsByNameAlias(keyword string, excludeIDs []string) (ret []string) {
ret = []string{}
notIn := "('" + strings.Join(excludeIDs, "','") + "')"
rows, err := query("SELECT DISTINCT(id), name, alias FROM blocks WHERE (name = ? OR alias LIKE ?) AND id NOT IN "+notIn, keyword, "%"+keyword+"%")
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var id, name, alias string
if err = rows.Scan(&id, &name, &alias); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
if name == keyword {
ret = append(ret, id)
continue
}
var hitAlias bool
aliases := strings.Split(alias, ",")
for _, a := range aliases {
if "" == a {
continue
}
if keyword == a {
hitAlias = true
}
}
if strings.Contains(alias, keyword) && !hitAlias {
continue
}
ret = append(ret, id)
}
return
}
func QueryChildDefIDsByRootDefID(rootDefID string) (ret []string) {
ret = []string{}
rows, err := query("SELECT DISTINCT(def_block_id) FROM refs WHERE def_block_root_id = ?", rootDefID)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var id string
if err = rows.Scan(&id); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret = append(ret, id)
}
return
}
func QueryRefIDsByDefID(defID string, containChildren bool) (refIDs, refTexts []string) {
refIDs = []string{}
var rows *sql.Rows
var err error
if containChildren {
rows, err = query("SELECT block_id, content FROM refs WHERE def_block_root_id = ?", defID)
} else {
rows, err = query("SELECT block_id, content FROM refs WHERE def_block_id = ?", defID)
}
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var id, content string
if err = rows.Scan(&id, &content); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
refIDs = append(refIDs, id)
refTexts = append(refTexts, content)
}
return
}
func QueryRefsRecent() (ret []*Ref) {
rows, err := query("SELECT * FROM refs GROUP BY def_block_id ORDER BY id desc LIMIT 32")
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
ref := scanRefRows(rows)
ret = append(ret, ref)
}
return
}
func QueryRefsByDefID(defBlockID string, containChildren bool) (ret []*Ref) {
sqlBlock := GetBlock(defBlockID)
if nil == sqlBlock {
return
}
var rows *sql.Rows
var err error
if "d" == sqlBlock.Type {
rows, err = query("SELECT * FROM refs WHERE def_block_root_id = ?", defBlockID)
} else {
if containChildren {
blockIDs := queryBlockChildrenIDs(defBlockID)
var params []string
for _, id := range blockIDs {
params = append(params, "\""+id+"\"")
}
rows, err = query("SELECT * FROM refs WHERE def_block_id IN (" + strings.Join(params, ",") + ")")
} else {
rows, err = query("SELECT * FROM refs WHERE def_block_id = ?", defBlockID)
}
}
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
ref := scanRefRows(rows)
ret = append(ret, ref)
}
return
}
func QueryRefsByDefIDRefID(defBlockID, refBlockID string) (ret []*Ref) {
stmt := "SELECT * FROM refs WHERE def_block_id = ? AND block_id = ?"
rows, err := query(stmt, defBlockID, refBlockID)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
ref := scanRefRows(rows)
ret = append(ret, ref)
}
return
}
func DefRefs(condition string) (ret []map[*Block]*Block) {
ret = []map[*Block]*Block{}
stmt := "SELECT ref.*, r.block_id || '@' || r.def_block_id AS rel FROM blocks AS ref, refs AS r WHERE ref.id = r.block_id"
if "" != condition {
stmt += " AND " + condition
}
rows, err := query(stmt)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
refs := map[string]*Block{}
for rows.Next() {
var ref Block
var rel string
if err = rows.Scan(&ref.ID, &ref.ParentID, &ref.RootID, &ref.Hash, &ref.Box, &ref.Path, &ref.HPath, &ref.Name, &ref.Alias, &ref.Memo, &ref.Tag, &ref.Content, &ref.FContent, &ref.Markdown, &ref.Length, &ref.Type, &ref.SubType, &ref.IAL, &ref.Sort, &ref.Created, &ref.Updated,
&rel); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
refs[rel] = &ref
}
rows, err = query("SELECT def.* FROM blocks AS def, refs AS r WHERE def.id = r.def_block_id")
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
defs := map[string]*Block{}
for rows.Next() {
if def := scanBlockRows(rows); nil != def {
defs[def.ID] = def
}
}
for rel, ref := range refs {
defID := strings.Split(rel, "@")[1]
def := defs[defID]
if nil == def {
continue
}
defRef := map[*Block]*Block{}
defRef[def] = ref
ret = append(ret, defRef)
}
return
}
func scanRefRows(rows *sql.Rows) (ret *Ref) {
var ref Ref
if err := rows.Scan(&ref.ID, &ref.DefBlockID, &ref.DefBlockParentID, &ref.DefBlockRootID, &ref.DefBlockPath, &ref.BlockID, &ref.RootID, &ref.Box, &ref.Path, &ref.Content, &ref.Markdown, &ref.Type); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret = &ref
return
}

133
kernel/sql/cache.go Normal file
View file

@ -0,0 +1,133 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"runtime/debug"
"time"
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/dgraph-io/ristretto"
gcache "github.com/patrickmn/go-cache"
)
var memCache, _ = ristretto.NewCache(&ristretto.Config{
NumCounters: 100000, // 10W
MaxCost: 1024 * 1024 * 10, // 10MB
BufferItems: 64,
})
var disabled = true
func EnableCache() {
disabled = false
}
func DisableCache() {
disabled = true
}
func ClearBlockCache() {
memCache.Clear()
debug.FreeOSMemory()
}
func putBlockCache(block *Block) {
if disabled {
return
}
memCache.Set(block.ID, block, 1)
}
func getBlockCache(id string) (ret *Block) {
if disabled {
return
}
b, _ := memCache.Get(id)
if nil != b {
ret = b.(*Block)
}
return
}
func removeBlockCache(id string) {
memCache.Del(id)
removeRefCacheByDefID(id)
}
func getVirtualRefKeywordsCache() ([]string, bool) {
if disabled {
return nil, false
}
if val, ok := memCache.Get("virtual_ref"); ok {
return val.([]string), true
}
return nil, false
}
func setVirtualRefKeywords(keywords []string) {
if disabled {
return
}
memCache.Set("virtual_ref", keywords, 1)
}
func ClearVirtualRefKeywords() {
memCache.Del("virtual_ref")
}
func GetRefsCacheByDefID(defID string) (ret []*Ref) {
for defBlockID, refs := range defIDRefsCache.Items() {
if defBlockID == defID {
for _, ref := range refs.Object.(map[string]*Ref) {
ret = append(ret, ref)
}
}
}
if 1 > len(ret) {
ret = QueryRefsByDefID(defID, false)
for _, ref := range ret {
putRefCache(ref)
}
}
return
}
var (
defIDRefsCache = gcache.New(30*time.Minute, 5*time.Minute) // [defBlockID]map[refBlockID]*Ref
)
func CacheRef(tree *parse.Tree, refIDNode *ast.Node) {
ref := buildRef(tree, refIDNode)
putRefCache(ref)
}
func putRefCache(ref *Ref) {
defBlockRefs, ok := defIDRefsCache.Get(ref.DefBlockID)
if !ok {
defBlockRefs = map[string]*Ref{}
}
defBlockRefs.(map[string]*Ref)[ref.BlockID] = ref
defIDRefsCache.SetDefault(ref.DefBlockID, defBlockRefs)
}
func removeRefCacheByDefID(defID string) {
defIDRefsCache.Delete(defID)
}

1152
kernel/sql/database.go Normal file
View file

File diff suppressed because it is too large Load diff

53
kernel/sql/file_annotation_ref.go Normal file
View file

@ -0,0 +1,53 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"github.com/siyuan-note/siyuan/kernel/util"
)
type FileAnnotationRef struct {
ID string
FilePath string
AnnotationID string
BlockID string
RootID string
Box string
Path string
Content string
Type string
}
func QueryRefIDsByAnnotationID(annotationID string) (refIDs, refTexts []string) {
refIDs = []string{}
rows, err := query("SELECT block_id, content FROM file_annotation_refs WHERE annotation_id = ?", annotationID)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
var id, content string
if err = rows.Scan(&id, &content); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
refIDs = append(refIDs, id)
refTexts = append(refTexts, content)
}
return
}

237
kernel/sql/queue.go Normal file
View file

@ -0,0 +1,237 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"bytes"
"crypto/sha256"
"database/sql"
"fmt"
"path"
"sync"
"time"
"github.com/88250/lute/parse"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/siyuan/kernel/util"
)
const (
upsertTreesFlushDelay = 3000
)
var (
operationQueue []*treeQueueOperation
upsertTreeQueueLock = sync.Mutex{}
txLock = sync.Mutex{}
)
type treeQueueOperation struct {
inQueueTime time.Time
action string // upsert/delete/delete_id/rename
upsertTree *parse.Tree // upsert
removeTreeBox, removeTreePath string // delete
removeTreeIDBox, removeTreeID string // delete_id
renameTreeBox, renameTreeID, renameTreeOldHPath, renameTreeNewHPath string // rename
}
func AutoFlushTreeQueue() {
for {
flushTreeQueue()
time.Sleep(time.Duration(upsertTreesFlushDelay) * time.Millisecond)
}
}
func WaitForWritingDatabase() {
var printLog bool
var lastPrintLog bool
for i := 0; isWritingDatabase(); i++ {
time.Sleep(50 * time.Millisecond)
if 200 < i && !printLog { // 10s 后打日志
util.LogWarnf("database is writing: \n%s", util.ShortStack())
printLog = true
}
if 1200 < i && !lastPrintLog { // 60s 后打日志
util.LogWarnf("database is still writing")
lastPrintLog = true
}
}
}
func isWritingDatabase() bool {
time.Sleep(time.Duration(upsertTreesFlushDelay+50) * time.Millisecond)
if 0 < len(operationQueue) || util.IsMutexLocked(&txLock) {
return true
}
return false
}
func flushTreeQueue() {
ops := mergeUpsertTrees()
if 1 > len(ops) {
return
}
txLock.Lock()
defer txLock.Unlock()
start := time.Now()
tx, err := BeginTx()
if nil != err {
return
}
boxes := hashset.New()
for _, op := range ops {
switch op.action {
case "upsert":
tree := op.upsertTree
if err = upsertTree(tx, tree); nil != err {
util.LogErrorf("upsert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
}
boxes.Add(op.upsertTree.Box)
case "delete":
batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath)
boxes.Add(op.removeTreeBox)
case "delete_id":
DeleteByRootID(tx, op.removeTreeID)
boxes.Add(op.removeTreeIDBox)
case "rename":
batchUpdateHPath(tx, op.renameTreeBox, op.renameTreeID, op.renameTreeOldHPath, op.renameTreeNewHPath)
updateRootContent(tx, path.Base(op.renameTreeNewHPath), op.renameTreeID)
boxes.Add(op.renameTreeBox)
default:
util.LogErrorf("unknown operation [%s]", op.action)
}
}
CommitTx(tx)
elapsed := time.Now().Sub(start).Milliseconds()
if 5000 < elapsed {
util.LogInfof("op tx [%dms]", elapsed)
}
start = time.Now()
tx, err = BeginTx()
if nil != err {
return
}
for _, box := range boxes.Values() {
updateBoxHash(tx, box.(string))
}
CommitTx(tx)
elapsed = time.Now().Sub(start).Milliseconds()
if 1000 < elapsed {
util.LogInfof("hash tx [%dms]", elapsed)
}
}
func mergeUpsertTrees() (ops []*treeQueueOperation) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
ops = operationQueue
operationQueue = nil
return
}
func UpsertTreeQueue(tree *parse.Tree) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
newOp := &treeQueueOperation{upsertTree: tree, inQueueTime: time.Now(), action: "upsert"}
for i, op := range operationQueue {
if "upsert" == op.action && op.upsertTree.ID == tree.ID { // 相同树则覆盖
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func RenameTreeQueue(tree *parse.Tree, oldHPath string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
newOp := &treeQueueOperation{renameTreeBox: tree.Box, renameTreeID: tree.ID, renameTreeOldHPath: oldHPath, renameTreeNewHPath: tree.HPath, inQueueTime: time.Now(), action: "rename"}
for i, op := range operationQueue {
if "rename" == op.action && op.renameTreeID == tree.ID { // 相同树则覆盖
operationQueue[i] = newOp
return
}
}
operationQueue = append(operationQueue, newOp)
}
func RemoveTreeQueue(box, rootID string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
var tmp []*treeQueueOperation
// 将已有的 upsert 操作去重
for _, op := range operationQueue {
if "upsert" == op.action && op.upsertTree.ID != rootID {
tmp = append(tmp, op)
}
}
operationQueue = tmp
newOp := &treeQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
operationQueue = append(operationQueue, newOp)
}
func RemoveTreePathQueue(treeBox, treePathPrefix string) {
upsertTreeQueueLock.Lock()
defer upsertTreeQueueLock.Unlock()
var tmp []*treeQueueOperation
// 将已有的 upsert 操作去重
for _, op := range operationQueue {
if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) {
tmp = append(tmp, op)
}
}
operationQueue = tmp
newOp := &treeQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
operationQueue = append(operationQueue, newOp)
}
func updateBoxHash(tx *sql.Tx, boxID string) {
sum := boxChecksum(boxID)
PutBoxHash(tx, boxID, sum)
}
func boxChecksum(box string) (ret string) {
rows, err := query("SELECT hash FROM blocks WHERE type = 'd' AND box = ? ORDER BY id DESC", box)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
buf := bytes.Buffer{}
for rows.Next() {
var hash string
if err = rows.Scan(&hash); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
buf.WriteString(hash)
}
ret = fmt.Sprintf("%x", sha256.Sum256(buf.Bytes()))
return
}

126
kernel/sql/span.go Normal file
View file

@ -0,0 +1,126 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"strconv"
"strings"
"github.com/88250/vitess-sqlparser/sqlparser"
"github.com/siyuan-note/siyuan/kernel/util"
)
type Span struct {
ID string
BlockID string
RootID string
Box string
Path string
Content string
Markdown string
Type string
IAL string
}
func SelectSpansRawStmt(stmt string, limit int) (ret []*Span) {
parsedStmt, err := sqlparser.Parse(stmt)
if nil != err {
//util.LogErrorf("select [%s] failed: %s", stmt, err)
return
}
switch parsedStmt.(type) {
case *sqlparser.Select:
slct := parsedStmt.(*sqlparser.Select)
if nil == slct.Limit {
slct.Limit = &sqlparser.Limit{
Rowcount: &sqlparser.SQLVal{
Type: sqlparser.IntVal,
Val: []byte(strconv.Itoa(limit)),
},
}
}
stmt = sqlparser.String(slct)
default:
return
}
stmt = strings.ReplaceAll(stmt, "\\'", "''")
stmt = strings.ReplaceAll(stmt, "\\\"", "\"")
stmt = strings.ReplaceAll(stmt, "\\\\*", "\\*")
stmt = strings.ReplaceAll(stmt, "from dual", "")
rows, err := query(stmt)
if nil != err {
if strings.Contains(err.Error(), "syntax error") {
return
}
util.LogWarnf("sql query [%s] failed: %s", stmt, err)
return
}
defer rows.Close()
for rows.Next() {
span := scanSpanRows(rows)
ret = append(ret, span)
}
return
}
func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) {
stmt := "SELECT * FROM spans WHERE type = 'tag' AND content LIKE '%" + keyword + "%'"
stmt += " LIMIT " + strconv.Itoa(limit)
rows, err := query(stmt)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
span := scanSpanRows(rows)
ret = append(ret, span)
}
return
}
func QueryTagSpans(p string, limit int) (ret []*Span) {
stmt := "SELECT * FROM spans WHERE type = 'tag'"
if "" != p {
stmt += " AND path = '" + p + "'"
}
stmt += " LIMIT " + strconv.Itoa(limit)
rows, err := query(stmt)
if nil != err {
util.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
span := scanSpanRows(rows)
ret = append(ret, span)
}
return
}
func scanSpanRows(rows *sql.Rows) (ret *Span) {
var span Span
if err := rows.Scan(&span.ID, &span.BlockID, &span.RootID, &span.Box, &span.Path, &span.Content, &span.Markdown, &span.Type, &span.IAL); nil != err {
util.LogErrorf("query scan field failed: %s", err)
return
}
ret = &span
return
}

100
kernel/sql/stat.go Normal file
View file

@ -0,0 +1,100 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"strings"
"github.com/siyuan-note/siyuan/kernel/util"
)
type Stat struct {
Key string `json:"key"`
Val string `json:"value"`
}
func getDatabaseVer() (ret string) {
key := "siyuan_database_ver"
stmt := "SELECT value FROM stat WHERE `key` = ?"
row := db.QueryRow(stmt, key)
if err := row.Scan(&ret); nil != err {
if !strings.Contains(err.Error(), "no such table") {
util.LogErrorf("query database version failed: %s", err)
}
}
return
}
func setDatabaseVer() {
key := "siyuan_database_ver"
tx, err := BeginTx()
if nil != err {
return
}
if err = putStat(tx, key, util.DatabaseVer); nil != err {
RollbackTx(tx)
return
}
CommitTx(tx)
}
func ClearBoxHash(tx *sql.Tx) {
stmt := "DELETE FROM stat WHERE `key` LIKE '%_hash'"
execStmtTx(tx, stmt)
}
func RemoveBoxHash(tx *sql.Tx, box string) {
key := box + "_hash"
stmt := "DELETE FROM stat WHERE `key` = '" + key + "'"
execStmtTx(tx, stmt)
}
func PutBoxHash(tx *sql.Tx, box, hash string) {
key := box + "_hash"
putStat(tx, key, hash)
}
func GetBoxHash(box string) string {
key := box + "_hash"
return getStat(key)
}
func putStat(tx *sql.Tx, key, value string) (err error) {
stmt := "DELETE FROM stat WHERE `key` = '" + key + "'"
if err = execStmtTx(tx, stmt); nil != err {
return
}
stmt = "INSERT INTO stat VALUES ('" + key + "', '" + value + "')"
err = execStmtTx(tx, stmt)
return
}
func getStat(key string) (ret string) {
stmt := "SELECT value FROM stat WHERE `key` = '" + key + "'"
row := queryRow(stmt)
row.Scan(&ret)
return
}
func CountAllDoc() (ret int) {
sqlStmt := "SELECT count(*) FROM blocks WHERE type = 'd'"
row := queryRow(sqlStmt)
row.Scan(&ret)
return
}

508
kernel/sql/upsert.go Normal file
View file

@ -0,0 +1,508 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"fmt"
"strings"
"github.com/88250/lute/parse"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
var luteEngine = util.NewLute()
func init() {
luteEngine.RenderOptions.KramdownBlockIAL = false // 数据库 markdown 字段为标准 md，但是要保留 span block ial
}
func InsertBlocksSpans(tx *sql.Tx, tree *parse.Tree) (err error) {
if err = insertBlocksSpans(tx, tree); nil != err {
util.LogErrorf("insert tree [%s] into database failed: %s", tree.Box+tree.Path, err)
}
return
}
func InsertRefs(tx *sql.Tx, tree *parse.Tree) {
if err := insertRef(tx, tree); nil != err {
util.LogErrorf("insert refs tree [%s] into database failed: %s", tree.Box+tree.Path, err)
}
}
const (
BlocksInsert = "INSERT INTO blocks (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
BlocksFTSInsert = "INSERT INTO blocks_fts (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
BlocksFTSCaseInsensitiveInsert = "INSERT INTO blocks_fts_case_insensitive (id, parent_id, root_id, hash, box, path, hpath, name, alias, memo, tag, content, fcontent, markdown, length, type, subtype, ial, sort, created, updated) VALUES %s"
BlocksPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
SpansInsert = "INSERT INTO spans (id, block_id, root_id, box, path, content, markdown, type, ial) VALUES %s"
SpansPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
AssetsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
AttributesPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?)"
RefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
FileAnnotationRefsPlaceholder = "(?, ?, ?, ?, ?, ?, ?, ?, ?)"
)
func insertBlocks(tx *sql.Tx, blocks []*Block) (err error) {
if 1 > len(blocks) {
return
}
var bulk []*Block
for _, block := range blocks {
bulk = append(bulk, block)
if 512 > len(bulk) {
continue
}
if err = insertBlocks0(tx, bulk); nil != err {
return
}
bulk = []*Block{}
}
if 0 < len(bulk) {
if err = insertBlocks0(tx, bulk); nil != err {
return
}
}
return
}
func insertBlocks0(tx *sql.Tx, bulk []*Block) (err error) {
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(BlocksPlaceholder, "?"))
for _, b := range bulk {
valueStrings = append(valueStrings, BlocksPlaceholder)
valueArgs = append(valueArgs, b.ID)
valueArgs = append(valueArgs, b.ParentID)
valueArgs = append(valueArgs, b.RootID)
valueArgs = append(valueArgs, b.Hash)
valueArgs = append(valueArgs, b.Box)
valueArgs = append(valueArgs, b.Path)
valueArgs = append(valueArgs, b.HPath)
valueArgs = append(valueArgs, b.Name)
valueArgs = append(valueArgs, b.Alias)
valueArgs = append(valueArgs, b.Memo)
valueArgs = append(valueArgs, b.Tag)
valueArgs = append(valueArgs, b.Content)
valueArgs = append(valueArgs, b.FContent)
valueArgs = append(valueArgs, b.Markdown)
valueArgs = append(valueArgs, b.Length)
valueArgs = append(valueArgs, b.Type)
valueArgs = append(valueArgs, b.SubType)
valueArgs = append(valueArgs, b.IAL)
valueArgs = append(valueArgs, b.Sort)
valueArgs = append(valueArgs, b.Created)
valueArgs = append(valueArgs, b.Updated)
putBlockCache(b)
}
stmt := fmt.Sprintf(BlocksInsert, strings.Join(valueStrings, ","))
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
}
stmt = fmt.Sprintf(BlocksFTSInsert, strings.Join(valueStrings, ","))
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
}
stmt = fmt.Sprintf(BlocksFTSCaseInsensitiveInsert, strings.Join(valueStrings, ","))
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
}
return
}
func insertAttributes(tx *sql.Tx, attributes []*Attribute) (err error) {
if 1 > len(attributes) {
return
}
var bulk []*Attribute
for _, attr := range attributes {
bulk = append(bulk, attr)
if 512 > len(bulk) {
continue
}
if err = insertAttribute0(tx, bulk); nil != err {
return
}
bulk = []*Attribute{}
}
if 0 < len(bulk) {
if err = insertAttribute0(tx, bulk); nil != err {
return
}
}
return
}
func insertAttribute0(tx *sql.Tx, bulk []*Attribute) (err error) {
if 1 > len(bulk) {
return
}
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AttributesPlaceholder, "?"))
for _, attr := range bulk {
valueStrings = append(valueStrings, AttributesPlaceholder)
valueArgs = append(valueArgs, attr.ID)
valueArgs = append(valueArgs, attr.Name)
valueArgs = append(valueArgs, attr.Value)
valueArgs = append(valueArgs, attr.Type)
valueArgs = append(valueArgs, attr.BlockID)
valueArgs = append(valueArgs, attr.RootID)
valueArgs = append(valueArgs, attr.Box)
valueArgs = append(valueArgs, attr.Path)
}
stmt := fmt.Sprintf("INSERT INTO attributes (id, name, value, type, block_id, root_id, box, path) VALUES %s", strings.Join(valueStrings, ","))
err = prepareExecInsertTx(tx, stmt, valueArgs)
return
}
func insertAssets(tx *sql.Tx, assets []*Asset) (err error) {
if 1 > len(assets) {
return
}
var bulk []*Asset
for _, asset := range assets {
bulk = append(bulk, asset)
if 512 > len(bulk) {
continue
}
if err = insertAsset0(tx, bulk); nil != err {
return
}
bulk = []*Asset{}
}
if 0 < len(bulk) {
if err = insertAsset0(tx, bulk); nil != err {
return
}
}
return
}
func insertAsset0(tx *sql.Tx, bulk []*Asset) (err error) {
if 1 > len(bulk) {
return
}
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AssetsPlaceholder, "?"))
for _, asset := range bulk {
valueStrings = append(valueStrings, AssetsPlaceholder)
valueArgs = append(valueArgs, asset.ID)
valueArgs = append(valueArgs, asset.BlockID)
valueArgs = append(valueArgs, asset.RootID)
valueArgs = append(valueArgs, asset.Box)
valueArgs = append(valueArgs, asset.DocPath)
valueArgs = append(valueArgs, asset.Path)
valueArgs = append(valueArgs, asset.Name)
valueArgs = append(valueArgs, asset.Title)
valueArgs = append(valueArgs, asset.Hash)
}
stmt := fmt.Sprintf("INSERT INTO assets (id, block_id, root_id, box, docpath, path, name, title, hash) VALUES %s", strings.Join(valueStrings, ","))
err = prepareExecInsertTx(tx, stmt, valueArgs)
return
}
func insertSpans(tx *sql.Tx, spans []*Span) (err error) {
if 1 > len(spans) {
return
}
var bulk []*Span
for _, span := range spans {
bulk = append(bulk, span)
if 512 > len(bulk) {
continue
}
if err = insertSpans0(tx, bulk); nil != err {
return
}
bulk = []*Span{}
}
if 0 < len(bulk) {
if err = insertSpans0(tx, bulk); nil != err {
return
}
}
return
}
func insertSpans0(tx *sql.Tx, bulk []*Span) (err error) {
if 1 > len(bulk) {
return
}
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(SpansPlaceholder, "?"))
for _, span := range bulk {
valueStrings = append(valueStrings, SpansPlaceholder)
valueArgs = append(valueArgs, span.ID)
valueArgs = append(valueArgs, span.BlockID)
valueArgs = append(valueArgs, span.RootID)
valueArgs = append(valueArgs, span.Box)
valueArgs = append(valueArgs, span.Path)
valueArgs = append(valueArgs, span.Content)
valueArgs = append(valueArgs, span.Markdown)
valueArgs = append(valueArgs, span.Type)
valueArgs = append(valueArgs, span.IAL)
}
stmt := fmt.Sprintf(SpansInsert, strings.Join(valueStrings, ","))
err = prepareExecInsertTx(tx, stmt, valueArgs)
return
}
func insertRefs(tx *sql.Tx, refs []*Ref) (err error) {
if 1 > len(refs) {
return
}
var bulk []*Ref
for _, ref := range refs {
bulk = append(bulk, ref)
if 512 > len(bulk) {
continue
}
if err = insertRefs0(tx, bulk); nil != err {
return
}
bulk = []*Ref{}
}
if 0 < len(bulk) {
if err = insertRefs0(tx, bulk); nil != err {
return
}
}
return
}
func insertRefs0(tx *sql.Tx, bulk []*Ref) (err error) {
if 1 > len(bulk) {
return
}
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(RefsPlaceholder, "?"))
for _, ref := range bulk {
valueStrings = append(valueStrings, RefsPlaceholder)
valueArgs = append(valueArgs, ref.ID)
valueArgs = append(valueArgs, ref.DefBlockID)
valueArgs = append(valueArgs, ref.DefBlockParentID)
valueArgs = append(valueArgs, ref.DefBlockRootID)
valueArgs = append(valueArgs, ref.DefBlockPath)
valueArgs = append(valueArgs, ref.BlockID)
valueArgs = append(valueArgs, ref.RootID)
valueArgs = append(valueArgs, ref.Box)
valueArgs = append(valueArgs, ref.Path)
valueArgs = append(valueArgs, ref.Content)
valueArgs = append(valueArgs, ref.Markdown)
valueArgs = append(valueArgs, ref.Type)
putRefCache(ref)
}
stmt := fmt.Sprintf("INSERT INTO refs (id, def_block_id, def_block_parent_id, def_block_root_id, def_block_path, block_id, root_id, box, path, content, markdown, type) VALUES %s", strings.Join(valueStrings, ","))
err = prepareExecInsertTx(tx, stmt, valueArgs)
return
}
func insertFileAnnotationRefs(tx *sql.Tx, refs []*FileAnnotationRef) (err error) {
if 1 > len(refs) {
return
}
var bulk []*FileAnnotationRef
for _, ref := range refs {
bulk = append(bulk, ref)
if 512 > len(bulk) {
continue
}
if err = insertFileAnnotationRefs0(tx, bulk); nil != err {
return
}
bulk = []*FileAnnotationRef{}
}
if 0 < len(bulk) {
if err = insertFileAnnotationRefs0(tx, bulk); nil != err {
return
}
}
return
}
func insertFileAnnotationRefs0(tx *sql.Tx, bulk []*FileAnnotationRef) (err error) {
if 1 > len(bulk) {
return
}
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(FileAnnotationRefsPlaceholder, "?"))
for _, ref := range bulk {
valueStrings = append(valueStrings, FileAnnotationRefsPlaceholder)
valueArgs = append(valueArgs, ref.ID)
valueArgs = append(valueArgs, ref.FilePath)
valueArgs = append(valueArgs, ref.AnnotationID)
valueArgs = append(valueArgs, ref.BlockID)
valueArgs = append(valueArgs, ref.RootID)
valueArgs = append(valueArgs, ref.Box)
valueArgs = append(valueArgs, ref.Path)
valueArgs = append(valueArgs, ref.Content)
valueArgs = append(valueArgs, ref.Type)
}
stmt := fmt.Sprintf("INSERT INTO file_annotation_refs (id, file_path, annotation_id, block_id, root_id, box, path, content, type) VALUES %s", strings.Join(valueStrings, ","))
err = prepareExecInsertTx(tx, stmt, valueArgs)
return
}
func insertBlocksSpans(tx *sql.Tx, tree *parse.Tree) (err error) {
blocks, spans, assets, attributes := fromTree(tree.Root, tree)
if err = insertBlocks(tx, blocks); nil != err {
return
}
if err = insertSpans(tx, spans); nil != err {
return
}
if err = insertAssets(tx, assets); nil != err {
return
}
if err = insertAttributes(tx, attributes); nil != err {
return
}
return
}
func insertRef(tx *sql.Tx, tree *parse.Tree) (err error) {
refs, fileAnnotationRefs := refsFromTree(tree)
if err = insertRefs(tx, refs); nil != err {
return
}
if err = insertFileAnnotationRefs(tx, fileAnnotationRefs); nil != err {
return
}
return err
}
func upsertTree(tx *sql.Tx, tree *parse.Tree) (err error) {
oldBlockHashes := queryBlockHashes(tree.ID)
blocks, spans, assets, attributes := fromTree(tree.Root, tree)
newBlockHashes := map[string]string{}
for _, block := range blocks {
newBlockHashes[block.ID] = block.Hash
}
unChanges := hashset.New()
var toRemoves []string
for id, hash := range oldBlockHashes {
if newHash, ok := newBlockHashes[id]; ok {
if newHash == hash {
unChanges.Add(id)
}
} else {
toRemoves = append(toRemoves, id)
}
}
tmp := blocks[:0]
for _, b := range blocks {
if !unChanges.Contains(b.ID) {
tmp = append(tmp, b)
}
}
blocks = tmp
for _, b := range blocks {
toRemoves = append(toRemoves, b.ID)
}
deleteBlocksByIDs(tx, toRemoves)
if err = deleteSpansByPathTx(tx, tree.Box, tree.Path); nil != err {
return
}
if err = deleteAssetsByPathTx(tx, tree.Box, tree.Path); nil != err {
return
}
if err = deleteAttributesByPathTx(tx, tree.Box, tree.Path); nil != err {
return
}
if err = deleteRefsByPathTx(tx, tree.Box, tree.Path); nil != err {
return
}
if err = deleteFileAnnotationRefsByPathTx(tx, tree.Box, tree.Path); nil != err {
return
}
if err = insertBlocks(tx, blocks); nil != err {
return
}
anchors := map[string]string{}
var refIDs []string
for _, block := range blocks {
if "" != block.Content {
// content 不为空的话说明是定值，不需要解析引用内容
continue
}
subTree := parse.Parse("", []byte(block.Markdown), luteEngine.ParseOptions)
if nil == subTree {
util.LogErrorf("parse temp block [%s] failed: %s", block.ID, err)
continue
}
if 0 < len(treenode.GetLegacyDynamicBlockRefDefIDs(subTree.Root)) {
refIDs = append(refIDs, block.ID)
}
}
// 先删除再插入会快很多
refBlocks := GetBlocks(refIDs)
for _, refBlock := range refBlocks {
blockContent := ResolveRefContent(refBlock, &anchors)
refBlock.Content = blockContent
}
deleteBlocksByIDs(tx, refIDs)
insertBlocks(tx, refBlocks)
refs, fileAnnotationRefs := refsFromTree(tree)
if err = insertRefs(tx, refs); nil != err {
return
}
if err = insertFileAnnotationRefs(tx, fileAnnotationRefs); nil != err {
return
}
if 0 < len(spans) {
// 移除文档标签，否则会重复添加 https://github.com/siyuan-note/siyuan/issues/3723
if err = deleteSpansByRootID(tx, tree.Root.ID); nil != err {
return
}
if err = insertSpans(tx, spans); nil != err {
return
}
}
if err = insertAssets(tx, assets); nil != err {
return
}
if err = insertAttributes(tx, attributes); nil != err {
return
}
return err
}