siyuan/kernel/model/storage.go
2025-11-09 21:57:45 +08:00

831 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// SiYuan - Refactor your thinking
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package model
import (
"errors"
"os"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"github.com/88250/gulu"
"github.com/88250/lute/parse"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
type RecentDoc struct {
RootID string `json:"rootID"`
Icon string `json:"icon,omitempty"`
Title string `json:"title,omitempty"`
ViewedAt int64 `json:"viewedAt"` // 浏览时间字段
ClosedAt int64 `json:"closedAt"` // 关闭时间字段
OpenAt int64 `json:"openAt"` // 文档第一次从文档树加载到页签的时间
}
type OutlineDoc struct {
DocID string `json:"docID"`
Data map[string]interface{} `json:"data"`
}
var recentDocLock = sync.Mutex{}
// 三种类型各保留 32 条记录
func trimRecentDocs(recentDocs []*RecentDoc) []*RecentDoc {
if len(recentDocs) <= 32 {
return recentDocs
}
// 分别统计三种类型的记录
var viewedDocs []*RecentDoc
var openedDocs []*RecentDoc
var closedDocs []*RecentDoc
for _, doc := range recentDocs {
if doc.ViewedAt > 0 {
viewedDocs = append(viewedDocs, doc)
}
if doc.OpenAt > 0 {
openedDocs = append(openedDocs, doc)
}
if doc.ClosedAt > 0 {
closedDocs = append(closedDocs, doc)
}
}
// 分别按时间排序并截取 32 条记录
if len(viewedDocs) > 32 {
sort.Slice(viewedDocs, func(i, j int) bool {
return viewedDocs[i].ViewedAt > viewedDocs[j].ViewedAt
})
viewedDocs = viewedDocs[:32]
}
if len(openedDocs) > 32 {
sort.Slice(openedDocs, func(i, j int) bool {
return openedDocs[i].OpenAt > openedDocs[j].OpenAt
})
openedDocs = openedDocs[:32]
}
if len(closedDocs) > 32 {
sort.Slice(closedDocs, func(i, j int) bool {
return closedDocs[i].ClosedAt > closedDocs[j].ClosedAt
})
closedDocs = closedDocs[:32]
}
// 合并三类记录
docMap := make(map[string]*RecentDoc, 64)
for _, doc := range viewedDocs {
docMap[doc.RootID] = doc
}
for _, doc := range openedDocs {
if _, ok := docMap[doc.RootID]; !ok {
docMap[doc.RootID] = doc
}
}
for _, doc := range closedDocs {
if _, ok := docMap[doc.RootID]; !ok {
docMap[doc.RootID] = doc
}
}
result := make([]*RecentDoc, 0, len(docMap))
for _, doc := range docMap {
result = append(result, doc)
}
return result
}
func RemoveRecentDoc(ids []string) {
recentDocLock.Lock()
defer recentDocLock.Unlock()
recentDocs, err := getRecentDocs("")
if err != nil {
return
}
ids = gulu.Str.RemoveDuplicatedElem(ids)
for i, doc := range recentDocs {
if gulu.Str.Contains(doc.RootID, ids) {
recentDocs = append(recentDocs[:i], recentDocs[i+1:]...)
break
}
}
err = setRecentDocs(recentDocs)
if err != nil {
return
}
}
func setRecentDocByTree(tree *parse.Tree) {
recentDoc := &RecentDoc{
RootID: tree.Root.ID,
Icon: "",
Title: "",
ViewedAt: time.Now().Unix(), // 使用当前时间作为浏览时间
ClosedAt: 0, // 初始化关闭时间为0表示未关闭
OpenAt: time.Now().Unix(), // 设置文档打开时间
}
recentDocLock.Lock()
defer recentDocLock.Unlock()
recentDocs, err := getRecentDocs("")
if err != nil {
return
}
for i, c := range recentDocs {
if c.RootID == recentDoc.RootID {
recentDoc.ClosedAt = c.ClosedAt
recentDocs = append(recentDocs[:i], recentDocs[i+1:]...)
break
}
}
recentDocs = append([]*RecentDoc{recentDoc}, recentDocs...)
recentDocs = trimRecentDocs(recentDocs)
err = setRecentDocs(recentDocs)
if err != nil {
return
}
}
// UpdateRecentDocOpenTime 更新文档打开时间(只在第一次从文档树加载到页签时调用)
func UpdateRecentDocOpenTime(rootID string) (err error) {
recentDocLock.Lock()
defer recentDocLock.Unlock()
recentDocs, err := getRecentDocs("")
if err != nil {
return
}
// 查找文档并更新打开时间
found := false
for _, doc := range recentDocs {
if doc.RootID == rootID {
doc.OpenAt = time.Now().Unix()
doc.ClosedAt = 0
found = true
break
}
}
if found {
err = setRecentDocs(recentDocs)
}
return
}
// UpdateRecentDocViewTime 更新文档浏览时间
func UpdateRecentDocViewTime(rootID string) (err error) {
recentDocLock.Lock()
defer recentDocLock.Unlock()
recentDocs, err := getRecentDocs("")
if err != nil {
return
}
// 查找文档并更新浏览时间
found := false
for _, doc := range recentDocs {
if doc.RootID == rootID {
doc.ViewedAt = time.Now().Unix()
doc.ClosedAt = 0
found = true
break
}
}
if found {
// 按浏览时间降序排序
sort.Slice(recentDocs, func(i, j int) bool {
return recentDocs[i].ViewedAt > recentDocs[j].ViewedAt
})
err = setRecentDocs(recentDocs)
}
return
}
// UpdateRecentDocCloseTime 更新文档关闭时间
func UpdateRecentDocCloseTime(rootID string) (err error) {
return BatchUpdateRecentDocCloseTime([]string{rootID})
}
// BatchUpdateRecentDocCloseTime 批量更新文档关闭时间
func BatchUpdateRecentDocCloseTime(rootIDs []string) (err error) {
if len(rootIDs) == 0 {
return
}
recentDocLock.Lock()
defer recentDocLock.Unlock()
recentDocs, err := getRecentDocs("")
if err != nil {
return
}
rootIDs = gulu.Str.RemoveDuplicatedElem(rootIDs)
rootIDsMap := make(map[string]bool, len(rootIDs))
for _, id := range rootIDs {
rootIDsMap[id] = true
}
closeTime := time.Now().Unix()
// 更新已存在的文档
updated := false
for _, doc := range recentDocs {
if rootIDsMap[doc.RootID] {
doc.ClosedAt = closeTime
updated = true
delete(rootIDsMap, doc.RootID) // 标记已处理
}
}
// 为不存在的文档创建新记录
for rootID := range rootIDsMap {
tree, loadErr := LoadTreeByBlockID(rootID)
if loadErr != nil {
continue
}
recentDoc := &RecentDoc{
RootID: tree.Root.ID,
Icon: tree.Root.IALAttr("icon"),
Title: tree.Root.IALAttr("title"),
ViewedAt: 0, // 未浏览过,设为 0
ClosedAt: closeTime, // 设置关闭时间
OpenAt: 0, // 未记录打开时间,设为 0
}
recentDocs = append([]*RecentDoc{recentDoc}, recentDocs...)
updated = true
}
if updated {
err = setRecentDocs(recentDocs)
}
return
}
func GetRecentDocs(sortBy string) (ret []*RecentDoc, err error) {
recentDocLock.Lock()
defer recentDocLock.Unlock()
return getRecentDocs(sortBy)
}
func setRecentDocs(recentDocs []*RecentDoc) (err error) {
dirPath := filepath.Join(util.DataDir, "storage")
if err = os.MkdirAll(dirPath, 0755); err != nil {
logging.LogErrorf("create storage [recent-doc] dir failed: %s", err)
return
}
// 不保存 Title 和 Icon
for _, doc := range recentDocs {
doc.Title = ""
doc.Icon = ""
}
data, err := gulu.JSON.MarshalIndentJSON(recentDocs, "", " ")
if err != nil {
logging.LogErrorf("marshal storage [recent-doc] failed: %s", err)
return
}
lsPath := filepath.Join(dirPath, "recent-doc.json")
err = filelock.WriteFile(lsPath, data)
if err != nil {
logging.LogErrorf("write storage [recent-doc] failed: %s", err)
return
}
return
}
func getRecentDocs(sortBy string) (ret []*RecentDoc, err error) {
var tmp []*RecentDoc
dataPath := filepath.Join(util.DataDir, "storage/recent-doc.json")
if !filelock.IsExist(dataPath) {
return
}
data, err := filelock.ReadFile(dataPath)
if err != nil {
logging.LogErrorf("read storage [recent-doc] failed: %s", err)
return
}
if err = gulu.JSON.UnmarshalJSON(data, &tmp); err != nil {
logging.LogErrorf("unmarshal storage [recent-doc] failed: %s", err)
if err = setRecentDocs([]*RecentDoc{}); err != nil {
logging.LogErrorf("reset storage [recent-doc] failed: %s", err)
}
ret = []*RecentDoc{}
return
}
var rootIDs []string
for _, doc := range tmp {
rootIDs = append(rootIDs, doc.RootID)
}
bts := treenode.GetBlockTrees(rootIDs)
var notExists []string
for _, doc := range tmp {
if bt := bts[doc.RootID]; nil != bt {
// 获取最新的文档标题和图标
doc.Title = path.Base(bt.HPath) // Recent docs not updated after renaming https://github.com/siyuan-note/siyuan/issues/7827
ial := sql.GetBlockAttrs(doc.RootID)
if "" != ial["icon"] {
doc.Icon = ial["icon"]
}
ret = append(ret, doc)
} else {
notExists = append(notExists, doc.RootID)
}
}
if 0 < len(notExists) {
err := setRecentDocs(ret)
if err != nil {
return nil, err
}
}
// 根据排序参数进行排序
switch sortBy {
case "updated": // 按更新时间排序
// 从数据库查询最近修改的文档
sqlBlocks := sql.SelectBlocksRawStmt("SELECT * FROM blocks WHERE type = 'd' ORDER BY updated DESC", 1, 32)
ret = []*RecentDoc{}
if 1 > len(sqlBlocks) {
return
}
// 获取文档树信息
var rootIDs []string
for _, sqlBlock := range sqlBlocks {
rootIDs = append(rootIDs, sqlBlock.ID)
}
bts := treenode.GetBlockTrees(rootIDs)
for _, sqlBlock := range sqlBlocks {
// 解析 IAL 获取 icon
icon := ""
if sqlBlock.IAL != "" {
ialStr := strings.TrimPrefix(sqlBlock.IAL, "{:")
ialStr = strings.TrimSuffix(ialStr, "}")
ial := parse.Tokens2IAL([]byte(ialStr))
for _, kv := range ial {
if kv[0] == "icon" {
icon = kv[1]
break
}
}
}
// 获取文档标题
title := ""
if bt := bts[sqlBlock.ID]; nil != bt {
title = path.Base(bt.HPath)
}
if title == "" {
title = sqlBlock.Content
if title == "" {
title = sqlBlock.HPath
if title == "" {
title = sqlBlock.ID
}
}
}
doc := &RecentDoc{
RootID: sqlBlock.ID,
Icon: icon,
Title: title,
}
ret = append(ret, doc)
}
case "closedAt": // 按关闭时间排序
var filtered []*RecentDoc
for _, doc := range ret {
if doc.ClosedAt > 0 {
filtered = append(filtered, doc)
}
}
ret = filtered
sort.Slice(ret, func(i, j int) bool {
return ret[i].ClosedAt > ret[j].ClosedAt
})
case "openAt": // 按打开时间排序
var filtered []*RecentDoc
for _, doc := range ret {
if doc.OpenAt > 0 {
filtered = append(filtered, doc)
}
}
ret = filtered
sort.Slice(ret, func(i, j int) bool {
return ret[i].OpenAt > ret[j].OpenAt
})
default: // 默认按浏览时间排序
var filtered []*RecentDoc
for _, doc := range ret {
if doc.ViewedAt > 0 {
filtered = append(filtered, doc)
}
}
ret = filtered
sort.Slice(ret, func(i, j int) bool {
return ret[i].ViewedAt > ret[j].ViewedAt
})
}
return
}
type Criterion struct {
Name string `json:"name"`
Sort int `json:"sort"` // 0按块类型默认1按创建时间升序2按创建时间降序3按更新时间升序4按更新时间降序5按内容顺序仅在按文档分组时
Group int `json:"group"` // 0不分组1按文档分组
HasReplace bool `json:"hasReplace"` // 是否有替换
Method int `json:"method"` // 0文本1查询语法2SQL3正则表达式
HPath string `json:"hPath"`
IDPath []string `json:"idPath"`
K string `json:"k"` // 搜索关键字
R string `json:"r"` // 替换关键字
Types *CriterionTypes `json:"types"` // 类型过滤选项
ReplaceTypes *CriterionReplaceTypes `json:"replaceTypes"` // 替换类型过滤选项
}
type CriterionTypes struct {
MathBlock bool `json:"mathBlock"`
Table bool `json:"table"`
Blockquote bool `json:"blockquote"`
SuperBlock bool `json:"superBlock"`
Paragraph bool `json:"paragraph"`
Document bool `json:"document"`
Heading bool `json:"heading"`
List bool `json:"list"`
ListItem bool `json:"listItem"`
CodeBlock bool `json:"codeBlock"`
HtmlBlock bool `json:"htmlBlock"`
EmbedBlock bool `json:"embedBlock"`
DatabaseBlock bool `json:"databaseBlock"`
AudioBlock bool `json:"audioBlock"`
VideoBlock bool `json:"videoBlock"`
IFrameBlock bool `json:"iframeBlock"`
WidgetBlock bool `json:"widgetBlock"`
}
type CriterionReplaceTypes struct {
Text bool `json:"text"`
ImgText bool `json:"imgText"`
ImgTitle bool `json:"imgTitle"`
ImgSrc bool `json:"imgSrc"`
AText bool `json:"aText"`
ATitle bool `json:"aTitle"`
AHref bool `json:"aHref"`
Code bool `json:"code"`
Em bool `json:"em"`
Strong bool `json:"strong"`
InlineMath bool `json:"inlineMath"`
InlineMemo bool `json:"inlineMemo"`
BlockRef bool `json:"blockRef"`
FileAnnotationRef bool `json:"fileAnnotationRef"`
Kbd bool `json:"kbd"`
Mark bool `json:"mark"`
S bool `json:"s"`
Sub bool `json:"sub"`
Sup bool `json:"sup"`
Tag bool `json:"tag"`
U bool `json:"u"`
DocTitle bool `json:"docTitle"`
CodeBlock bool `json:"codeBlock"`
MathBlock bool `json:"mathBlock"`
HtmlBlock bool `json:"htmlBlock"`
}
var criteriaLock = sync.Mutex{}
func RemoveCriterion(name string) (err error) {
criteriaLock.Lock()
defer criteriaLock.Unlock()
criteria, err := getCriteria()
if err != nil {
return
}
for i, c := range criteria {
if c.Name == name {
criteria = append(criteria[:i], criteria[i+1:]...)
break
}
}
err = setCriteria(criteria)
return
}
func SetCriterion(criterion *Criterion) (err error) {
if "" == criterion.Name {
return errors.New(Conf.Language(142))
}
criteriaLock.Lock()
defer criteriaLock.Unlock()
criteria, err := getCriteria()
if err != nil {
return
}
update := false
for i, c := range criteria {
if c.Name == criterion.Name {
criteria[i] = criterion
update = true
break
}
}
if !update {
criteria = append(criteria, criterion)
}
err = setCriteria(criteria)
return
}
func GetCriteria() (ret []*Criterion) {
criteriaLock.Lock()
defer criteriaLock.Unlock()
ret, _ = getCriteria()
return
}
func setCriteria(criteria []*Criterion) (err error) {
dirPath := filepath.Join(util.DataDir, "storage")
if err = os.MkdirAll(dirPath, 0755); err != nil {
logging.LogErrorf("create storage [criteria] dir failed: %s", err)
return
}
data, err := gulu.JSON.MarshalIndentJSON(criteria, "", " ")
if err != nil {
logging.LogErrorf("marshal storage [criteria] failed: %s", err)
return
}
lsPath := filepath.Join(dirPath, "criteria.json")
err = filelock.WriteFile(lsPath, data)
if err != nil {
logging.LogErrorf("write storage [criteria] failed: %s", err)
return
}
return
}
func getCriteria() (ret []*Criterion, err error) {
ret = []*Criterion{}
dataPath := filepath.Join(util.DataDir, "storage/criteria.json")
if !filelock.IsExist(dataPath) {
return
}
data, err := filelock.ReadFile(dataPath)
if err != nil {
logging.LogErrorf("read storage [criteria] failed: %s", err)
return
}
if err = gulu.JSON.UnmarshalJSON(data, &ret); err != nil {
logging.LogErrorf("unmarshal storage [criteria] failed: %s", err)
return
}
return
}
var localStorageLock = sync.Mutex{}
func RemoveLocalStorageVals(keys []string) (err error) {
localStorageLock.Lock()
defer localStorageLock.Unlock()
localStorage := getLocalStorage()
for _, key := range keys {
delete(localStorage, key)
}
return setLocalStorage(localStorage)
}
func SetLocalStorageVal(key string, val interface{}) (err error) {
localStorageLock.Lock()
defer localStorageLock.Unlock()
localStorage := getLocalStorage()
localStorage[key] = val
return setLocalStorage(localStorage)
}
func SetLocalStorage(val interface{}) (err error) {
localStorageLock.Lock()
defer localStorageLock.Unlock()
return setLocalStorage(val)
}
func GetLocalStorage() (ret map[string]interface{}) {
localStorageLock.Lock()
defer localStorageLock.Unlock()
return getLocalStorage()
}
func setLocalStorage(val interface{}) (err error) {
dirPath := filepath.Join(util.DataDir, "storage")
if err = os.MkdirAll(dirPath, 0755); err != nil {
logging.LogErrorf("create storage [local] dir failed: %s", err)
return
}
data, err := gulu.JSON.MarshalIndentJSON(val, "", " ")
if err != nil {
logging.LogErrorf("marshal storage [local] failed: %s", err)
return
}
lsPath := filepath.Join(dirPath, "local.json")
err = filelock.WriteFile(lsPath, data)
if err != nil {
logging.LogErrorf("write storage [local] failed: %s", err)
return
}
return
}
func getLocalStorage() (ret map[string]interface{}) {
// When local.json is corrupted, clear the file to avoid being unable to enter the main interface https://github.com/siyuan-note/siyuan/issues/7911
ret = map[string]interface{}{}
lsPath := filepath.Join(util.DataDir, "storage/local.json")
if !filelock.IsExist(lsPath) {
return
}
data, err := filelock.ReadFile(lsPath)
if err != nil {
logging.LogErrorf("read storage [local] failed: %s", err)
return
}
if err = gulu.JSON.UnmarshalJSON(data, &ret); err != nil {
logging.LogErrorf("unmarshal storage [local] failed: %s", err)
return
}
return
}
var outlineStorageLock = sync.Mutex{}
func GetOutlineStorage(docID string) (ret map[string]interface{}, err error) {
outlineStorageLock.Lock()
defer outlineStorageLock.Unlock()
ret = map[string]interface{}{}
outlineDocs, err := getOutlineDocs()
if err != nil {
return
}
for _, doc := range outlineDocs {
if doc.DocID == docID {
ret = doc.Data
break
}
}
return
}
func SetOutlineStorage(docID string, val interface{}) (err error) {
outlineStorageLock.Lock()
defer outlineStorageLock.Unlock()
outlineDoc := &OutlineDoc{
DocID: docID,
Data: make(map[string]interface{}),
}
if valMap, ok := val.(map[string]interface{}); ok {
outlineDoc.Data = valMap
}
outlineDocs, err := getOutlineDocs()
if err != nil {
return
}
// 如果文档已存在,先移除旧的
for i, doc := range outlineDocs {
if doc.DocID == docID {
outlineDocs = append(outlineDocs[:i], outlineDocs[i+1:]...)
break
}
}
// 将新的文档信息添加到最前面
outlineDocs = append([]*OutlineDoc{outlineDoc}, outlineDocs...)
// 限制为2000个文档
if 2000 < len(outlineDocs) {
outlineDocs = outlineDocs[:2000]
}
err = setOutlineDocs(outlineDocs)
return
}
func RemoveOutlineStorage(docID string) (err error) {
outlineStorageLock.Lock()
defer outlineStorageLock.Unlock()
outlineDocs, err := getOutlineDocs()
if err != nil {
return
}
for i, doc := range outlineDocs {
if doc.DocID == docID {
outlineDocs = append(outlineDocs[:i], outlineDocs[i+1:]...)
break
}
}
err = setOutlineDocs(outlineDocs)
return
}
func setOutlineDocs(outlineDocs []*OutlineDoc) (err error) {
dirPath := filepath.Join(util.DataDir, "storage")
if err = os.MkdirAll(dirPath, 0755); err != nil {
logging.LogErrorf("create storage [outline] dir failed: %s", err)
return
}
data, err := gulu.JSON.MarshalJSON(outlineDocs)
if err != nil {
logging.LogErrorf("marshal storage [outline] failed: %s", err)
return
}
lsPath := filepath.Join(dirPath, "outline.json")
err = filelock.WriteFile(lsPath, data)
if err != nil {
logging.LogErrorf("write storage [outline] failed: %s", err)
return
}
return
}
func getOutlineDocs() (ret []*OutlineDoc, err error) {
ret = []*OutlineDoc{}
dataPath := filepath.Join(util.DataDir, "storage/outline.json")
if !filelock.IsExist(dataPath) {
return
}
data, err := filelock.ReadFile(dataPath)
if err != nil {
logging.LogErrorf("read storage [outline] failed: %s", err)
return
}
if err = gulu.JSON.UnmarshalJSON(data, &ret); err != nil {
logging.LogErrorf("unmarshal storage [outline] failed: %s", err)
return
}
return
}