Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2026-01-21 07:46:09 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
Vanessa 2023-01-26 18:54:11 +08:00
commit bf99eb6833
24 changed files with 833 additions and 625 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/appearance/langs/en_US.json
View file

@ -945,7 +945,7 @@
"55": "Indexed references of [%d] documents",
"56": "Reindexing, please wait until rebuilding is complete before trying to open",
"57": "Failed to create temp key",
"58": "TODO",
"58": "Verifying index...",
"59": "Failed to set sync ignore list",
"60": "Failed to get the update package: %s",
"61": "⬆️ The new version installation package is ready, do you want to install the new version now?",
@ -980,7 +980,7 @@
"90": "[%d/%d] Created [%d] of search indexes of block-level elements [%s]",
"91": "Reading block tree data...",
"92": "Parsing document tree [%s]",
"93": "TODO",
"93": "[%d/%d] Cleaned up the index related to document [%s]",
"94": "Upload failed: %s",
"95": "Exiting...",
"96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data",

4
app/appearance/langs/es_ES.json
View file

@ -945,7 +945,7 @@
"55": "Referencias indexadas de [%d] documentos",
"56": "Reindexando, espere hasta que se complete la reconstrucción antes de intentar abrir",
"57": "Fallo en la creación de la clave temporal",
"58": "TODO",
"58": "Verificando índice...",
"59": " Falló la configuración de sincronización de la lista de ignorados",
"60": "Fallo al obtener el paquete de actualización: %s",
"61": "⬆️ El paquete de instalación de la nueva versión está listo, ¿quieres instalar la nueva versión ahora?",
@ -980,7 +980,7 @@
"90": "[%d/%d] Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
"91": "Leyendo datos del árbol de bloques...",
"92": "Analizando el árbol del documento [%s]",
"93": "TODO",
"93": "[%d/%d] ha limpiado el índice relacionado con el documento [%s]",
"94": "Carga fallida: %s",
"95": "Saliendo...",
"96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube",

4
app/appearance/langs/fr_FR.json
View file

@ -945,7 +945,7 @@
"55": "Références indexées de [%d] documents",
"56": "Réindexation, veuillez attendre que la reconstruction soit terminée avant d'essayer d'ouvrir",
"57": "Échec de la création d'une clé temporaire",
"58": "TODO",
"58": "Vérification de l'index...",
"59": "Échec de la définition de la liste des ignores de synchronisation",
"60": "Échec de la récupération du paquet de mise à jour : %s",
"61": "⬆️ Le package d'installation de la nouvelle version est prêt, voulez-vous installer la nouvelle version maintenant ?",
@ -980,7 +980,7 @@
"90": "[%d/%d] Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
"91": "Lecture des données de l'arborescence des blocs...",
"92": "Analyse de l'arborescence du document [%s]",
"93": "TODO",
"93": "[%d/%d] a nettoyé l'index lié au document [%s]",
"94": "Échec du téléchargement : %s",
"95": "Quitter le programme...",
"96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud",

4
app/appearance/langs/zh_CHT.json
View file

@ -945,7 +945,7 @@
"55": "已完成索引 [%d] 篇文檔的引用關係",
"56": "正在重建索引，請等重建索引完畢後再嘗試打開",
"57": "創建臨時金鑰失敗",
"58": "TODO",
"58": "正在校驗索引...",
"59": "設置同步忽略列表失敗",
"60": "獲取更新包失敗：%s",
"61": "⬆️ 新版本安裝包已經準備就緒，是否現在安裝新版本？",
@ -980,7 +980,7 @@
"90": "[%d/%d] 已經建立 [%d] 個塊級元素的搜索索引 [%s]",
"91": "正在讀取塊樹數據...",
"92": "正在解析文檔樹 [%s]",
"93": "TODO",
"93": "[%d/%d] 已經清理文檔 [%s] 相關的索引",
"94": "上傳失敗：%s",
"95": "正在退出...",
"96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致",

4
app/appearance/langs/zh_CN.json
View file

@ -945,7 +945,7 @@
"55": "已完成索引 [%d] 篇文档的引用关系",
"56": "正在重建索引，请等重建索引完毕后再尝试打开",
"57": "创建临时密钥失败",
"58": "TODO",
"58": "正在校验索引...",
"59": "设置同步忽略列表失败",
"60": "获取更新包失败：%s",
"61": "⬆️ 新版本安装包已经准备就绪，是否现在安装新版本？",
@ -980,7 +980,7 @@
"90": "[%d/%d] 已经建立 [%d] 个块级元素的搜索索引 [%s]",
"91": "正在读取块树数据...",
"92": "正在解析文档树 [%s]",
"93": "TODO",
"93": "[%d/%d] 已经清理文档 [%s] 相关的索引",
"94": "上传失败：%s",
"95": "正在退出...",
"96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致",

3
app/stage/auth.html
View file

@ -530,8 +530,7 @@
}
// 用于授权页保持连接，避免非常驻内存内核自动退出 https://github.com/siyuan-note/insider/issues/1099
new WebSocket(
window.location.protocol === 'https:' ? 'wss' : 'ws' + '://' + window.location.host + '/ws?app=siyuan&id=auth')
new WebSocket((window.location.protocol === 'https:' ? 'wss' : 'ws') + '://' + window.location.host + '/ws?app=siyuan&id=auth')
</script>
</body>
</html>

2
kernel/go.mod
View file

@ -41,7 +41,7 @@ require (
github.com/shirou/gopsutil/v3 v3.22.12
github.com/siyuan-note/dejavu v0.0.0-20230117131301-821aa3adc1e7
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da
github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e
github.com/siyuan-note/httpclient v0.0.0-20230116125720-ee36ddf6f223
github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a

2
kernel/go.sum
View file

@ -377,6 +377,8 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da h1:/jNhl7LC+9BhkWvNxuJDdsNfA/2wvfuj9mqWx4CbV90=
github.com/siyuan-note/eventbus v0.0.0-20220916025349-3ac6e75522da/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2 h1:njoj0265FOHRjF/O1aWDzdwEdYdLTZwhbuPd/hoLJT0=
github.com/siyuan-note/eventbus v0.0.0-20230126092943-c6bf51e65ae2/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e h1:i3RKrdrddr4AuaHJtoWYAEVNuR7Y9wIsEqPmuFFbJC4=
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e/go.mod h1:NmpSIVtIGy8eNWapjDIiiCw5+5r5wxC76k40oG+WRXQ=
github.com/siyuan-note/httpclient v0.0.0-20230116125720-ee36ddf6f223 h1:hG+gucj92x4Dl4lIe2G0WkPgBdlEBnnQCmYpghHeW54=

4
kernel/model/box.go
View file

@ -25,6 +25,7 @@ import (
"os"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
"sync"
@ -516,12 +517,15 @@ func fullReindex() {
}
treenode.InitBlockTree(true)
sql.DisableCache()
openedBoxes := Conf.GetOpenedBoxes()
for _, openedBox := range openedBoxes {
index(openedBox.ID)
}
sql.EnableCache()
treenode.SaveBlockTree(true)
LoadFlashcards()
runtime.GC()
}
func ChangeBoxSort(boxIDs []string) {

15
kernel/model/conf.go
View file

@ -696,20 +696,7 @@ func clearCorruptedNotebooks() {
boxDirPath := filepath.Join(util.DataDir, dir.Name())
boxConfPath := filepath.Join(boxDirPath, ".siyuan", "conf.json")
if !gulu.File.IsExist(boxConfPath) {
if IsUserGuide(dir.Name()) {
filelock.Remove(boxDirPath)
continue
}
to := filepath.Join(util.WorkspaceDir, "corrupted", time.Now().Format("2006-01-02-150405"), dir.Name())
if copyErr := filelock.Copy(boxDirPath, to); nil != copyErr {
logging.LogErrorf("copy corrupted box [%s] failed: %s", boxDirPath, copyErr)
continue
}
if removeErr := filelock.Remove(boxDirPath); nil != removeErr {
logging.LogErrorf("remove corrupted box [%s] failed: %s", boxDirPath, removeErr)
continue
}
logging.LogWarnf("moved corrupted box [%s] to [%s]", boxDirPath, to)
logging.LogWarnf("found a corrupted box [%s]", boxDirPath)
continue
}
}

5
kernel/model/import.go
View file

@ -31,7 +31,7 @@ import (
"os"
"path"
"path/filepath"
"runtime/debug"
"runtime"
"sort"
"strconv"
"strings"
@ -708,8 +708,9 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
}
IncSync()
}
debug.FreeOSMemory()
IncSync()
runtime.GC()
return
}

56
kernel/model/index.go
View file

@ -125,6 +125,7 @@ func index(boxID string) {
end := time.Now()
elapsed := end.Sub(start).Seconds()
logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize)))
runtime.GC()
return
}
@ -186,6 +187,49 @@ func IndexRefs() {
util.PushStatusBar(fmt.Sprintf(Conf.Language(55), i))
}
// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
func AutoIndexEmbedBlock() {
for {
embedBlocks := sql.QueryEmptyContentEmbedBlocks()
task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks)
time.Sleep(10 * time.Minute)
}
}
func autoIndexEmbedBlock(embedBlocks []*sql.Block) {
for i, embedBlock := range embedBlocks {
stmt := strings.TrimPrefix(embedBlock.Markdown, "{{")
stmt = strings.TrimSuffix(stmt, "}}")
queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400)
for _, block := range queryResultBlocks {
embedBlock.Content += block.Content
}
if "" == embedBlock.Content {
embedBlock.Content = "no query result"
}
sql.UpdateBlockContent(embedBlock)
if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿
break
}
}
}
func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) {
embedBlock := sql.GetBlock(embedBlockID)
if nil == embedBlock {
return
}
for _, block := range queryResultBlocks {
embedBlock.Content += block.Block.Markdown
}
if "" == embedBlock.Content {
embedBlock.Content = "no query result"
}
sql.UpdateBlockContent(embedBlock)
}
func init() {
//eventbus.Subscribe(eventbus.EvtSQLInsertBlocks, func(context map[string]interface{}, current, total, blockCount int, hash string) {
// if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
@ -209,4 +253,16 @@ func init() {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtSQLDeleteBlocks, func(context map[string]interface{}, rootID string) {
if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
// Android/iOS 端不显示数据索引和搜索索引状态提示 https://github.com/siyuan-note/siyuan/issues/6392
return
}
current := context["current"].(int) + 1
total := context["total"]
msg := fmt.Sprintf(Conf.Language(93), current, total, rootID)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
}

261
kernel/model/index_fix.go Normal file
View file

@ -0,0 +1,261 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package model
import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/88250/lute/html"
"github.com/88250/lute/parse"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
func AutoFixIndex() {
for {
task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
time.Sleep(10 * time.Minute)
}
}
var autoFixLock = sync.Mutex{}
func autoFixIndex() {
defer logging.Recover()
autoFixLock.Lock()
defer autoFixLock.Unlock()
util.PushStatusBar(Conf.Language(58))
// 去除重复的数据库块记录
duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks")
if 1 > len(duplicatedRootIDs) {
duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts")
if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive {
duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive")
}
}
util.PushStatusBar(Conf.Language(58))
roots := sql.GetBlocks(duplicatedRootIDs)
rootMap := map[string]*sql.Block{}
for _, root := range roots {
rootMap[root.ID] = root
}
var deletes int
for _, rootID := range duplicatedRootIDs {
root := rootMap[rootID]
if nil == root {
continue
}
//logging.LogWarnf("exist more than one tree [%s], reindex it", rootID)
sql.RemoveTreeQueue(root.Box, rootID)
deletes++
if util.IsExiting {
break
}
}
if 0 < deletes {
logging.LogWarnf("exist more than one tree duplicated [%d], reindex it", deletes)
}
util.PushStatusBar(Conf.Language(58))
sql.WaitForWritingDatabase()
util.PushStatusBar(Conf.Language(58))
// 根据文件系统补全块树
boxes := Conf.GetOpenedBoxes()
for _, box := range boxes {
boxPath := filepath.Join(util.DataDir, box.ID)
var paths []string
filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() && filepath.Ext(path) == ".sy" {
p := path[len(boxPath):]
p = filepath.ToSlash(p)
paths = append(paths, p)
}
return nil
})
size := len(paths)
redundantPaths := treenode.GetRedundantPaths(box.ID, paths)
for _, p := range redundantPaths {
treenode.RemoveBlockTreesByPath(p)
}
missingPaths := treenode.GetNotExistPaths(box.ID, paths)
for i, p := range missingPaths {
id := path.Base(p)
id = strings.TrimSuffix(id, ".sy")
if !ast.IsNodeIDPattern(id) {
continue
}
reindexTreeByPath(box.ID, p, i, size)
if util.IsExiting {
break
}
}
if util.IsExiting {
break
}
}
util.PushStatusBar(Conf.Language(58))
sql.WaitForWritingDatabase()
util.PushStatusBar(Conf.Language(58))
// 清理已关闭的笔记本块树
boxes = Conf.GetClosedBoxes()
for _, box := range boxes {
treenode.RemoveBlockTreesByBoxID(box.ID)
}
// 对比块树和数据库并订正数据库
rootUpdatedMap := treenode.GetRootUpdated()
dbRootUpdatedMap, err := sql.GetRootUpdated()
if nil == err {
reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap)
}
util.PushStatusBar(Conf.Language(58))
sql.WaitForWritingDatabase()
util.PushStatusBar(Conf.Language(185))
}
func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
i := -1
size := len(rootUpdatedMap)
for rootID, updated := range rootUpdatedMap {
i++
if util.IsExiting {
break
}
rootUpdated := dbRootUpdatedMap[rootID]
if "" == rootUpdated {
//logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
reindexTree(rootID, i, size)
continue
}
if "" == updated {
// BlockTree 迁移，v2.6.3 之前没有 updated 字段
reindexTree(rootID, i, size)
continue
}
btUpdated, _ := time.Parse("20060102150405", updated)
dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
reindexTree(rootID, i, size)
continue
}
if util.IsExiting {
break
}
}
var rootIDs []string
for rootID, _ := range dbRootUpdatedMap {
if _, ok := rootUpdatedMap[rootID]; !ok {
rootIDs = append(rootIDs, rootID)
}
if util.IsExiting {
break
}
}
rootIDs = gulu.Str.RemoveDuplicatedElem(rootIDs)
roots := map[string]*sql.Block{}
blocks := sql.GetBlocks(rootIDs)
for _, block := range blocks {
roots[block.RootID] = block
}
for id, root := range roots {
if nil == root {
continue
}
logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", id, root.Box)
sql.RemoveTreeQueue(root.Box, root.ID)
if util.IsExiting {
break
}
}
}
func reindexTreeByPath(box, p string, i, size int) {
tree, err := LoadTree(box, p)
if nil != err {
return
}
reindexTree0(tree, i, size)
}
func reindexTree(rootID string, i, size int) {
root := treenode.GetBlockTree(rootID)
if nil == root {
logging.LogWarnf("root block not found", rootID)
return
}
tree, err := LoadTree(root.BoxID, root.Path)
if nil != err {
if os.IsNotExist(err) {
// 文件系统上没有找到该 .sy 文件，则订正块树
treenode.RemoveBlockTreesByRootID(rootID)
}
return
}
reindexTree0(tree, i, size)
}
func reindexTree0(tree *parse.Tree, i, size int) {
updated := tree.Root.IALAttr("updated")
if "" == updated {
updated = util.TimeFromID(tree.Root.ID)
tree.Root.SetIALAttr("updated", updated)
indexWriteJSONQueue(tree)
} else {
treenode.IndexBlockTree(tree)
sql.IndexTreeQueue(tree.Box, tree.Path)
}
if 0 == i%64 {
util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath))))
}
}

2
kernel/model/mount.go
View file

@ -21,7 +21,6 @@ import (
"fmt"
"os"
"path/filepath"
"runtime/debug"
"strings"
"time"
"unicode/utf8"
@ -132,7 +131,6 @@ func unmount0(boxID string) {
boxConf.Closed = true
box.SaveConf(boxConf)
box.Unindex()
debug.FreeOSMemory()
}
func Mount(boxID string) (alreadyMount bool, err error) {

242
kernel/model/transaction.go
View file

@ -20,9 +20,6 @@ import (
"bytes"
"errors"
"fmt"
"github.com/siyuan-note/siyuan/kernel/task"
"os"
"path"
"path/filepath"
"strings"
"sync"
@ -31,7 +28,6 @@ import (
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/88250/lute/editor"
"github.com/88250/lute/html"
"github.com/88250/lute/lex"
"github.com/88250/lute/parse"
"github.com/emirpasic/gods/sets/hashset"
@ -1235,241 +1231,3 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha
})
return
}
// AutoIndexEmbedBlock 嵌入块支持搜索 https://github.com/siyuan-note/siyuan/issues/7112
func AutoIndexEmbedBlock() {
for {
embedBlocks := sql.QueryEmptyContentEmbedBlocks()
task.AppendTask(task.DatabaseIndexEmbedBlock, autoIndexEmbedBlock, embedBlocks)
time.Sleep(10 * time.Minute)
}
}
func autoIndexEmbedBlock(embedBlocks []*sql.Block) {
for i, embedBlock := range embedBlocks {
stmt := strings.TrimPrefix(embedBlock.Markdown, "{{")
stmt = strings.TrimSuffix(stmt, "}}")
queryResultBlocks := sql.SelectBlocksRawStmtNoParse(stmt, 102400)
for _, block := range queryResultBlocks {
embedBlock.Content += block.Content
}
if "" == embedBlock.Content {
embedBlock.Content = "no query result"
}
sql.UpdateBlockContent(embedBlock)
if 63 <= i { // 一次任务中最多处理 64 个嵌入块，防止卡顿
break
}
}
}
func updateEmbedBlockContent(embedBlockID string, queryResultBlocks []*EmbedBlock) {
embedBlock := sql.GetBlock(embedBlockID)
if nil == embedBlock {
return
}
for _, block := range queryResultBlocks {
embedBlock.Content += block.Block.Markdown
}
if "" == embedBlock.Content {
embedBlock.Content = "no query result"
}
sql.UpdateBlockContent(embedBlock)
}
// AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016
func AutoFixIndex() {
for {
task.AppendTask(task.DatabaseIndexFix, autoFixIndex)
time.Sleep(10 * time.Minute)
}
}
var autoFixLock = sync.Mutex{}
func autoFixIndex() {
defer logging.Recover()
// 根据文件系统补全块树
boxes := Conf.GetOpenedBoxes()
for _, box := range boxes {
boxPath := filepath.Join(util.DataDir, box.ID)
var paths []string
filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() && filepath.Ext(path) == ".sy" {
p := path[len(boxPath):]
p = filepath.ToSlash(p)
paths = append(paths, p)
}
return nil
})
size := len(paths)
redundantPaths := treenode.GetRedundantPaths(box.ID, paths)
for _, p := range redundantPaths {
treenode.RemoveBlockTreesByPath(p)
}
missingPaths := treenode.GetNotExistPaths(box.ID, paths)
for i, p := range missingPaths {
id := path.Base(p)
id = strings.TrimSuffix(id, ".sy")
if !ast.IsNodeIDPattern(id) {
continue
}
reindexTreeByPath(box.ID, p, i, size)
if util.IsExiting {
break
}
}
if util.IsExiting {
break
}
}
// 清理已关闭的笔记本块树
boxes = Conf.GetClosedBoxes()
for _, box := range boxes {
treenode.RemoveBlockTreesByBoxID(box.ID)
}
// 对比块树和数据库并订正数据库
rootUpdatedMap := treenode.GetRootUpdated()
dbRootUpdatedMap, err := sql.GetRootUpdated("blocks")
if nil == err {
reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap, "blocks")
}
dbFtsRootUpdatedMap, err := sql.GetRootUpdated("blocks_fts")
if nil == err {
reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts")
}
if !Conf.Search.CaseSensitive {
dbFtsRootUpdatedMap, err = sql.GetRootUpdated("blocks_fts_case_insensitive")
if nil == err {
reindexTreeByUpdated(rootUpdatedMap, dbFtsRootUpdatedMap, "blocks_fts_case_insensitive")
}
}
// 去除重复的数据库块记录
duplicatedRootIDs := sql.GetDuplicatedRootIDs("blocks")
if 1 > len(duplicatedRootIDs) {
duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts")
if 1 > len(duplicatedRootIDs) && !Conf.Search.CaseSensitive {
duplicatedRootIDs = sql.GetDuplicatedRootIDs("blocks_fts_case_insensitive")
}
}
size := len(duplicatedRootIDs)
for i, rootID := range duplicatedRootIDs {
root := sql.GetBlock(rootID)
if nil == root {
continue
}
logging.LogWarnf("exist more than one tree [%s], reindex it", rootID)
sql.RemoveTreeQueue(root.Box, rootID)
reindexTree(rootID, i, size)
if util.IsExiting {
break
}
}
util.PushStatusBar(Conf.Language(185))
}
func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string, blocksTable string) {
i := -1
size := len(rootUpdatedMap)
for rootID, updated := range rootUpdatedMap {
i++
if util.IsExiting {
break
}
rootUpdated := dbRootUpdatedMap[rootID]
if "" == rootUpdated {
logging.LogWarnf("not found tree [%s] in database, reindex it", rootID)
reindexTree(rootID, i, size)
continue
}
if "" == updated {
// BlockTree 迁移，v2.6.3 之前没有 updated 字段
reindexTree(rootID, i, size)
continue
}
btUpdated, _ := time.Parse("20060102150405", updated)
dbUpdated, _ := time.Parse("20060102150405", rootUpdated)
if dbUpdated.Before(btUpdated.Add(-10 * time.Minute)) {
logging.LogWarnf("tree [%s] is not up to date, reindex it", rootID)
reindexTree(rootID, i, size)
continue
}
if util.IsExiting {
break
}
}
for rootID, _ := range dbRootUpdatedMap {
if _, ok := rootUpdatedMap[rootID]; !ok {
logging.LogWarnf("tree [%s] is not in block tree, remove it from [%s]", rootID, blocksTable)
sql.DeleteTree(blocksTable, rootID)
}
if util.IsExiting {
break
}
}
}
func reindexTreeByPath(box, p string, i, size int) {
tree, err := LoadTree(box, p)
if nil != err {
return
}
reindexTree0(tree, i, size)
}
func reindexTree(rootID string, i, size int) {
root := treenode.GetBlockTree(rootID)
if nil == root {
logging.LogWarnf("root block not found", rootID)
return
}
tree, err := LoadTree(root.BoxID, root.Path)
if nil != err {
if os.IsNotExist(err) {
// 文件系统上没有找到该 .sy 文件，则订正块树
treenode.RemoveBlockTreesByRootID(rootID)
}
return
}
reindexTree0(tree, i, size)
}
func reindexTree0(tree *parse.Tree, i, size int) {
updated := tree.Root.IALAttr("updated")
if "" == updated {
updated = util.TimeFromID(tree.Root.ID)
tree.Root.SetIALAttr("updated", updated)
indexWriteJSONQueue(tree)
} else {
treenode.IndexBlockTree(tree)
sql.IndexTreeQueue(tree.Box, tree.Path)
}
if 0 == i%64 {
util.PushStatusBar(fmt.Sprintf(Conf.Language(183), i, size, html.EscapeHTMLStr(path.Base(tree.HPath))))
}
}

5
kernel/server/serve.go
View file

@ -63,10 +63,7 @@ func Serve(fastMode bool) {
})
ginServer.Use(sessions.Sessions("siyuan", cookieStore))
if "dev" == util.Mode {
serveDebug(ginServer)
}
serveDebug(ginServer)
serveAssets(ginServer)
serveAppearance(ginServer)
serveWebSocket(ginServer)

14
kernel/sql/block.go
View file

@ -92,17 +92,3 @@ func UpdateBlockContent(block *Block) {
tx.Commit()
putBlockCache(block)
}
func DeleteTree(table, rootID string) {
tx, err := beginTx()
if nil != err {
return
}
stmt := "DELETE FROM `" + table + "` WHERE root_id = ?"
if err = execStmtTx(tx, stmt, rootID); nil != err {
tx.Rollback()
return
}
tx.Commit()
}

4
kernel/sql/block_query.go
View file

@ -597,8 +597,8 @@ func GetBlock(id string) (ret *Block) {
return
}
func GetRootUpdated(blocksTable string) (ret map[string]string, err error) {
rows, err := query("SELECT root_id, updated FROM `" + blocksTable + "` WHERE type = 'd'")
func GetRootUpdated() (ret map[string]string, err error) {
rows, err := query("SELECT root_id, updated FROM `blocks` WHERE type = 'd'")
if nil != err {
logging.LogErrorf("sql query failed: %s", err)
return

4
kernel/sql/cache.go
View file

@ -17,7 +17,7 @@
package sql
import (
"runtime/debug"
"runtime"
"time"
"github.com/88250/lute/ast"
@ -45,7 +45,7 @@ func DisableCache() {
func ClearBlockCache() {
memCache.Clear()
debug.FreeOSMemory()
runtime.GC()
}
func putBlockCache(block *Block) {

34
kernel/sql/database.go
View file

@ -23,6 +23,7 @@ import (
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
"time"
"unicode/utf8"
@ -33,6 +34,7 @@ import (
"github.com/88250/lute/parse"
"github.com/mattn/go-sqlite3"
_ "github.com/mattn/go-sqlite3"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -65,7 +67,6 @@ func InitDatabase(forceRebuild bool) (err error) {
if forceRebuild {
ClearQueue()
WaitForWritingDatabase()
}
initDBConnection()
@ -80,12 +81,12 @@ func InitDatabase(forceRebuild bool) (err error) {
// 不存在库或者版本不一致都会走到这里
db.Close()
closeDatabase()
if gulu.File.IsExist(util.DBPath) {
if err = removeDatabaseFile(); nil != err {
logging.LogErrorf("remove database file [%s] failed: %s", util.DBPath, err)
util.PushClearProgress()
return
err = nil
}
}
if gulu.File.IsExist(util.BlockTreePath) {
@ -209,7 +210,7 @@ func initHistoryDBTables() {
func initDBConnection() {
if nil != db {
db.Close()
closeDatabase()
}
dsn := util.DBPath + "?_journal_mode=WAL" +
"&_synchronous=OFF" +
@ -947,11 +948,19 @@ func deleteFileAnnotationRefsByBoxTx(tx *sql.Tx, box string) (err error) {
return
}
func deleteByRootID(tx *sql.Tx, rootID string) (err error) {
func deleteByRootID(tx *sql.Tx, rootID string, context map[string]interface{}) (err error) {
stmt := "DELETE FROM blocks WHERE root_id = ?"
if err = execStmtTx(tx, stmt, rootID); nil != err {
return
}
stmt = "DELETE FROM blocks_fts WHERE root_id = ?"
if err = execStmtTx(tx, stmt, rootID); nil != err {
return
}
stmt = "DELETE FROM blocks_fts_case_insensitive WHERE root_id = ?"
if err = execStmtTx(tx, stmt, rootID); nil != err {
return
}
stmt = "DELETE FROM spans WHERE root_id = ?"
if err = execStmtTx(tx, stmt, rootID); nil != err {
return
@ -969,6 +978,7 @@ func deleteByRootID(tx *sql.Tx, rootID string) (err error) {
return
}
ClearBlockCache()
eventbus.Publish(eventbus.EvtSQLDeleteBlocks, context, rootID)
return
}
@ -1023,7 +1033,7 @@ func batchUpdateHPath(tx *sql.Tx, boxID, rootID, oldHPath, newHPath string) (err
}
func CloseDatabase() {
if err := db.Close(); nil != err {
if err := closeDatabase(); nil != err {
logging.LogErrorf("close database failed: %s", err)
return
}
@ -1111,7 +1121,7 @@ func execStmtTx(tx *sql.Tx, stmt string, args ...interface{}) (err error) {
if _, err = tx.Exec(stmt, args...); nil != err {
if strings.Contains(err.Error(), "database disk image is malformed") {
tx.Rollback()
db.Close()
closeDatabase()
removeDatabaseFile()
logging.LogFatalf("database disk image [%s] is malformed, please restart SiYuan kernel to rebuild it", util.DBPath)
}
@ -1180,3 +1190,13 @@ func removeDatabaseFile() (err error) {
}
return
}
func closeDatabase() (err error) {
if nil == db {
return
}
err = db.Close()
runtime.GC() // 没有这句的话文件句柄不会释放，后面就无法删除文件
return
}

28
kernel/sql/queue.go
View file

@ -147,7 +147,7 @@ func execOp(op *dbQueueOperation, tx *sql.Tx, context map[string]interface{}) (e
case "delete":
err = batchDeleteByPathPrefix(tx, op.removeTreeBox, op.removeTreePath)
case "delete_id":
err = deleteByRootID(tx, op.removeTreeID)
err = deleteByRootID(tx, op.removeTreeID, context)
case "rename":
err = batchUpdateHPath(tx, op.renameTree.Box, op.renameTree.ID, op.renameTreeOldHPath, op.renameTree.HPath)
if nil != err {
@ -285,16 +285,13 @@ func RemoveTreeQueue(box, rootID string) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
var tmp []*dbQueueOperation
// 将已有的 upsert 操作去重
for _, op := range operationQueue {
if "upsert" == op.action && op.upsertTree.ID != rootID {
tmp = append(tmp, op)
newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
for i, op := range operationQueue {
if "delete_id" == op.action && op.removeTreeIDBox == box && op.removeTreeID == rootID {
operationQueue[i] = newOp
return
}
}
operationQueue = tmp
newOp := &dbQueueOperation{removeTreeIDBox: box, removeTreeID: rootID, inQueueTime: time.Now(), action: "delete_id"}
operationQueue = append(operationQueue, newOp)
}
@ -302,15 +299,12 @@ func RemoveTreePathQueue(treeBox, treePathPrefix string) {
dbQueueLock.Lock()
defer dbQueueLock.Unlock()
var tmp []*dbQueueOperation
// 将已有的 upsert 操作去重
for _, op := range operationQueue {
if "upsert" == op.action && (op.removeTreeBox != treeBox || op.upsertTree.Path != treePathPrefix) {
tmp = append(tmp, op)
newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
for i, op := range operationQueue {
if "delete" == op.action && (op.removeTreeBox == treeBox && op.removeTreePath == treePathPrefix) {
operationQueue[i] = newOp
return
}
}
operationQueue = tmp
newOp := &dbQueueOperation{removeTreeBox: treeBox, removeTreePath: treePathPrefix, inQueueTime: time.Now(), action: "delete"}
operationQueue = append(operationQueue, newOp)
}

753
kernel/treenode/blocktree.go
View file

@ -19,7 +19,8 @@ package treenode
import (
"io"
"os"
"runtime/debug"
"path/filepath"
"runtime"
"strings"
"sync"
"time"
@ -28,14 +29,19 @@ import (
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/dustin/go-humanize"
util2 "github.com/siyuan-note/dejavu/util"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
"github.com/vmihailenco/msgpack/v5"
)
var blockTrees = map[string]*BlockTree{}
var blockTreesLock = sync.Mutex{}
var blockTreesChanged = time.Time{}
var blockTrees = sync.Map{}
type btSlice struct {
data map[string]*BlockTree
changed time.Time
m *sync.Mutex
}
type BlockTree struct {
ID string // 块 ID
@ -49,46 +55,458 @@ type BlockTree struct {
}
func GetRootUpdated() (ret map[string]string) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
ret = map[string]string{}
for _, b := range blockTrees {
if b.RootID == b.ID {
ret[b.RootID] = b.Updated
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.RootID == b.ID {
ret[b.RootID] = b.Updated
}
}
}
slice.m.Unlock()
return true
})
return
}
func GetBlockTreeByPath(path string) *BlockTree {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
for _, b := range blockTrees {
if b.Path == path {
return b
func GetBlockTreeByPath(path string) (ret *BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.Path == path {
ret = b
break
}
}
}
return nil
slice.m.Unlock()
return nil == ret
})
return
}
func CountTrees() (ret int) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
roots := map[string]bool{}
for _, b := range blockTrees {
roots[b.RootID] = true
}
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
roots[b.RootID] = true
}
slice.m.Unlock()
return true
})
ret = len(roots)
return
}
func CountBlocks() (ret int) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
return len(blockTrees)
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
ret += len(slice.data)
slice.m.Unlock()
return true
})
return
}
func GetRedundantPaths(boxID string, paths []string) (ret []string) {
pathsMap := map[string]bool{}
for _, path := range paths {
pathsMap[path] = true
}
btPathsMap := map[string]bool{}
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID {
btPathsMap[b.Path] = true
}
}
slice.m.Unlock()
return true
})
for p, _ := range btPathsMap {
if !pathsMap[p] {
ret = append(ret, p)
}
}
ret = gulu.Str.RemoveDuplicatedElem(ret)
return
}
func GetNotExistPaths(boxID string, paths []string) (ret []string) {
pathsMap := map[string]bool{}
for _, path := range paths {
pathsMap[path] = true
}
btPathsMap := map[string]bool{}
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID {
btPathsMap[b.Path] = true
}
}
slice.m.Unlock()
return true
})
for p, _ := range pathsMap {
if !btPathsMap[p] {
ret = append(ret, p)
}
}
ret = gulu.Str.RemoveDuplicatedElem(ret)
return
}
func GetBlockTreeRootByPath(boxID, path string) (ret *BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID && b.Path == path && b.RootID == b.ID {
ret = b
break
}
}
slice.m.Unlock()
return nil == ret
})
return
}
func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID && b.HPath == hPath && b.RootID == b.ID {
ret = b
break
}
}
slice.m.Unlock()
return nil == ret
})
return
}
func GetBlockTree(id string) (ret *BlockTree) {
if "" == id {
return
}
hash := btHash(id)
val, ok := blockTrees.Load(hash)
if !ok {
return
}
slice := val.(*btSlice)
slice.m.Lock()
ret = slice.data[id]
slice.m.Unlock()
return
}
func SetBlockTreePath(tree *parse.Tree) {
hash := btHash(tree.ID)
val, ok := blockTrees.Load(hash)
if !ok {
val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}}
blockTrees.Store(hash, val)
}
slice := val.(*btSlice)
slice.m.Lock()
slice.data[tree.ID] = &BlockTree{
ID: tree.ID,
RootID: tree.Root.ID,
BoxID: tree.Box,
Path: tree.Path,
HPath: tree.HPath,
Updated: tree.Root.IALAttr("updated"),
Type: TypeAbbr(ast.NodeDocument.String()),
}
slice.m.Unlock()
slice.changed = time.Now()
}
func RemoveBlockTreesByRootID(rootID string) {
var ids []string
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.RootID == rootID {
ids = append(ids, b.RootID)
}
}
slice.m.Unlock()
return true
})
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
val, ok := blockTrees.Load(btHash(id))
if !ok {
continue
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.m.Unlock()
slice.changed = time.Now()
}
}
func RemoveBlockTreesByPath(path string) {
var ids []string
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.Path == path {
ids = append(ids, b.RootID)
}
}
slice.m.Unlock()
return true
})
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
val, ok := blockTrees.Load(btHash(id))
if !ok {
continue
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.m.Unlock()
slice.changed = time.Now()
}
}
func RemoveBlockTreesByPathPrefix(pathPrefix string) {
var ids []string
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if strings.HasPrefix(b.Path, pathPrefix) {
ids = append(ids, b.RootID)
}
}
slice.m.Unlock()
return true
})
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
val, ok := blockTrees.Load(btHash(id))
if !ok {
continue
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.m.Unlock()
slice.changed = time.Now()
}
}
func RemoveBlockTreesByBoxID(boxID string) (ids []string) {
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
slice.m.Lock()
for _, b := range slice.data {
if b.BoxID == boxID {
ids = append(ids, b.RootID)
}
}
slice.m.Unlock()
return true
})
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
val, ok := blockTrees.Load(btHash(id))
if !ok {
continue
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.m.Unlock()
slice.changed = time.Now()
}
return
}
func RemoveBlockTree(id string) {
val, ok := blockTrees.Load(btHash(id))
if !ok {
return
}
slice := val.(*btSlice)
slice.m.Lock()
delete(slice.data, id)
slice.m.Unlock()
slice.changed = time.Now()
}
func IndexBlockTree(tree *parse.Tree) {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || !n.IsBlock() {
return ast.WalkContinue
}
var parentID string
if nil != n.Parent {
parentID = n.Parent.ID
}
if "" == n.ID {
return ast.WalkContinue
}
hash := btHash(n.ID)
val, ok := blockTrees.Load(hash)
if !ok {
val = &btSlice{data: map[string]*BlockTree{}, changed: time.Time{}, m: &sync.Mutex{}}
blockTrees.Store(hash, val)
}
slice := val.(*btSlice)
slice.m.Lock()
if bt := slice.data[n.ID]; nil != bt {
if bt.Updated != n.IALAttr("updated") {
slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())}
slice.changed = time.Now()
}
} else {
slice.data[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: n.IALAttr("updated"), Type: TypeAbbr(n.Type.String())}
slice.changed = time.Now()
}
slice.m.Unlock()
return ast.WalkContinue
})
}
func AutoFlushBlockTree() {
for {
SaveBlockTree(false)
time.Sleep(1 * time.Second)
}
}
func InitBlockTree(force bool) {
start := time.Now()
if force {
err := os.RemoveAll(util.BlockTreePath)
if nil != err {
logging.LogErrorf("remove blocktree file failed: %s", err)
}
return
}
entries, err := os.ReadDir(util.BlockTreePath)
if nil != err {
logging.LogErrorf("read block tree dir failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return
}
size := uint64(0)
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), ".msgpack") {
continue
}
p := filepath.Join(util.BlockTreePath, entry.Name())
var fh *os.File
fh, err = os.OpenFile(p, os.O_RDWR, 0644)
if nil != err {
logging.LogErrorf("open block tree file failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return
}
var data []byte
data, err = io.ReadAll(fh)
fh.Close()
if nil != err {
logging.LogErrorf("read block tree failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return
}
sliceData := map[string]*BlockTree{}
if err = msgpack.Unmarshal(data, &sliceData); nil != err {
logging.LogErrorf("unmarshal block tree failed: %s", err)
if err = os.RemoveAll(util.BlockTreePath); nil != err {
logging.LogErrorf("removed corrupted block tree failed: %s", err)
}
os.Exit(util.ExitCodeBlockTreeErr)
return
}
name := entry.Name()[0:strings.Index(entry.Name(), ".")]
blockTrees.Store(name, &btSlice{data: sliceData, changed: time.Time{}, m: &sync.Mutex{}})
size += uint64(len(data))
}
runtime.GC()
if elapsed := time.Since(start).Seconds(); 2 < elapsed {
logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes((size)), util.BlockTreePath, elapsed)
}
return
}
func SaveBlockTree(force bool) {
start := time.Now()
os.MkdirAll(util.BlockTreePath, 0755)
size := uint64(0)
blockTrees.Range(func(key, value interface{}) bool {
slice := value.(*btSlice)
if !force && (slice.changed.IsZero() || slice.changed.After(start.Add(-7*time.Second))) {
return true
}
slice.m.Lock()
data, err := msgpack.Marshal(slice.data)
if nil != err {
logging.LogErrorf("marshal block tree failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return false
}
slice.m.Unlock()
p := filepath.Join(util.BlockTreePath, key.(string)) + ".msgpack"
if err = gulu.File.WriteFileSafer(p, data, 0644); nil != err {
logging.LogErrorf("write block tree failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return false
}
slice.changed = time.Time{}
size += uint64(len(data))
return true
})
runtime.GC()
if elapsed := time.Since(start).Seconds(); 2 < elapsed {
logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(size), util.BlockTreePath, elapsed)
}
}
func CeilTreeCount(count int) int {
@ -117,279 +535,6 @@ func CeilBlockCount(count int) int {
return 10000*100 + 1
}
func GetRedundantPaths(boxID string, paths []string) (ret []string) {
pathsMap := map[string]bool{}
for _, path := range paths {
pathsMap[path] = true
}
tmp := blockTrees
btPathsMap := map[string]bool{}
for _, blockTree := range tmp {
if blockTree.BoxID != boxID {
continue
}
btPathsMap[blockTree.Path] = true
}
for p, _ := range btPathsMap {
if !pathsMap[p] {
ret = append(ret, p)
}
}
ret = gulu.Str.RemoveDuplicatedElem(ret)
return
}
func GetNotExistPaths(boxID string, paths []string) (ret []string) {
pathsMap := map[string]bool{}
for _, path := range paths {
pathsMap[path] = true
}
tmp := blockTrees
btPathsMap := map[string]bool{}
for _, blockTree := range tmp {
if blockTree.BoxID != boxID {
continue
}
btPathsMap[blockTree.Path] = true
}
for p, _ := range pathsMap {
if !btPathsMap[p] {
ret = append(ret, p)
}
}
ret = gulu.Str.RemoveDuplicatedElem(ret)
return
}
func GetBlockTreeRootByPath(boxID, path string) *BlockTree {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
for _, blockTree := range blockTrees {
if blockTree.BoxID == boxID && blockTree.Path == path && blockTree.RootID == blockTree.ID {
return blockTree
}
}
return nil
}
func GetBlockTreeRootByHPath(boxID, hPath string) *BlockTree {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
for _, blockTree := range blockTrees {
if blockTree.BoxID == boxID && blockTree.HPath == hPath && blockTree.RootID == blockTree.ID {
return blockTree
}
}
return nil
}
func GetBlockTree(id string) *BlockTree {
if "" == id {
return nil
}
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
return blockTrees[id]
}
func SetBlockTreePath(tree *parse.Tree) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
for _, b := range blockTrees {
if b.RootID == tree.ID {
b.BoxID, b.Path, b.HPath, b.Updated, b.Type = tree.Box, tree.Path, tree.HPath, tree.Root.IALAttr("updated"), TypeAbbr(ast.NodeDocument.String())
}
}
blockTreesChanged = time.Now()
}
func RemoveBlockTreesByRootID(rootID string) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
var ids []string
for _, b := range blockTrees {
if b.RootID == rootID {
ids = append(ids, b.RootID)
}
}
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
delete(blockTrees, id)
}
blockTreesChanged = time.Now()
}
func RemoveBlockTreesByPath(path string) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
var ids []string
for _, b := range blockTrees {
if b.Path == path {
ids = append(ids, b.ID)
}
}
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
delete(blockTrees, id)
}
blockTreesChanged = time.Now()
}
func RemoveBlockTreesByPathPrefix(pathPrefix string) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
var ids []string
for _, b := range blockTrees {
if strings.HasPrefix(b.Path, pathPrefix) {
ids = append(ids, b.ID)
}
}
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
delete(blockTrees, id)
}
blockTreesChanged = time.Now()
}
func RemoveBlockTreesByBoxID(boxID string) (ids []string) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
for _, b := range blockTrees {
if b.BoxID == boxID {
ids = append(ids, b.ID)
}
}
ids = gulu.Str.RemoveDuplicatedElem(ids)
for _, id := range ids {
delete(blockTrees, id)
}
blockTreesChanged = time.Now()
return
}
func RemoveBlockTree(id string) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
delete(blockTrees, id)
blockTreesChanged = time.Now()
}
func IndexBlockTree(tree *parse.Tree) {
blockTreesLock.Lock()
defer blockTreesLock.Unlock()
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || !n.IsBlock() {
return ast.WalkContinue
}
var parentID string
if nil != n.Parent {
parentID = n.Parent.ID
}
if "" == n.ID {
return ast.WalkContinue
}
blockTrees[n.ID] = &BlockTree{ID: n.ID, ParentID: parentID, RootID: tree.ID, BoxID: tree.Box, Path: tree.Path, HPath: tree.HPath, Updated: tree.Root.IALAttr("updated"), Type: TypeAbbr(n.Type.String())}
return ast.WalkContinue
})
blockTreesChanged = time.Now()
}
func AutoFlushBlockTree() {
for {
SaveBlockTree(false)
time.Sleep(1 * time.Second)
}
}
func InitBlockTree(force bool) {
start := time.Now()
if force {
err := os.RemoveAll(util.BlockTreePath)
if nil != err {
logging.LogErrorf("remove blocktree file failed: %s", err)
}
return
}
var err error
fh, err := os.OpenFile(util.BlockTreePath, os.O_RDWR, 0644)
if nil != err {
logging.LogErrorf("open block tree file failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return
}
defer fh.Close()
data, err := io.ReadAll(fh)
if nil != err {
logging.LogErrorf("read block tree failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return
}
blockTreesLock.Lock()
if err = msgpack.Unmarshal(data, &blockTrees); nil != err {
logging.LogErrorf("unmarshal block tree failed: %s", err)
if err = os.RemoveAll(util.BlockTreePath); nil != err {
logging.LogErrorf("removed corrupted block tree failed: %s", err)
}
os.Exit(util.ExitCodeBlockTreeErr)
return
}
blockTreesLock.Unlock()
debug.FreeOSMemory()
if elapsed := time.Since(start).Seconds(); 2 < elapsed {
logging.LogWarnf("read block tree [%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed)
}
return
}
func SaveBlockTree(force bool) {
if !force && blockTreesChanged.IsZero() {
return
}
start := time.Now()
if blockTreesChanged.After(start.Add(-7 * time.Second)) {
return
}
blockTreesLock.Lock()
data, err := msgpack.Marshal(blockTrees)
if nil != err {
logging.LogErrorf("marshal block tree failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return
}
blockTreesLock.Unlock()
if err = gulu.File.WriteFileSafer(util.BlockTreePath, data, 0644); nil != err {
logging.LogErrorf("write block tree failed: %s", err)
os.Exit(util.ExitCodeBlockTreeErr)
return
}
debug.FreeOSMemory()
if elapsed := time.Since(start).Seconds(); 2 < elapsed {
logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(uint64(len(data))), util.BlockTreePath, elapsed)
}
blockTreesChanged = time.Time{}
func btHash(id string) string {
return util2.Hash([]byte(id))[0:2]
}

2
kernel/util/working.go
View file

@ -253,7 +253,7 @@ func initWorkspaceDir(workspaceArg string) {
os.Setenv("TMP", osTmpDir)
DBPath = filepath.Join(TempDir, DBName)
HistoryDBPath = filepath.Join(TempDir, "history.db")
BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack")
BlockTreePath = filepath.Join(TempDir, "blocktree")
SnippetsPath = filepath.Join(DataDir, "snippets")
}

2
kernel/util/working_mobile.go
View file

@ -155,7 +155,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) {
os.Setenv("TMP", osTmpDir)
DBPath = filepath.Join(TempDir, DBName)
HistoryDBPath = filepath.Join(TempDir, "history.db")
BlockTreePath = filepath.Join(TempDir, "blocktree.msgpack")
BlockTreePath = filepath.Join(TempDir, "blocktree")
SnippetsPath = filepath.Join(DataDir, "snippets")
AppearancePath = filepath.Join(ConfDir, "appearance")