Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2026-03-11 15:12:33 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

⚡ Improve data sync performance for booting https://github.com/siyuan-note/siyuan/issues/13216

Browse source
This commit is contained in:
Daniel 2024-11-21 10:59:29 +08:00
parent eba4dfa0da
commit 89f1887c3b
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
16 changed files with 153 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

10
kernel/model/asset_content.go
View file

@ -387,13 +387,13 @@ func (searcher *AssetsSearcher) FullIndex() {
}
var results []*AssetParseResult
filelock.Walk(assetsDir, func(absPath string, info fs.FileInfo, err error) error {
filelock.Walk(assetsDir, func(absPath string, d fs.DirEntry, err error) error {
if err != nil {
logging.LogErrorf("walk dir [%s] failed: %s", absPath, err)
return err
}
if info.IsDir() {
if d.IsDir() {
return nil
}
@ -410,6 +410,12 @@ func (searcher *AssetsSearcher) FullIndex() {
return nil
}
info, err := d.Info()
if err != nil {
logging.LogErrorf("stat file [%s] failed: %s", absPath, err)
return nil
}
result.Path = "assets" + filepath.ToSlash(strings.TrimPrefix(absPath, assetsDir))
result.Size = info.Size()
result.Updated = info.ModTime().Unix()

38
kernel/model/assets.go
View file

@ -311,9 +311,9 @@ func GetAssetAbsPath(relativePath string) (ret string, err error) {
// 在笔记本下搜索
for _, notebook := range notebooks {
notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
filelock.Walk(notebookAbsPath, func(path string, info fs.FileInfo, _ error) error {
if isSkipFile(info.Name()) {
if info.IsDir() {
filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
@ -1270,12 +1270,12 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
// 笔记本 assets
for _, notebook := range notebooks {
notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
filelock.Walk(notebookAbsPath, func(path string, info fs.FileInfo, err error) error {
filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error {
if notebookAbsPath == path {
return nil
}
if isSkipFile(info.Name()) {
if info.IsDir() {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
@ -1286,20 +1286,20 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
return nil
}
if info.IsDir() && "assets" == info.Name() {
filelock.Walk(path, func(assetPath string, info fs.FileInfo, err error) error {
if d.IsDir() && "assets" == d.Name() {
filelock.Walk(path, func(assetPath string, d fs.DirEntry, err error) error {
if path == assetPath {
return nil
}
if isSkipFile(info.Name()) {
if info.IsDir() {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
relPath := filepath.ToSlash(assetPath)
relPath = relPath[strings.Index(relPath, "assets/"):]
if info.IsDir() {
if d.IsDir() {
relPath += "/"
}
assetsAbsPathMap[relPath] = assetPath
@ -1313,13 +1313,13 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
// 全局 assets
dataAssetsAbsPath := util.GetDataAssetsAbsPath()
filelock.Walk(dataAssetsAbsPath, func(assetPath string, info fs.FileInfo, err error) error {
filelock.Walk(dataAssetsAbsPath, func(assetPath string, d fs.DirEntry, err error) error {
if dataAssetsAbsPath == assetPath {
return nil
}
if isSkipFile(info.Name()) {
if info.IsDir() {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
@ -1332,7 +1332,7 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
relPath := filepath.ToSlash(assetPath)
relPath = relPath[strings.Index(relPath, "assets/"):]
if info.IsDir() {
if d.IsDir() {
relPath += "/"
}
assetsAbsPathMap[relPath] = assetPath
@ -1356,14 +1356,12 @@ func copyDocAssetsToDataAssets(boxID, parentDocPath string) {
func copyAssetsToDataAssets(rootPath string) {
var assetsDirPaths []string
filelock.Walk(rootPath, func(path string, info fs.FileInfo, err error) error {
if rootPath == path || nil == info {
filelock.Walk(rootPath, func(path string, d fs.DirEntry, err error) error {
if nil != err || rootPath == path || nil == d {
return nil
}
isDir := info.IsDir()
name := info.Name()
isDir, name := d.IsDir(), d.Name()
if isSkipFile(name) {
if isDir {
return filepath.SkipDir

5
kernel/model/file.go
View file

@ -20,6 +20,7 @@ import (
"bytes"
"errors"
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
@ -1461,11 +1462,11 @@ func MoveDocs(fromPaths []string, toBoxID, toPath string, callback interface{})
func countSubDocs(box, p string) (ret int) {
p = strings.TrimSuffix(p, ".sy")
_ = filepath.Walk(filepath.Join(util.DataDir, box, p), func(path string, info os.FileInfo, err error) error {
_ = filelock.Walk(filepath.Join(util.DataDir, box, p), func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if info.IsDir() {
if d.IsDir() {
return nil
}
if strings.HasSuffix(path, ".sy") {

19
kernel/model/history.go
View file

@ -673,21 +673,26 @@ var boxLatestHistoryTime = map[string]time.Time{}
func (box *Box) recentModifiedDocs() (ret []string) {
latestHistoryTime := boxLatestHistoryTime[box.ID]
filelock.Walk(filepath.Join(util.DataDir, box.ID), func(path string, info fs.FileInfo, err error) error {
if nil == info {
filelock.Walk(filepath.Join(util.DataDir, box.ID), func(path string, d fs.DirEntry, err error) error {
if nil != err || nil == d {
return nil
}
if isSkipFile(info.Name()) {
if info.IsDir() {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.IsDir() {
if d.IsDir() {
return nil
}
info, err := d.Info()
if nil != err {
return err
}
if info.ModTime().After(latestHistoryTime) {
ret = append(ret, path)
}
@ -817,8 +822,8 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) {
entryPath := filepath.Join(util.HistoryDir, name)
var docs, assets []string
filelock.Walk(entryPath, func(path string, info os.FileInfo, err error) error {
if strings.HasSuffix(info.Name(), ".sy") {
filelock.Walk(entryPath, func(path string, d fs.DirEntry, err error) error {
if strings.HasSuffix(d.Name(), ".sy") {
docs = append(docs, path)
} else if strings.Contains(path, "assets"+string(os.PathSeparator)) {
assets = append(assets, path)

52
kernel/model/import.go
View file

@ -118,12 +118,12 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
defer os.RemoveAll(unzipPath)
var syPaths []string
filelock.Walk(unzipPath, func(path string, info fs.FileInfo, err error) error {
filelock.Walk(unzipPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !info.IsDir() && strings.HasSuffix(info.Name(), ".sy") {
if !d.IsDir() && strings.HasSuffix(d.Name(), ".sy") {
syPaths = append(syPaths, path)
}
return nil
@ -226,14 +226,14 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
renameAvPaths := map[string]string{}
if gulu.File.IsExist(storageAvDir) {
// 重新生成数据库数据
filelock.Walk(storageAvDir, func(path string, info fs.FileInfo, err error) error {
if !strings.HasSuffix(path, ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(info.Name(), ".json")) {
filelock.Walk(storageAvDir, func(path string, d fs.DirEntry, err error) error {
if !strings.HasSuffix(path, ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(d.Name(), ".json")) {
return nil
}
// 重命名数据库
newAvID := ast.NewNodeID()
oldAvID := strings.TrimSuffix(info.Name(), ".json")
oldAvID := strings.TrimSuffix(d.Name(), ".json")
newPath := filepath.Join(filepath.Dir(path), newAvID+".json")
renameAvPaths[path] = newPath
avIDs[oldAvID] = newAvID
@ -460,12 +460,12 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
// 重命名文件路径
renamePaths := map[string]string{}
filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
filelock.Walk(unzipRootPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if info.IsDir() && ast.IsNodeIDPattern(info.Name()) {
if d.IsDir() && ast.IsNodeIDPattern(d.Name()) {
renamePaths[path] = path
}
return nil
@ -535,8 +535,8 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
// 将包含的资源文件统一移动到 data/assets/ 下
var assetsDirs []string
filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
if strings.Contains(path, "assets") && info.IsDir() {
filelock.Walk(unzipRootPath, func(path string, d fs.DirEntry, err error) error {
if strings.Contains(path, "assets") && d.IsDir() {
assetsDirs = append(assetsDirs, path)
}
return nil
@ -570,15 +570,15 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
}
var treePaths []string
filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
filelock.Walk(unzipRootPath, func(path string, d fs.DirEntry, err error) error {
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(info.Name(), ".sy") {
if !strings.HasSuffix(d.Name(), ".sy") {
return nil
}
@ -699,18 +699,18 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
if gulu.File.IsDir(localPath) { // 导入文件夹
// 收集所有资源文件
assets := map[string]string{}
filelock.Walk(localPath, func(currentPath string, info os.FileInfo, walkErr error) error {
filelock.Walk(localPath, func(currentPath string, d fs.DirEntry, err error) error {
if localPath == currentPath {
return nil
}
if strings.HasPrefix(info.Name(), ".") {
if info.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(info.Name(), ".md") && !strings.HasSuffix(info.Name(), ".markdown") {
if !strings.HasSuffix(d.Name(), ".md") && !strings.HasSuffix(d.Name(), ".markdown") {
assets[currentPath] = currentPath
return nil
}
@ -721,9 +721,9 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
assetsDone := map[string]string{}
// md 转换 sy
filelock.Walk(localPath, func(currentPath string, info os.FileInfo, walkErr error) error {
if strings.HasPrefix(info.Name(), ".") {
if info.IsDir() {
filelock.Walk(localPath, func(currentPath string, d fs.DirEntry, err error) error {
if strings.HasPrefix(d.Name(), ".") {
if d.IsDir() {
return filepath.SkipDir
}
return nil
@ -731,10 +731,10 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
var tree *parse.Tree
var ext string
title := info.Name()
if !info.IsDir() {
ext = path.Ext(info.Name())
title = strings.TrimSuffix(info.Name(), ext)
title := d.Name()
if !d.IsDir() {
ext = path.Ext(d.Name())
title = strings.TrimSuffix(d.Name(), ext)
}
id := ast.NewNodeID()
@ -759,7 +759,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
id = strings.TrimSuffix(path.Base(targetPath), ".sy")
}
if info.IsDir() {
if d.IsDir() {
if subMdFiles := util.GetFilePathsByExts(currentPath, []string{".md", ".markdown"}); 1 > len(subMdFiles) {
// 如果该文件夹中不包含 Markdown 文件则不处理 https://github.com/siyuan-note/siyuan/issues/11567
return nil
@ -776,7 +776,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
return nil
}
if !strings.HasSuffix(info.Name(), ".md") && !strings.HasSuffix(info.Name(), ".markdown") {
if !strings.HasSuffix(d.Name(), ".md") && !strings.HasSuffix(d.Name(), ".markdown") {
return nil
}

2
kernel/model/index.go
View file

@ -82,7 +82,7 @@ func RemoveIndexes(paths []string) {
func listSyFiles(dir string) (ret []string) {
dirPath := filepath.Join(util.DataDir, dir)
err := filelock.Walk(dirPath, func(path string, d fs.FileInfo, err error) error {
err := filelock.Walk(dirPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
logging.LogWarnf("walk dir [%s] failed: %s", dirPath, err)
return err

27
kernel/model/index_fix.go
View file

@ -18,6 +18,7 @@ package model
import (
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
@ -151,22 +152,22 @@ func resetDuplicateBlocksOnFileSys() {
boxPath := filepath.Join(util.DataDir, box.ID)
var duplicatedTrees []*parse.Tree
filelock.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
if nil == info {
filelock.Walk(boxPath, func(path string, d fs.DirEntry, err error) error {
if err != nil || nil == d {
return nil
}
if info.IsDir() {
if d.IsDir() {
if boxPath == path {
// 跳过笔记本文件夹
return nil
}
if strings.HasPrefix(info.Name(), ".") {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
if !ast.IsNodeIDPattern(info.Name()) {
if !ast.IsNodeIDPattern(d.Name()) {
return nil
}
return nil
@ -176,7 +177,7 @@ func resetDuplicateBlocksOnFileSys() {
return nil
}
if !ast.IsNodeIDPattern(strings.TrimSuffix(info.Name(), ".sy")) {
if !ast.IsNodeIDPattern(strings.TrimSuffix(d.Name(), ".sy")) {
logging.LogWarnf("invalid .sy file name [%s]", path)
box.moveCorruptedData(path)
return nil
@ -285,18 +286,18 @@ func fixBlockTreeByFileSys() {
for _, box := range boxes {
boxPath := filepath.Join(util.DataDir, box.ID)
var paths []string
filelock.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
filelock.Walk(boxPath, func(path string, d fs.DirEntry, err error) error {
if nil != err || nil == d {
return nil
}
if boxPath == path {
// 跳过根路径（笔记本文件夹）
return nil
}
if nil == info {
return nil
}
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil

6
kernel/model/template.go
View file

@ -95,10 +95,10 @@ func SearchTemplate(keyword string) (ret []*Block) {
if group.IsDir() {
var templateBlocks []*Block
templateDir := filepath.Join(templates, group.Name())
filelock.Walk(templateDir, func(path string, info fs.FileInfo, err error) error {
name := strings.ToLower(info.Name())
filelock.Walk(templateDir, func(path string, d fs.DirEntry, err error) error {
name := strings.ToLower(d.Name())
if strings.HasPrefix(name, ".") {
if info.IsDir() {
if d.IsDir() {
return filepath.SkipDir
}
return nil

20
kernel/model/tree.go
View file

@ -130,15 +130,19 @@ func resetTree(tree *parse.Tree, titleSuffix string, removeAvBinding bool) {
func pagedPaths(localPath string, pageSize int) (ret map[int][]string) {
ret = map[int][]string{}
page := 1
filelock.Walk(localPath, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
filelock.Walk(localPath, func(path string, d fs.DirEntry, err error) error {
if nil != err || nil == d {
return nil
}
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(info.Name(), ".sy") {
if !strings.HasSuffix(d.Name(), ".sy") {
return nil
}
@ -248,15 +252,15 @@ func searchTreeInFilesystem(rootID string) {
logging.LogWarnf("searching tree on filesystem [rootID=%s]", rootID)
var treePath string
filepath.Walk(util.DataDir, func(path string, info fs.FileInfo, err error) error {
if info.IsDir() {
if strings.HasPrefix(info.Name(), ".") {
filelock.Walk(util.DataDir, func(path string, d fs.DirEntry, err error) error {
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(info.Name(), ".sy") {
if !strings.HasSuffix(d.Name(), ".sy") {
return nil
}