🐛 Data sync may cause data loss on Android 14 https://github.com/siyuan-note/siyuan/issues/10205

kernel/cache/asset.go

@@ -23,6 +23,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
@@ -68,7 +69,7 @@ func LoadAssets() {
 
 	assetsCache = map[string]*Asset{}
 	assets := util.GetDataAssetsAbsPath()
-	filepath.Walk(assets, func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(assets, func(path string, info fs.FileInfo, err error) error {
 		if nil == info {
 			return err
 		}

kernel/go.mod

@@ -174,7 +174,6 @@ replace github.com/mattn/go-sqlite3 => github.com/88250/go-sqlite3 v1.14.13-0.20
 //replace github.com/siyuan-note/filelock => D:\88250\filelock
 //replace github.com/88250/lute => D:\gogogo\src\github.com\88250\lute
 //replace github.com/88250/pdfcpu => D:\88250\pdfcpu
-//replace github.com/siyuan-note/filelock => D:\88250\filelock
 //replace github.com/88250/gulu => D:\88250\gulu
 //replace github.com/mattn/go-sqlite3 => D:\88250\go-sqlite3
 //replace github.com/88250/epub => D:\88250\epub

kernel/model/asset_content.go

@@ -384,7 +384,7 @@ func (searcher *AssetsSearcher) FullIndex() {
 	}
 
 	var results []*AssetParseResult
-	filepath.Walk(assetsDir, func(absPath string, info fs.FileInfo, err error) error {
+	filelock.Walk(assetsDir, func(absPath string, info fs.FileInfo, err error) error {
 		if nil != err {
 			logging.LogErrorf("walk dir [%s] failed: %s", absPath, err)
 			return err

kernel/model/assets.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/88250/lute/editor"
 	"io"
 	"io/fs"
 	"mime"
@@ -35,6 +34,7 @@ import (
 
 	"github.com/88250/gulu"
 	"github.com/88250/lute/ast"
+	"github.com/88250/lute/editor"
 	"github.com/88250/lute/html"
 	"github.com/88250/lute/parse"
 	"github.com/dustin/go-humanize"
@@ -472,7 +472,7 @@ func GetAssetAbsPath(relativePath string) (ret string, err error) {
 	// 在笔记本下搜索
 	for _, notebook := range notebooks {
 		notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
-		filepath.Walk(notebookAbsPath, func(path string, info fs.FileInfo, _ error) error {
+		filelock.Walk(notebookAbsPath, func(path string, info fs.FileInfo, _ error) error {
 			if isSkipFile(info.Name()) {
 				if info.IsDir() {
 					return filepath.SkipDir
@@ -1190,7 +1190,7 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
 	// 笔记本 assets
 	for _, notebook := range notebooks {
 		notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
-		filepath.Walk(notebookAbsPath, func(path string, info fs.FileInfo, err error) error {
+		filelock.Walk(notebookAbsPath, func(path string, info fs.FileInfo, err error) error {
 			if notebookAbsPath == path {
 				return nil
 			}
@@ -1202,7 +1202,7 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
 			}
 
 			if info.IsDir() && "assets" == info.Name() {
-				filepath.Walk(path, func(assetPath string, info fs.FileInfo, err error) error {
+				filelock.Walk(path, func(assetPath string, info fs.FileInfo, err error) error {
 					if path == assetPath {
 						return nil
 					}
@@ -1228,7 +1228,7 @@ func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
 
 	// 全局 assets
 	dataAssetsAbsPath := util.GetDataAssetsAbsPath()
-	filepath.Walk(dataAssetsAbsPath, func(assetPath string, info fs.FileInfo, err error) error {
+	filelock.Walk(dataAssetsAbsPath, func(assetPath string, info fs.FileInfo, err error) error {
 		if dataAssetsAbsPath == assetPath {
 			return nil
 		}
@@ -1265,7 +1265,7 @@ func copyDocAssetsToDataAssets(boxID, parentDocPath string) {
 
 func copyAssetsToDataAssets(rootPath string) {
 	var assetsDirPaths []string
-	filepath.Walk(rootPath, func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(rootPath, func(path string, info fs.FileInfo, err error) error {
 		if rootPath == path || nil == info {
 			return nil
 		}

kernel/model/history.go

@@ -567,7 +567,7 @@ var boxLatestHistoryTime = map[string]time.Time{}
 
 func (box *Box) recentModifiedDocs() (ret []string) {
 	latestHistoryTime := boxLatestHistoryTime[box.ID]
-	filepath.Walk(filepath.Join(util.DataDir, box.ID), func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(filepath.Join(util.DataDir, box.ID), func(path string, info fs.FileInfo, err error) error {
 		if nil == info {
 			return nil
 		}
@@ -666,7 +666,7 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) {
 
 	entryPath := filepath.Join(util.HistoryDir, name)
 	var docs, assets []string
-	filepath.Walk(entryPath, func(path string, info os.FileInfo, err error) error {
+	filelock.Walk(entryPath, func(path string, info os.FileInfo, err error) error {
 		if strings.HasSuffix(info.Name(), ".sy") {
 			docs = append(docs, path)
 		} else if strings.Contains(path, "assets"+string(os.PathSeparator)) {

kernel/model/import.go

@@ -98,7 +98,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 	defer os.RemoveAll(unzipPath)
 
 	var syPaths []string
-	filepath.Walk(unzipPath, func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(unzipPath, func(path string, info fs.FileInfo, err error) error {
 		if nil != err {
 			return err
 		}
@@ -206,7 +206,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 	renameAvPaths := map[string]string{}
 	if gulu.File.IsExist(storageAvDir) {
 		// 重新生成数据库数据
-		filepath.Walk(storageAvDir, func(path string, info fs.FileInfo, err error) error {
+		filelock.Walk(storageAvDir, func(path string, info fs.FileInfo, err error) error {
 			if !strings.HasSuffix(path, ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(info.Name(), ".json")) {
 				return nil
 			}
@@ -397,7 +397,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 
 	// 重命名文件路径
 	renamePaths := map[string]string{}
-	filepath.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
 		if nil != err {
 			return err
 		}
@@ -472,7 +472,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 
 	// 将包含的资源文件统一移动到 data/assets/ 下
 	var assetsDirs []string
-	filepath.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
 		if strings.Contains(path, "assets") && info.IsDir() {
 			assetsDirs = append(assetsDirs, path)
 		}
@@ -507,7 +507,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 	}
 
 	var treePaths []string
-	filepath.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(unzipRootPath, func(path string, info fs.FileInfo, err error) error {
 		if info.IsDir() {
 			if strings.HasPrefix(info.Name(), ".") {
 				return filepath.SkipDir
@@ -632,7 +632,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
 	if gulu.File.IsDir(localPath) {
 		// 收集所有资源文件
 		assets := map[string]string{}
-		filepath.Walk(localPath, func(currentPath string, info os.FileInfo, walkErr error) error {
+		filelock.Walk(localPath, func(currentPath string, info os.FileInfo, walkErr error) error {
 			if localPath == currentPath {
 				return nil
 			}
@@ -654,7 +654,7 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
 		assetsDone := map[string]string{}
 
 		// md 转换 sy
-		filepath.Walk(localPath, func(currentPath string, info os.FileInfo, walkErr error) error {
+		filelock.Walk(localPath, func(currentPath string, info os.FileInfo, walkErr error) error {
 			if strings.HasPrefix(info.Name(), ".") {
 				if info.IsDir() {
 					return filepath.SkipDir

kernel/model/index.go

@@ -19,7 +19,7 @@ package model
 import (
 	"bytes"
 	"fmt"
-	"os"
+	"io/fs"
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
@@ -80,7 +80,7 @@ func RemoveIndexes(paths []string) {
 
 func listSyFiles(dir string) (ret []string) {
 	dirPath := filepath.Join(util.DataDir, dir)
-	err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
+	err := filelock.Walk(dirPath, func(path string, d fs.FileInfo, err error) error {
 		if nil != err {
 			logging.LogWarnf("walk dir [%s] failed: %s", dirPath, err)
 			return err

kernel/model/index_fix.go

@@ -31,6 +31,7 @@ import (
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/html"
 	"github.com/88250/lute/parse"
+	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 	"github.com/siyuan-note/siyuan/kernel/filesys"
 	"github.com/siyuan-note/siyuan/kernel/sql"
@@ -153,7 +154,7 @@ func resetDuplicateBlocksOnFileSys() {
 
 		boxPath := filepath.Join(util.DataDir, box.ID)
 		var duplicatedTrees []*parse.Tree
-		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
+		filelock.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
 			if nil == info {
 				return nil
 			}
@@ -301,7 +302,7 @@ func fixBlockTreeByFileSys() {
 	for _, box := range boxes {
 		boxPath := filepath.Join(util.DataDir, box.ID)
 		var paths []string
-		filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
+		filelock.Walk(boxPath, func(path string, info os.FileInfo, err error) error {
 			if boxPath == path {
 				// 跳过根路径（笔记本文件夹）
 				return nil

kernel/model/template.go

@@ -97,11 +97,10 @@ func SearchTemplate(keyword string) (ret []*Block) {
 		if group.IsDir() {
 			var templateBlocks []*Block
 			templateDir := filepath.Join(templates, group.Name())
-			// filepath.Walk 与 filepath.WalkDir 均不支持跟踪符号链接
+			filelock.Walk(templateDir, func(path string, info fs.FileInfo, err error) error {
-			filepath.WalkDir(templateDir, func(path string, entry fs.DirEntry, err error) error {
+				name := strings.ToLower(info.Name())
-				name := strings.ToLower(entry.Name())
 				if strings.HasPrefix(name, ".") {
-					if entry.IsDir() {
+					if info.IsDir() {
 						return filepath.SkipDir
 					}
 					return nil

kernel/model/tree.go

@@ -107,7 +107,7 @@ func resetTree(tree *parse.Tree, titleSuffix string) {
 func pagedPaths(localPath string, pageSize int) (ret map[int][]string) {
 	ret = map[int][]string{}
 	page := 1
-	filepath.Walk(localPath, func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(localPath, func(path string, info fs.FileInfo, err error) error {
 		if info.IsDir() {
 			if strings.HasPrefix(info.Name(), ".") {
 				return filepath.SkipDir

kernel/treenode/tree.go

@@ -29,6 +29,7 @@ import (
 	"github.com/88250/lute"
 	"github.com/88250/lute/ast"
 	"github.com/88250/lute/parse"
+	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
@@ -101,7 +102,7 @@ func RootChildIDs(rootID string) (ret []string) {
 	if !gulu.File.IsDir(subFolder) {
 		return
 	}
-	filepath.Walk(subFolder, func(path string, info fs.FileInfo, err error) error {
+	filelock.Walk(subFolder, func(path string, info fs.FileInfo, err error) error {
 		if strings.HasSuffix(path, ".sy") {
 			name := filepath.Base(path)
 			id := strings.TrimSuffix(name, ".sy")

kernel/util/file.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/88250/gulu"
 	"github.com/88250/lute/ast"
+	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 )
 
@@ -215,7 +216,7 @@ func IsSubPath(absPath, toCheckPath string) bool {
 }
 
 func SizeOfDirectory(path string) (size int64, err error) {
-	err = filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+	err = filelock.Walk(path, func(_ string, info os.FileInfo, err error) error {
 		if nil != err {
 			return err
 		}
@@ -234,7 +235,7 @@ func SizeOfDirectory(path string) (size int64, err error) {
 }
 
 func DataSize() (dataSize, assetsSize int64) {
-	filepath.Walk(DataDir, func(path string, info os.FileInfo, err error) error {
+	filelock.Walk(DataDir, func(path string, info os.FileInfo, err error) error {
 		if nil != err {
 			if os.IsNotExist(err) {
 				return nil
@@ -283,7 +284,7 @@ func WalkWithSymlinks(root string, fn filepath.WalkFunc) error {
 	}
 
 	visitedDirs := make(map[string]struct{})
-	return filepath.Walk(rr, getWalkFn(visitedDirs, fn))
+	return filelock.Walk(rr, getWalkFn(visitedDirs, fn))
 }
 
 func getWalkFn(visitedDirs map[string]struct{}, fn filepath.WalkFunc) filepath.WalkFunc {
@@ -319,7 +320,7 @@ func getWalkFn(visitedDirs map[string]struct{}, fn filepath.WalkFunc) filepath.W
 		}
 
 		if ri.IsDir() {
-			return filepath.Walk(rp, getWalkFn(visitedDirs, fn))
+			return filelock.Walk(rp, getWalkFn(visitedDirs, fn))
 		}
 
 		return nil

kernel/util/path.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/88250/gulu"
+	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
 )
 
@@ -166,7 +167,7 @@ func GetChildDocDepth(treeAbsPath string) (ret int) {
 
 	baseDepth := strings.Count(filepath.ToSlash(treeAbsPath), "/")
 	depth := 1
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+	filelock.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		p := filepath.ToSlash(path)
 		currentDepth := strings.Count(p, "/")
 		if depth < currentDepth {