Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2025-09-22 00:20:47 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

🎨 细化云端同步锁提升稳定性 https://github.com/siyuan-note/siyuan/issues/5887

Browse source
This commit is contained in:
Liang Ding 2022-09-18 09:14:34 +08:00
parent 963b409e4e
commit 11e4c88f5a
No known key found for this signature in database
GPG key ID: 136F30F901A2231D
15 changed files with 110 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

3
kernel/api/asset.go
View file

@ -24,6 +24,7 @@ import (
"github.com/88250/gulu"
"github.com/gin-gonic/gin"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -85,7 +86,7 @@ func setFileAnnotation(c *gin.Context) {
ret.Msg = err.Error()
return
}
if err := util.WriteFileSafer(writePath, []byte(data)); nil != err {
if err := filesys.WriteFileSafer(writePath, []byte(data)); nil != err {
ret.Code = -1
ret.Msg = err.Error()
return

3
kernel/api/extension.go
View file

@ -30,6 +30,7 @@ import (
"github.com/88250/lute/ast"
"github.com/gin-gonic/gin"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/model"
"github.com/siyuan-note/siyuan/kernel/util"
)
@ -106,7 +107,7 @@ func extensionCopy(c *gin.Context) {
}
fName = fName + "-" + ast.NewNodeID() + ext
writePath := filepath.Join(assets, fName)
if err = util.WriteFileSafer(writePath, data); nil != err {
if err = filesys.WriteFileSafer(writePath, data); nil != err {
ret.Code = -1
ret.Msg = err.Error()
break

3
kernel/bazaar/package.go
View file

@ -32,6 +32,7 @@ import (
"github.com/imroc/req/v3"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/util"
textUnicode "golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
@ -374,7 +375,7 @@ func installPackage(data []byte, installPath string) (err error) {
}
}
srcPath := filepath.Join(unzipPath, dir)
if err = util.Copy(srcPath, installPath); nil != err {
if err = filesys.Copy(srcPath, installPath); nil != err {
return
}
return

67
kernel/filesys/io.go Normal file
View file

@ -0,0 +1,67 @@
package filesys
import (
"io"
"os"
"sync"
"github.com/88250/gulu"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
)
var writingFileLock = sync.Mutex{}
func LockWriteFile() {
writingFileLock.Lock()
}
func UnlockWriteFile() {
writingFileLock.Unlock()
}
func WriteFileSaferByReader(writePath string, reader io.Reader) (err error) {
writingFileLock.Lock()
defer writingFileLock.Unlock()
if err = gulu.File.WriteFileSaferByReader(writePath, reader, 0644); nil != err {
logging.LogErrorf("write file [%s] failed: %s", writePath, err)
return
}
return
}
func WriteFileSafer(writePath string, data []byte) (err error) {
writingFileLock.Lock()
defer writingFileLock.Unlock()
if err = gulu.File.WriteFileSafer(writePath, data, 0644); nil != err {
logging.LogErrorf("write file [%s] failed: %s", writePath, err)
return
}
return
}
func Copy(source, dest string) (err error) {
writingFileLock.Lock()
defer writingFileLock.Unlock()
filelock.ReleaseFileLocks(source)
if err = gulu.File.Copy(source, dest); nil != err {
logging.LogErrorf("copy [%s] to [%s] failed: %s", source, dest, err)
return
}
return
}
func RemoveAll(p string) (err error) {
writingFileLock.Lock()
defer writingFileLock.Unlock()
filelock.ReleaseFileLocks(p)
if err = os.RemoveAll(p); nil != err {
logging.LogErrorf("remove all [%s] failed: %s", p, err)
return
}
return
}

7
kernel/model/assets.go
View file

@ -39,6 +39,7 @@ import (
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/search"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
@ -141,7 +142,7 @@ func NetImg2LocalAssets(rootID string) (err error) {
name = util.FilterFileName(name)
name = "net-img-" + name + "-" + ast.NewNodeID() + ext
writePath := filepath.Join(util.DataDir, "assets", name)
if err = util.WriteFileSafer(writePath, data); nil != err {
if err = filesys.WriteFileSafer(writePath, data); nil != err {
logging.LogErrorf("write downloaded net img [%s] to local assets [%s] failed: %s", u, writePath, err)
return ast.WalkSkipChildren
}
@ -383,7 +384,7 @@ func saveWorkspaceAssets(assets []string) {
logging.LogErrorf("create assets conf failed: %s", err)
return
}
if err = util.WriteFileSafer(confPath, data); nil != err {
if err = filesys.WriteFileSafer(confPath, data); nil != err {
logging.LogErrorf("write assets conf failed: %s", err)
return
}
@ -478,7 +479,7 @@ func RenameAsset(oldPath, newName string) (err error) {
newName = util.AssetName(newName) + filepath.Ext(oldPath)
newPath := "assets/" + newName
if err = util.Copy(filepath.Join(util.DataDir, oldPath), filepath.Join(util.DataDir, newPath)); nil != err {
if err = filesys.Copy(filepath.Join(util.DataDir, oldPath), filepath.Join(util.DataDir, newPath)); nil != err {
logging.LogErrorf("copy asset [%s] failed: %s", oldPath, err)
return
}

9
kernel/model/box.go
View file

@ -35,6 +35,7 @@ import (
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -202,7 +203,7 @@ func (box *Box) saveConf0(data []byte) {
if err := os.MkdirAll(filepath.Join(util.DataDir, box.ID, ".siyuan"), 0755); nil != err {
logging.LogErrorf("save box conf [%s] failed: %s", confPath, err)
}
if err := util.WriteFileSafer(confPath, data); nil != err {
if err := filesys.WriteFileSafer(confPath, data); nil != err {
logging.LogErrorf("save box conf [%s] failed: %s", confPath, err)
}
}
@ -292,8 +293,8 @@ func (box *Box) Move(oldPath, newPath string) error {
toPath := filepath.Join(boxLocalPath, newPath)
filelock.ReleaseFileLocks(fromPath)
util.LockWriteFile()
defer util.UnlockWriteFile()
filesys.LockWriteFile()
defer filesys.UnlockWriteFile()
if err := os.Rename(fromPath, toPath); nil != err {
msg := fmt.Sprintf(Conf.Language(5), box.Name, fromPath, err)
logging.LogErrorf("move [path=%s] in box [%s] failed: %s", fromPath, box.Name, err)
@ -313,7 +314,7 @@ func (box *Box) Move(oldPath, newPath string) error {
func (box *Box) Remove(path string) error {
boxLocalPath := filepath.Join(util.DataDir, box.ID)
filePath := filepath.Join(boxLocalPath, path)
if err := util.RemoveAll(filePath); nil != err {
if err := filesys.RemoveAll(filePath); nil != err {
msg := fmt.Sprintf(Conf.Language(7), box.Name, path, err)
logging.LogErrorf("remove [path=%s] in box [%s] failed: %s", path, box.ID, err)
return errors.New(msg)

5
kernel/model/export.go
View file

@ -42,6 +42,7 @@ import (
"github.com/emirpasic/gods/stacks/linkedliststack"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -158,8 +159,8 @@ func exportData(exportFolder string) (err error) {
return
}
util.LockWriteFile()
defer util.UnlockWriteFile()
filesys.LockWriteFile()
defer filesys.UnlockWriteFile()
err = filelock.ReleaseAllFileLocks()
if nil != err {

6
kernel/model/file.go
View file

@ -1203,7 +1203,7 @@ func RemoveDoc(boxID, p string) (err error) {
historyPath := filepath.Join(historyDir, boxID, p)
absPath := filepath.Join(util.DataDir, boxID, p)
if err = util.Copy(absPath, historyPath); nil != err {
if err = filesys.Copy(absPath, historyPath); nil != err {
return errors.New(fmt.Sprintf(Conf.Language(70), box.Name, absPath, err))
}
@ -1534,8 +1534,8 @@ func ChangeFileTreeSort(boxID string, paths []string) {
}
WaitForWritingFiles()
util.LockWriteFile()
defer util.UnlockWriteFile()
filesys.LockWriteFile()
defer filesys.UnlockWriteFile()
box := Conf.Box(boxID)
sortIDs := map[string]int{}

11
kernel/model/history.go
View file

@ -37,6 +37,7 @@ import (
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/search"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
@ -238,7 +239,7 @@ func RollbackDocHistory(boxID, historyPath string) (err error) {
}
WaitForWritingFiles()
util.LockWriteFile()
filesys.LockWriteFile()
srcPath := historyPath
var destPath string
@ -249,22 +250,22 @@ func RollbackDocHistory(boxID, historyPath string) (err error) {
workingDoc := treenode.GetBlockTree(id)
if nil != workingDoc {
if err = os.RemoveAll(filepath.Join(util.DataDir, boxID, workingDoc.Path)); nil != err {
util.UnlockWriteFile()
filesys.UnlockWriteFile()
return
}
}
destPath, err = getRollbackDockPath(boxID, historyPath)
if nil != err {
util.UnlockWriteFile()
filesys.UnlockWriteFile()
return
}
if err = gulu.File.Copy(srcPath, destPath); nil != err {
util.UnlockWriteFile()
filesys.UnlockWriteFile()
return
}
util.UnlockWriteFile()
filesys.UnlockWriteFile()
FullReindex()
IncSync()

10
kernel/model/import.go
View file

@ -259,7 +259,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
for _, assets := range assetsDirs {
if gulu.File.IsDir(assets) {
dataAssets := filepath.Join(util.DataDir, "assets")
if err = util.Copy(assets, dataAssets); nil != err {
if err = filesys.Copy(assets, dataAssets); nil != err {
logging.LogErrorf("copy assets from [%s] to [%s] failed: %s", assets, dataAssets, err)
return
}
@ -267,8 +267,8 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
os.RemoveAll(assets)
}
util.LockWriteFile()
defer util.UnlockWriteFile()
filesys.LockWriteFile()
defer filesys.UnlockWriteFile()
filelock.ReleaseAllFileLocks()
@ -331,8 +331,8 @@ func ImportData(zipPath string) (err error) {
return errors.New("invalid data.zip")
}
util.LockWriteFile()
defer util.UnlockWriteFile()
filesys.LockWriteFile()
defer filesys.UnlockWriteFile()
filelock.ReleaseAllFileLocks()
tmpDataPath := filepath.Join(unzipPath, dirs[0].Name())

9
kernel/model/mount.go
View file

@ -29,13 +29,12 @@ import (
"github.com/88250/lute/ast"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
func CreateBox(name string) (id string, err error) {
WaitForWritingFiles()
id = ast.NewNodeID()
boxLocalPath := filepath.Join(util.DataDir, id)
err = os.MkdirAll(boxLocalPath, 0755)
@ -52,10 +51,6 @@ func CreateBox(name string) (id string, err error) {
}
func RenameBox(boxID, name string) (err error) {
WaitForWritingFiles()
util.LockWriteFile()
defer util.UnlockWriteFile()
box := Conf.Box(boxID)
if nil == box {
return errors.New(Conf.Language(0))
@ -103,7 +98,7 @@ func RemoveBox(boxID string) (err error) {
}
unmount0(boxID)
if err = os.RemoveAll(localPath); nil != err {
if err = filesys.RemoveAll(localPath); nil != err {
return
}
IncSync()

5
kernel/model/repository.go
View file

@ -40,6 +40,7 @@ import (
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -221,8 +222,8 @@ func CheckoutRepo(id string) (err error) {
}
util.PushEndlessProgress(Conf.Language(63))
util.LockWriteFile()
defer util.UnlockWriteFile()
filesys.LockWriteFile()
defer filesys.UnlockWriteFile()
WaitForWritingFiles()
sql.WaitForWritingDatabase()
filelock.ReleaseAllFileLocks()

5
kernel/model/sync.go
View file

@ -32,6 +32,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/siyuan-note/dejavu"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -63,8 +64,8 @@ func SyncData(boot, exit, byHand bool) {
return
}
util.LockWriteFile()
defer util.UnlockWriteFile()
filesys.LockWriteFile()
defer filesys.UnlockWriteFile()
if util.IsMutexLocked(&syncLock) {
logging.LogWarnf("sync is in progress")

5
kernel/model/upload.go
View file

@ -28,6 +28,7 @@ import (
"github.com/88250/lute/ast"
"github.com/gin-gonic/gin"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
@ -85,7 +86,7 @@ func InsertLocalAssets(id string, assetPaths []string) (succMap map[string]inter
f.Close()
return
}
if err = util.WriteFileSaferByReader(writePath, f); nil != err {
if err = filesys.WriteFileSaferByReader(writePath, f); nil != err {
f.Close()
return
}
@ -168,7 +169,7 @@ func Upload(c *gin.Context) {
f.Close()
break
}
if err = util.WriteFileSaferByReader(writePath, f); nil != err {
if err = filesys.WriteFileSaferByReader(writePath, f); nil != err {
errFiles = append(errFiles, fName)
ret.Msg = err.Error()
f.Close()

68
kernel/util/file.go
View file

@ -17,85 +17,17 @@
package util
import (
"io"
"os"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/88250/gulu"
"github.com/88250/lute/ast"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
)
var writingFileLock = sync.Mutex{}
func LockWriteFile() {
if IsMutexLocked(&writingFileLock) {
logging.LogWarnf("write file is locked")
return
}
writingFileLock.Lock()
}
func UnlockWriteFile() {
if !IsMutexLocked(&writingFileLock) {
logging.LogWarnf("write file is not locked")
return
}
writingFileLock.Unlock()
}
func WriteFileSaferByReader(writePath string, reader io.Reader) (err error) {
writingFileLock.Lock()
defer writingFileLock.Unlock()
if err = gulu.File.WriteFileSaferByReader(writePath, reader, 0644); nil != err {
logging.LogErrorf("write file [%s] failed: %s", writePath, err)
return
}
return
}
func WriteFileSafer(writePath string, data []byte) (err error) {
writingFileLock.Lock()
defer writingFileLock.Unlock()
if err = gulu.File.WriteFileSafer(writePath, data, 0644); nil != err {
logging.LogErrorf("write file [%s] failed: %s", writePath, err)
return
}
return
}
func Copy(source, dest string) (err error) {
writingFileLock.Lock()
defer writingFileLock.Unlock()
filelock.ReleaseFileLocks(source)
if err = gulu.File.Copy(source, dest); nil != err {
logging.LogErrorf("copy [%s] to [%s] failed: %s", source, dest, err)
return
}
return
}
func RemoveAll(p string) (err error) {
writingFileLock.Lock()
defer writingFileLock.Unlock()
filelock.ReleaseFileLocks(p)
if err = os.RemoveAll(p); nil != err {
logging.LogErrorf("remove all [%s] failed: %s", p, err)
return
}
return
}
func IsEmptyDir(p string) bool {
if !gulu.File.IsDir(p) {
return false