Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2026-02-23 15:34:06 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

🐛 Notebook data may be corrupted during data synchronization https://github.com/siyuan-note/siyuan/issues/9594

Browse source
This commit is contained in:
Daniel 2023-11-06 22:13:04 +08:00
parent a721bbe6f0
commit 70c56c8e25
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
24 changed files with 88 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

12
kernel/model/assets.go
View file

@ -440,7 +440,7 @@ func RemoveUnusedAssets() (ret []string) {
var hashes []string
for _, p := range unusedAssets {
historyPath := filepath.Join(historyDir, p)
if p = filepath.Join(util.DataDir, p); gulu.File.IsExist(p) {
if p = filepath.Join(util.DataDir, p); filelock.IsExist(p) {
if err = filelock.Copy(p, historyPath); nil != err {
return
}
@ -453,7 +453,7 @@ func RemoveUnusedAssets() (ret []string) {
sql.BatchRemoveAssetsQueue(hashes)
for _, unusedAsset := range unusedAssets {
if unusedAsset = filepath.Join(util.DataDir, unusedAsset); gulu.File.IsExist(unusedAsset) {
if unusedAsset = filepath.Join(util.DataDir, unusedAsset); filelock.IsExist(unusedAsset) {
if err := os.RemoveAll(unusedAsset); nil != err {
logging.LogErrorf("remove unused asset [%s] failed: %s", unusedAsset, err)
}
@ -471,7 +471,7 @@ func RemoveUnusedAssets() (ret []string) {
func RemoveUnusedAsset(p string) (ret string) {
absPath := filepath.Join(util.DataDir, p)
if !gulu.File.IsExist(absPath) {
if !filelock.IsExist(absPath) {
return absPath
}
@ -483,7 +483,7 @@ func RemoveUnusedAsset(p string) (ret string) {
newP := strings.TrimPrefix(absPath, util.DataDir)
historyPath := filepath.Join(historyDir, newP)
if gulu.File.IsExist(absPath) {
if filelock.IsExist(absPath) {
if err = filelock.Copy(absPath, historyPath); nil != err {
return
}
@ -528,7 +528,7 @@ func RenameAsset(oldPath, newName string) (err error) {
return
}
if gulu.File.IsExist(filepath.Join(util.DataDir, oldPath+".sya")) {
if filelock.IsExist(filepath.Join(util.DataDir, oldPath+".sya")) {
// Rename the .sya annotation file when renaming a PDF asset https://github.com/siyuan-note/siyuan/issues/9390
if err = filelock.Copy(filepath.Join(util.DataDir, oldPath+".sya"), filepath.Join(util.DataDir, newPath+".sya")); nil != err {
logging.LogErrorf("copy PDF annotation [%s] failed: %s", oldPath+".sya", err)
@ -804,7 +804,7 @@ func MissingAssets() (ret []string) {
if "" == assetsPathMap[dest] {
if strings.HasPrefix(dest, "assets/.") {
// Assets starting with `.` should not be considered missing assets https://github.com/siyuan-note/siyuan/issues/8821
if !gulu.File.IsExist(filepath.Join(util.DataDir, dest)) {
if !filelock.IsExist(filepath.Join(util.DataDir, dest)) {
ret = append(ret, dest)
}
} else {

3
kernel/model/attribute_view.go
View file

@ -27,6 +27,7 @@ import (
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/Masterminds/sprig/v3"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/av"
"github.com/siyuan-note/siyuan/kernel/treenode"
@ -183,7 +184,7 @@ func GetBlockAttributeViewKeys(blockID string) (ret []*BlockAttributeViewKeys) {
func RenderAttributeView(avID string) (viewable av.Viewable, attrView *av.AttributeView, err error) {
waitForSyncingStorages()
if avJSONPath := av.GetAttributeViewDataPath(avID); !gulu.File.IsExist(avJSONPath) {
if avJSONPath := av.GetAttributeViewDataPath(avID); !filelock.IsExist(avJSONPath) {
attrView = av.NewAttributeView(avID)
if err = av.SaveAttributeView(attrView); nil != err {
logging.LogErrorf("save attribute view [%s] failed: %s", avID, err)

6
kernel/model/box.go
View file

@ -111,7 +111,7 @@ func ListNotebooks() (ret []*Box, err error) {
boxConf := conf.NewBoxConf()
boxDirPath := filepath.Join(util.DataDir, dir.Name())
boxConfPath := filepath.Join(boxDirPath, ".siyuan", "conf.json")
if !gulu.File.IsExist(boxConfPath) {
if !filelock.IsExist(boxConfPath) {
// Automatically move corrupted notebook folders to the corrupted folder https://github.com/siyuan-note/siyuan/issues/9202
logging.LogWarnf("found a corrupted box [%s]", boxDirPath)
to := filepath.Join(util.WorkspaceDir, "corrupted", time.Now().Format("2006-01-02-150405"), dir.Name())
@ -184,7 +184,7 @@ func (box *Box) GetConf() (ret *conf.BoxConf) {
ret = conf.NewBoxConf()
confPath := filepath.Join(util.DataDir, box.ID, ".siyuan/conf.json")
if !gulu.File.IsExist(confPath) {
if !filelock.IsExist(confPath) {
return
}
@ -305,7 +305,7 @@ func (box *Box) Stat(p string) (ret *FileInfo) {
}
func (box *Box) Exist(p string) bool {
return gulu.File.IsExist(filepath.Join(util.DataDir, box.ID, p))
return filelock.IsExist(filepath.Join(util.DataDir, box.ID, p))
}
func (box *Box) Mkdir(path string) error {

4
kernel/model/conf.go
View file

@ -812,7 +812,7 @@ func clearCorruptedNotebooks() {
boxDirPath := filepath.Join(util.DataDir, dir.Name())
boxConfPath := filepath.Join(boxDirPath, ".siyuan", "conf.json")
if !gulu.File.IsExist(boxConfPath) {
if !filelock.IsExist(boxConfPath) {
logging.LogWarnf("found a corrupted box [%s]", boxDirPath)
continue
}
@ -898,7 +898,7 @@ func upgradeUserGuide() {
boxDirPath := filepath.Join(util.DataDir, boxID)
boxConf := conf.NewBoxConf()
boxConfPath := filepath.Join(boxDirPath, ".siyuan", "conf.json")
if !gulu.File.IsExist(boxConfPath) {
if !filelock.IsExist(boxConfPath) {
logging.LogWarnf("found a corrupted box [%s]", boxDirPath)
continue
}

8
kernel/model/export.go
View file

@ -1347,7 +1347,7 @@ func exportSYZip(boxID, rootDirPath, baseFolderName string, docPaths []string) (
if !gulu.File.IsDir(srcPath) && strings.HasSuffix(strings.ToLower(srcPath), ".pdf") {
sya := srcPath + ".sya"
if gulu.File.IsExist(sya) {
if filelock.IsExist(sya) {
// Related PDF annotation information is not exported when exporting .sy.zip https://github.com/siyuan-note/siyuan/issues/7836
if syaErr := filelock.Copy(sya, destPath+".sya"); nil != syaErr {
logging.LogErrorf("copy sya from [%s] to [%s] failed: %s", sya, destPath+".sya", syaErr)
@ -1373,7 +1373,7 @@ func exportSYZip(boxID, rootDirPath, baseFolderName string, docPaths []string) (
avID := n.AttributeViewID
avJSONPath := av.GetAttributeViewDataPath(avID)
if !gulu.File.IsExist(avJSONPath) {
if !filelock.IsExist(avJSONPath) {
return ast.WalkContinue
}
@ -1410,7 +1410,7 @@ func exportSYZip(boxID, rootDirPath, baseFolderName string, docPaths []string) (
sortIDs := map[string]int{}
var sortData []byte
var sortErr error
if gulu.File.IsExist(sortPath) {
if filelock.IsExist(sortPath) {
sortData, sortErr = filelock.ReadFile(sortPath)
if nil != sortErr {
logging.LogErrorf("read sort conf failed: %s", sortErr)
@ -1900,7 +1900,7 @@ func exportTree(tree *parse.Tree, wysiwyg, expandKaTexMacros, keepFold bool,
}
avID := n.AttributeViewID
if avJSONPath := av.GetAttributeViewDataPath(avID); !gulu.File.IsExist(avJSONPath) {
if avJSONPath := av.GetAttributeViewDataPath(avID); !filelock.IsExist(avJSONPath) {
return ast.WalkContinue
}

14
kernel/model/file.go
View file

@ -1205,7 +1205,7 @@ func moveDoc(fromBox *Box, fromPath string, toBox *Box, toPath string, luteEngin
} else {
absFromPath := filepath.Join(util.DataDir, fromBox.ID, fromFolder)
absToPath := filepath.Join(util.DataDir, toBox.ID, newFolder)
if gulu.File.IsExist(absToPath) {
if filelock.IsExist(absToPath) {
filelock.Remove(absToPath)
}
if err = filelock.Rename(absFromPath, absToPath); nil != err {
@ -1445,7 +1445,7 @@ func CreateDailyNote(boxID string) (p string, existed bool, err error) {
var dom string
if "" != boxConf.DailyNoteTemplatePath {
tplPath := filepath.Join(util.DataDir, "templates", boxConf.DailyNoteTemplatePath)
if !gulu.File.IsExist(tplPath) {
if !filelock.IsExist(tplPath) {
logging.LogWarnf("not found daily note template [%s]", tplPath)
} else {
dom, err = renderTemplate(tplPath, id, false)
@ -1570,7 +1570,7 @@ func moveSorts(rootID, fromBox, toBox string) {
ids := treenode.RootChildIDs(rootID)
fromConfPath := filepath.Join(util.DataDir, fromBox, ".siyuan", "sort.json")
fromFullSortIDs := map[string]int{}
if gulu.File.IsExist(fromConfPath) {
if filelock.IsExist(fromConfPath) {
data, err := filelock.ReadFile(fromConfPath)
if nil != err {
logging.LogErrorf("read sort conf failed: %s", err)
@ -1587,7 +1587,7 @@ func moveSorts(rootID, fromBox, toBox string) {
toConfPath := filepath.Join(util.DataDir, toBox, ".siyuan", "sort.json")
toFullSortIDs := map[string]int{}
if gulu.File.IsExist(toConfPath) {
if filelock.IsExist(toConfPath) {
data, err := filelock.ReadFile(toConfPath)
if nil != err {
logging.LogErrorf("read sort conf failed: %s", err)
@ -1663,7 +1663,7 @@ func ChangeFileTreeSort(boxID string, paths []string) {
confPath := filepath.Join(confDir, "sort.json")
fullSortIDs := map[string]int{}
var data []byte
if gulu.File.IsExist(confPath) {
if filelock.IsExist(confPath) {
data, err = filelock.ReadFile(confPath)
if nil != err {
logging.LogErrorf("read sort conf failed: %s", err)
@ -1694,7 +1694,7 @@ func ChangeFileTreeSort(boxID string, paths []string) {
func (box *Box) fillSort(files *[]*File) {
confPath := filepath.Join(util.DataDir, box.ID, ".siyuan", "sort.json")
if !gulu.File.IsExist(confPath) {
if !filelock.IsExist(confPath) {
return
}
@ -1718,7 +1718,7 @@ func (box *Box) fillSort(files *[]*File) {
func (box *Box) removeSort(ids []string) {
confPath := filepath.Join(util.DataDir, box.ID, ".siyuan", "sort.json")
if !gulu.File.IsExist(confPath) {
if !filelock.IsExist(confPath) {
return
}

9
kernel/model/flashcard.go
View file

@ -30,6 +30,7 @@ import (
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/open-spaced-repetition/go-fsrs"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/riff"
"github.com/siyuan-note/siyuan/kernel/cache"
@ -814,15 +815,15 @@ func RemoveDeck(deckID string) (err error) {
riffSavePath := getRiffDir()
deckPath := filepath.Join(riffSavePath, deckID+".deck")
if gulu.File.IsExist(deckPath) {
if err = os.Remove(deckPath); nil != err {
if filelock.IsExist(deckPath) {
if err = filelock.Remove(deckPath); nil != err {
return
}
}
cardsPath := filepath.Join(riffSavePath, deckID+".cards")
if gulu.File.IsExist(cardsPath) {
if err = os.Remove(cardsPath); nil != err {
if filelock.IsExist(cardsPath) {
if err = filelock.Remove(cardsPath); nil != err {
return
}
}

5
kernel/model/history.go
View file

@ -142,6 +142,7 @@ func ClearWorkspaceHistory() (err error) {
func GetDocHistoryContent(historyPath, keyword string) (id, rootID, content string, isLargeDoc bool, err error) {
if !gulu.File.IsExist(historyPath) {
logging.LogWarnf("doc history [%s] not exist", historyPath)
return
}
@ -214,6 +215,7 @@ func GetDocHistoryContent(historyPath, keyword string) (id, rootID, content stri
func RollbackDocHistory(boxID, historyPath string) (err error) {
if !gulu.File.IsExist(historyPath) {
logging.LogWarnf("doc history [%s] not exist", historyPath)
return
}
@ -268,6 +270,7 @@ func getRollbackDockPath(boxID, historyPath string) (destPath string, err error)
func RollbackAssetsHistory(historyPath string) (err error) {
historyPath = filepath.Join(util.WorkspaceDir, historyPath)
if !gulu.File.IsExist(historyPath) {
logging.LogWarnf("assets history [%s] not exist", historyPath)
return
}
@ -285,6 +288,7 @@ func RollbackAssetsHistory(historyPath string) (err error) {
func RollbackNotebookHistory(historyPath string) (err error) {
if !gulu.File.IsExist(historyPath) {
logging.LogWarnf("notebook history [%s] not exist", historyPath)
return
}
@ -482,6 +486,7 @@ func (box *Box) generateDocHistory0() {
func clearOutdatedHistoryDir(historyDir string) {
if !gulu.File.IsExist(historyDir) {
logging.LogWarnf("history dir [%s] not exist", historyDir)
return
}

4
kernel/model/import.go
View file

@ -349,7 +349,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
var sortData []byte
var sortErr error
sortPath := filepath.Join(unzipRootPath, ".siyuan", "sort.json")
if gulu.File.IsExist(sortPath) {
if filelock.IsExist(sortPath) {
sortData, sortErr = filelock.ReadFile(sortPath)
if nil != sortErr {
logging.LogErrorf("read import sort conf failed: %s", sortErr)
@ -360,7 +360,7 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
}
boxSortPath := filepath.Join(util.DataDir, boxID, ".siyuan", "sort.json")
if gulu.File.IsExist(boxSortPath) {
if filelock.IsExist(boxSortPath) {
sortData, sortErr = filelock.ReadFile(boxSortPath)
if nil != sortErr {
logging.LogErrorf("read box sort conf failed: %s", sortErr)

2
kernel/model/mount.go
View file

@ -78,7 +78,7 @@ func RemoveBox(boxID string) (err error) {
}
localPath := filepath.Join(util.DataDir, boxID)
if !gulu.File.IsExist(localPath) {
if !filelock.IsExist(localPath) {
return
}
if !gulu.File.IsDir(localPath) {

18
kernel/model/ocr.go
View file

@ -1,7 +1,6 @@
package model
import (
"io"
"os"
"path/filepath"
"runtime/debug"
@ -10,6 +9,7 @@ import (
"github.com/88250/gulu"
"github.com/dustin/go-humanize"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/task"
@ -57,7 +57,7 @@ func cleanNotExistAssetsTexts() {
for asset, _ := range util.AssetsTexts {
assetAbsPath := strings.TrimPrefix(asset, "assets")
assetAbsPath = filepath.Join(assetsPath, assetAbsPath)
if !gulu.File.IsExist(assetAbsPath) {
if !filelock.IsExist(assetAbsPath) {
toRemoves = append(toRemoves, asset)
}
}
@ -98,20 +98,12 @@ func FlushAssetsTextsJob() {
func LoadAssetsTexts() {
assetsPath := util.GetDataAssetsAbsPath()
assetsTextsPath := filepath.Join(assetsPath, "ocr-texts.json")
if !gulu.File.IsExist(assetsTextsPath) {
if !filelock.IsExist(assetsTextsPath) {
return
}
start := time.Now()
var err error
fh, err := os.OpenFile(assetsTextsPath, os.O_RDWR, 0644)
if nil != err {
logging.LogErrorf("open assets texts failed: %s", err)
return
}
defer fh.Close()
data, err := io.ReadAll(fh)
data, err := filelock.ReadFile(assetsTextsPath)
if nil != err {
logging.LogErrorf("read assets texts failed: %s", err)
return
@ -151,7 +143,7 @@ func SaveAssetsTexts() {
assetsPath := util.GetDataAssetsAbsPath()
assetsTextsPath := filepath.Join(assetsPath, "ocr-texts.json")
if err = gulu.File.WriteFileSafer(assetsTextsPath, data, 0644); nil != err {
if err = filelock.WriteFile(assetsTextsPath, data); nil != err {
logging.LogErrorf("write assets texts failed: %s", err)
return
}

6
kernel/model/plugin.go
View file

@ -101,7 +101,7 @@ func LoadPetals(frontend string) (ret []*Petal) {
func loadCode(petal *Petal) {
pluginDir := filepath.Join(util.DataDir, "plugins", petal.Name)
jsPath := filepath.Join(pluginDir, "index.js")
if !gulu.File.IsExist(jsPath) {
if !filelock.IsExist(jsPath) {
logging.LogErrorf("plugin [%s] js not found", petal.Name)
return
}
@ -114,7 +114,7 @@ func loadCode(petal *Petal) {
petal.JS = string(data)
cssPath := filepath.Join(pluginDir, "index.css")
if gulu.File.IsExist(cssPath) {
if filelock.IsExist(cssPath) {
data, err = filelock.ReadFile(cssPath)
if nil != err {
logging.LogErrorf("read plugin [%s] css failed: %s", petal.Name, err)
@ -204,7 +204,7 @@ func getPetals() (ret []*Petal) {
}
confPath := filepath.Join(petalDir, "petals.json")
if !gulu.File.IsExist(confPath) {
if !filelock.IsExist(confPath) {
data, err := gulu.JSON.MarshalIndentJSON(ret, "", "\t")
if nil != err {
logging.LogErrorf("marshal petals failed: %s", err)

2
kernel/model/snippet.go
View file

@ -68,7 +68,7 @@ func LoadSnippets() (ret []*conf.Snippet, err error) {
func loadSnippets() (ret []*conf.Snippet, err error) {
ret = []*conf.Snippet{}
confPath := filepath.Join(util.SnippetsPath, "conf.json")
if !gulu.File.IsExist(confPath) {
if !filelock.IsExist(confPath) {
return
}

6
kernel/model/storage.go
View file

@ -129,7 +129,7 @@ func setRecentDocs(recentDocs []*RecentDoc) (err error) {
func getRecentDocs() (ret []*RecentDoc, err error) {
tmp := []*RecentDoc{}
dataPath := filepath.Join(util.DataDir, "storage/recent-doc.json")
if !gulu.File.IsExist(dataPath) {
if !filelock.IsExist(dataPath) {
return
}
@ -270,7 +270,7 @@ func setCriteria(criteria []*Criterion) (err error) {
func getCriteria() (ret []*Criterion, err error) {
ret = []*Criterion{}
dataPath := filepath.Join(util.DataDir, "storage/criteria.json")
if !gulu.File.IsExist(dataPath) {
if !filelock.IsExist(dataPath) {
return
}
@ -351,7 +351,7 @@ func getLocalStorage() (ret map[string]interface{}) {
// When local.json is corrupted, clear the file to avoid being unable to enter the main interface https://github.com/siyuan-note/siyuan/issues/7911
ret = map[string]interface{}{}
lsPath := filepath.Join(util.DataDir, "storage/local.json")
if !gulu.File.IsExist(lsPath) {
if !filelock.IsExist(lsPath) {
return
}

2
kernel/model/template.go
View file

@ -153,7 +153,7 @@ func DocSaveAsTemplate(id, name string, overwrite bool) (code int, err error) {
name = util.FilterFileName(name) + ".md"
name = util.TruncateLenFileName(name)
savePath := filepath.Join(util.DataDir, "templates", name)
if gulu.File.IsExist(savePath) {
if filelock.IsExist(savePath) {
if !overwrite {
code = 1
return

4
kernel/model/upload.go
View file

@ -278,9 +278,9 @@ func Upload(c *gin.Context) {
func getAssetsDir(boxLocalPath, docDirLocalPath string) (assets string) {
assets = filepath.Join(docDirLocalPath, "assets")
if !gulu.File.IsExist(assets) {
if !filelock.IsExist(assets) {
assets = filepath.Join(boxLocalPath, "assets")
if !gulu.File.IsExist(assets) {
if !filelock.IsExist(assets) {
assets = filepath.Join(util.DataDir, "assets")
}
}