mirror of
https://github.com/siyuan-note/siyuan.git
synced 2026-01-01 22:38:49 +01:00
🎨 Sync repo
This commit is contained in:
Liang Ding
parent
3d71a2644b
commit
eaa83998a4
6 changed files with 86 additions and 81 deletions
|
|
@ -40,7 +40,7 @@ require (
|
|||
github.com/qiniu/go-sdk/v7 v7.13.0
|
||||
github.com/radovskyb/watcher v1.0.7
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
|
||||
github.com/siyuan-note/dejavu v0.0.0-20220627141634-53a7e4ef9840
|
||||
github.com/siyuan-note/dejavu v0.0.0-20220630143851-b239e9224291
|
||||
github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676
|
||||
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f
|
||||
github.com/siyuan-note/filelock v0.0.0-20220616063212-74cfba0754ee
|
||||
|
|
@ -83,7 +83,7 @@ require (
|
|||
github.com/imdario/mergo v0.3.13 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/juju/errors v0.0.0-20220331221717-b38fca44723b // indirect
|
||||
github.com/klauspost/compress v1.15.6 // indirect
|
||||
github.com/klauspost/compress v1.15.7 // indirect
|
||||
github.com/leodido/go-urn v1.2.1 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||
|
|
@ -103,7 +103,7 @@ require (
|
|||
golang.org/x/mod v0.5.1 // indirect
|
||||
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
|
||||
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
|
||||
golang.org/x/sys v0.0.0-20220624220833-87e55d714810 // indirect
|
||||
golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect
|
||||
golang.org/x/tools v0.1.8 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
google.golang.org/protobuf v1.28.0 // indirect
|
||||
|
|
|
|||
|
|
@ -320,8 +320,8 @@ github.com/juju/testing v0.0.0-20191001232224-ce9dec17d28b/go.mod h1:63prj8cnj0t
|
|||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||
github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
|
||||
github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok=
|
||||
github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
|
|
@ -421,8 +421,8 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ
|
|||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/siyuan-note/dejavu v0.0.0-20220627141634-53a7e4ef9840 h1:H/Gb6d/Q9/7LK/T9KuuqIXlZGdj5rLFIVNFsJ00r00U=
|
||||
github.com/siyuan-note/dejavu v0.0.0-20220627141634-53a7e4ef9840/go.mod h1:0t+TpAdsrrv+6LkhDlJY2Gl92CQexgp8KWOBp5Ah9kU=
|
||||
github.com/siyuan-note/dejavu v0.0.0-20220630143851-b239e9224291 h1:q6ZASwpPzlLseZIK0B37p3IQ8mTN4HnyFhD7NN3oVtI=
|
||||
github.com/siyuan-note/dejavu v0.0.0-20220630143851-b239e9224291/go.mod h1:0t+TpAdsrrv+6LkhDlJY2Gl92CQexgp8KWOBp5Ah9kU=
|
||||
github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 h1:QB9TjJQFhXhZ6dAtPpY02DlzHAQm1C+WqZq6OadG8mI=
|
||||
github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
|
||||
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f h1:JMobMNZ7AqaKKyEK+WeWFhix/2TDQXgPZDajU00IybU=
|
||||
|
|
@ -662,8 +662,8 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220624220833-87e55d714810 h1:rHZQSjJdAI4Xf5Qzeh2bBc5YJIkPFVM6oDtMFYmgws0=
|
||||
golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8=
|
||||
golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.0.0-20180302201248-b7ef84aaf62a/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
|
|||
|
|
@ -488,13 +488,9 @@ func InitBoxes() {
|
|||
util.IncBootProgress(1, "Reading block trees...")
|
||||
}
|
||||
}()
|
||||
if err := treenode.ReadBlockTree(); nil == err {
|
||||
initialized = true
|
||||
} else {
|
||||
if err = os.RemoveAll(util.BlockTreePath); nil != err {
|
||||
util.LogErrorf("remove block tree [%s] failed: %s", util.BlockTreePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
treenode.InitBlockTree()
|
||||
initialized = true
|
||||
}
|
||||
} else { // 大于 1 的话说明在同步阶段已经加载过了
|
||||
initialized = true
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import (
|
|||
"github.com/siyuan-note/encryption"
|
||||
"github.com/siyuan-note/eventbus"
|
||||
"github.com/siyuan-note/filelock"
|
||||
"github.com/siyuan-note/siyuan/kernel/cache"
|
||||
"github.com/siyuan-note/siyuan/kernel/sql"
|
||||
"github.com/siyuan-note/siyuan/kernel/util"
|
||||
)
|
||||
|
|
@ -317,7 +318,7 @@ func CheckoutRepo(id string) (err error) {
|
|||
Conf.Sync.Enabled = false
|
||||
Conf.Save()
|
||||
|
||||
err = repo.Checkout(id, map[string]interface{}{
|
||||
_, _, err = repo.Checkout(id, map[string]interface{}{
|
||||
CtxPushMsg: CtxPushMsgToStatusBarAndProgress,
|
||||
})
|
||||
if nil != err {
|
||||
|
|
@ -444,12 +445,45 @@ func syncRepo() (err error) {
|
|||
}
|
||||
|
||||
start := time.Now()
|
||||
err = repo.Sync(Conf.Sync.CloudName, Conf.User.UserId, Conf.User.UserToken, Conf.System.NetworkProxy.String(), util.AliyunServer, map[string]interface{}{
|
||||
latest, fetchedFiles, err := repo.Sync(Conf.Sync.CloudName, Conf.User.UserId, Conf.User.UserToken, Conf.System.NetworkProxy.String(), util.AliyunServer, map[string]interface{}{
|
||||
CtxPushMsg: CtxPushMsgToStatusBar,
|
||||
})
|
||||
elapsed := time.Since(start)
|
||||
util.LogInfof("sync data repo elapsed [%.2fs]", elapsed.Seconds())
|
||||
if nil != err {
|
||||
util.LogErrorf("sync data repo failed: %s", err)
|
||||
util.PushStatusBar("Sync data repo failed: " + err.Error())
|
||||
return
|
||||
}
|
||||
util.PushStatusBar(fmt.Sprintf(Conf.Language(149), elapsed.Seconds()))
|
||||
if 1 > len(fetchedFiles) {
|
||||
// 没有下载到新文件，直接返回
|
||||
return
|
||||
}
|
||||
|
||||
// 下载到文件后，需要恢复到工作区并重建索引
|
||||
|
||||
CloseWatchAssets()
|
||||
defer WatchAssets()
|
||||
|
||||
upsertFiles, removeFiles, err := repo.Checkout(latest.ID, map[string]interface{}{
|
||||
CtxPushMsg: CtxPushMsgToStatusBarAndProgress,
|
||||
})
|
||||
if nil != err {
|
||||
util.PushClearProgress()
|
||||
return
|
||||
}
|
||||
|
||||
var upserts, removes []string
|
||||
for _, file := range upsertFiles {
|
||||
upserts = append(upserts, file.Path)
|
||||
}
|
||||
for _, file := range removeFiles {
|
||||
removes = append(removes, file.Path)
|
||||
}
|
||||
incReindex(upserts, removes)
|
||||
cache.ClearDocsIAL()
|
||||
util.ReloadUI()
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -401,71 +401,13 @@ func SyncData(boot, exit, byHand bool) {
|
|||
|
||||
if boot && gulu.File.IsExist(util.BlockTreePath) {
|
||||
// 在 blocktree 存在的情况下不会重建索引，所以这里需要刷新 blocktree 和 database
|
||||
|
||||
if err = treenode.ReadBlockTree(); nil != err {
|
||||
os.RemoveAll(util.BlockTreePath)
|
||||
util.LogWarnf("removed block tree [%s] due to %s", util.BlockTreePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, upsertFile := range upsertFiles {
|
||||
if !strings.HasSuffix(upsertFile, ".sy") {
|
||||
continue
|
||||
}
|
||||
|
||||
upsertFile = filepath.ToSlash(upsertFile)
|
||||
box := upsertFile[:strings.Index(upsertFile, "/")]
|
||||
p := strings.TrimPrefix(upsertFile, box)
|
||||
tree, err0 := LoadTree(box, p)
|
||||
if nil != err0 {
|
||||
continue
|
||||
}
|
||||
treenode.ReindexBlockTree(tree)
|
||||
sql.UpsertTreeQueue(tree)
|
||||
}
|
||||
for _, removeFile := range removeFiles {
|
||||
if !strings.HasSuffix(removeFile, ".sy") {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSuffix(filepath.Base(removeFile), ".sy")
|
||||
block := treenode.GetBlockTree(id)
|
||||
if nil != block {
|
||||
treenode.RemoveBlockTreesByRootID(block.RootID)
|
||||
sql.RemoveTreeQueue(block.BoxID, block.RootID)
|
||||
}
|
||||
}
|
||||
treenode.InitBlockTree()
|
||||
incReindex(upsertFiles, removeFiles)
|
||||
return
|
||||
}
|
||||
|
||||
if !boot && !exit {
|
||||
// 增量索引
|
||||
for _, upsertFile := range upsertFiles {
|
||||
if !strings.HasSuffix(upsertFile, ".sy") {
|
||||
continue
|
||||
}
|
||||
|
||||
upsertFile = filepath.ToSlash(upsertFile)
|
||||
box := upsertFile[:strings.Index(upsertFile, "/")]
|
||||
p := strings.TrimPrefix(upsertFile, box)
|
||||
tree, err0 := LoadTree(box, p)
|
||||
if nil != err0 {
|
||||
continue
|
||||
}
|
||||
treenode.ReindexBlockTree(tree)
|
||||
sql.UpsertTreeQueue(tree)
|
||||
//util.LogInfof("sync index tree [%s]", tree.ID)
|
||||
}
|
||||
for _, removeFile := range removeFiles {
|
||||
if !strings.HasSuffix(removeFile, ".sy") {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSuffix(filepath.Base(removeFile), ".sy")
|
||||
block := treenode.GetBlockTree(id)
|
||||
if nil != block {
|
||||
treenode.RemoveBlockTreesByRootID(block.RootID)
|
||||
sql.RemoveTreeQueue(block.BoxID, block.RootID)
|
||||
//util.LogInfof("sync remove tree [%s]", block.RootID)
|
||||
}
|
||||
}
|
||||
incReindex(upsertFiles, removeFiles)
|
||||
cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939
|
||||
util.ReloadUI()
|
||||
}
|
||||
|
|
@ -495,6 +437,38 @@ func clearEmptyDirs(dir string) {
|
|||
}
|
||||
}
|
||||
|
||||
// incReindex 增量重建索引。
|
||||
func incReindex(upserts, removes []string) {
|
||||
for _, upsertFile := range upserts {
|
||||
if !strings.HasSuffix(upsertFile, ".sy") {
|
||||
continue
|
||||
}
|
||||
|
||||
upsertFile = filepath.ToSlash(upsertFile)
|
||||
box := upsertFile[:strings.Index(upsertFile, "/")]
|
||||
p := strings.TrimPrefix(upsertFile, box)
|
||||
tree, err0 := LoadTree(box, p)
|
||||
if nil != err0 {
|
||||
continue
|
||||
}
|
||||
treenode.ReindexBlockTree(tree)
|
||||
sql.UpsertTreeQueue(tree)
|
||||
//util.LogInfof("sync index tree [%s]", tree.ID)
|
||||
}
|
||||
for _, removeFile := range removes {
|
||||
if !strings.HasSuffix(removeFile, ".sy") {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSuffix(filepath.Base(removeFile), ".sy")
|
||||
block := treenode.GetBlockTree(id)
|
||||
if nil != block {
|
||||
treenode.RemoveBlockTreesByRootID(block.RootID)
|
||||
sql.RemoveTreeQueue(block.BoxID, block.RootID)
|
||||
//util.LogInfof("sync remove tree [%s]", block.RootID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SetCloudSyncDir(name string) {
|
||||
if Conf.Sync.CloudName == name {
|
||||
return
|
||||
|
|
|
|||
|
|
@ -217,13 +217,14 @@ func AutoFlushBlockTree() {
|
|||
}
|
||||
}
|
||||
|
||||
func ReadBlockTree() (err error) {
|
||||
func InitBlockTree() {
|
||||
start := time.Now()
|
||||
|
||||
if nil == blocktreeFileLock {
|
||||
blocktreeFileLock = flock.New(util.BlockTreePath)
|
||||
}
|
||||
|
||||
var err error
|
||||
if err = blocktreeFileLock.Lock(); nil != err {
|
||||
util.LogErrorf("read block tree failed: %s", err)
|
||||
os.Exit(util.ExitCodeBlockTreeErr)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue