From 262d1298dfef85db819c96611ca38a484ea7eac0 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 10 Nov 2023 22:09:44 +0800 Subject: [PATCH 1/4] :art: Improve data sync --- kernel/go.mod | 2 +- kernel/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index a12fb2d81..895b8e8ed 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -49,7 +49,7 @@ require ( github.com/radovskyb/watcher v1.0.7 github.com/sashabaranov/go-openai v1.17.5 github.com/shirou/gopsutil/v3 v3.23.10 - github.com/siyuan-note/dejavu v0.0.0-20231110121748-867038071de4 + github.com/siyuan-note/dejavu v0.0.0-20231110140645-e32f6a32cffa github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 github.com/siyuan-note/filelock v0.0.0-20231107122348-6ed75b0b525a diff --git a/kernel/go.sum b/kernel/go.sum index d665a7b3c..7c6c926b6 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -352,8 +352,8 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA= github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= -github.com/siyuan-note/dejavu v0.0.0-20231110121748-867038071de4 h1:370R/Uis+AY9ivXg671KgKA6iSksvylz9sfhiA2QVl4= -github.com/siyuan-note/dejavu v0.0.0-20231110121748-867038071de4/go.mod h1:6RdIofCDZP8vL0qYnVZfB12NhK7cUgh1ooUxRvRdlmI= +github.com/siyuan-note/dejavu v0.0.0-20231110140645-e32f6a32cffa h1:m7+UGC/nVyLKGy3NxWFJ1cT1sJDLUJ+D0sO/8cyN+xo= +github.com/siyuan-note/dejavu v0.0.0-20231110140645-e32f6a32cffa/go.mod h1:6RdIofCDZP8vL0qYnVZfB12NhK7cUgh1ooUxRvRdlmI= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 h1:XghjHKJd+SiL0DkGYFVC+UGUDFtnR4v9gkAbPeh9Eq8= From 22f78326f2fb766e901524f68ccfad58155c55af Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 10 Nov 2023 22:22:44 +0800 Subject: [PATCH 2/4] :art: Improve data sync --- kernel/model/repository.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 94332e97e..5a71f7e67 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -966,6 +966,7 @@ func syncRepoDownload() (err error) { return } + logging.LogInfof("downloading data repo [kernel=%s, provider=%d, mode=%s/%t]", KernelID, Conf.Sync.Provider, "d", true) start := time.Now() _, _, err = indexRepoBeforeCloudSync(repo) if nil != err { @@ -1034,6 +1035,7 @@ func syncRepoUpload() (err error) { return } + logging.LogInfof("uploading data repo [kernel=%s, provider=%d, mode=%s/%t]", KernelID, Conf.Sync.Provider, "u", true) start := time.Now() _, _, err = indexRepoBeforeCloudSync(repo) if nil != err { @@ -1193,6 +1195,7 @@ func syncRepo(exit, byHand bool) (dataChanged bool, err error) { return } + logging.LogInfof("syncing data repo [kernel=%s, provider=%d, mode=%s/%t]", KernelID, Conf.Sync.Provider, "a", byHand) start := time.Now() beforeIndex, afterIndex, err := indexRepoBeforeCloudSync(repo) if nil != err { @@ -1248,7 +1251,7 @@ func syncRepo(exit, byHand bool) (dataChanged bool, err error) { } func processSyncMergeResult(exit, byHand bool, mergeResult *dejavu.MergeResult, trafficStat *dejavu.TrafficStat, mode string, elapsed time.Duration) { - logging.LogInfof("synced data repo [kernel=%s, provider=%d, mode=%s/%t, ufc=%d, dfc=%d, ucc=%d, dcc=%d, ub=%s, db=%s] in [%.2fs], merge result [conflicts=%d, upserts=%d, removes=%d]", + logging.LogInfof("synced data repo [kernel=%s, provider=%d, mode=%s/%t, ufc=%d, dfc=%d, ucc=%d, dcc=%d, ub=%s, db=%s] in [%.2fs], merge result [conflicts=%d, upserts=%d, removes=%d]\n\n", KernelID, Conf.Sync.Provider, mode, byHand, trafficStat.UploadFileCount, trafficStat.DownloadFileCount, trafficStat.UploadChunkCount, trafficStat.DownloadChunkCount, humanize.Bytes(uint64(trafficStat.UploadBytes)), humanize.Bytes(uint64(trafficStat.DownloadBytes)), elapsed.Seconds(), From eb6f892a47a7c8e8390c18650b552e28a5df4fab Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 10 Nov 2023 23:00:17 +0800 Subject: [PATCH 3/4] :art: Improve S3/WebDAV data sync error message Fix https://github.com/siyuan-note/siyuan/issues/9626 --- app/appearance/langs/en_US.json | 3 ++- app/appearance/langs/es_ES.json | 3 ++- app/appearance/langs/fr_FR.json | 3 ++- app/appearance/langs/zh_CHT.json | 3 ++- app/appearance/langs/zh_CN.json | 3 ++- kernel/go.mod | 2 +- kernel/go.sum | 2 ++ kernel/model/sync.go | 2 ++ 8 files changed, 15 insertions(+), 6 deletions(-) diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 258cb7267..a8d9b38aa 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -1317,6 +1317,7 @@ "215": "Save failed: The target file is being used by another program", "216": "Rebuilding asset content data index, please wait...", "217": "[%d/%d] Created asset content data index", - "218": "Too many snapshots of the data repo have been detected, which slows down program startup and data sync. Please consider executing [Settings - About - Data repo purge]" + "218": "Too many snapshots of the data repo have been detected, which slows down program startup and data sync. Please consider executing [Settings - About - Data repo purge]", + "219": "The cloud storage service is unavailable, please try again later" } } diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 7839bc5ca..1863f7899 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -1317,6 +1317,7 @@ "215": "Error al guardar: el archivo de destino está siendo utilizado por otro programa", "216": "Reconstruyendo el índice de datos de contenido de recursos, espere...", "217": "[%d/%d] Índice de datos de contenido de activos creado", - "218": "Se han detectado demasiadas instantáneas del repositorio de datos, lo que ralentiza el inicio del programa y la sincronización de datos. Considere ejecutar [Configuración - Acerca de - Purga del repositorio de datos]" + "218": "Se han detectado demasiadas instantáneas del repositorio de datos, lo que ralentiza el inicio del programa y la sincronización de datos. Considere ejecutar [Configuración - Acerca de - Purga del repositorio de datos]", + "219": "El servicio de almacenamiento en la nube no está disponible, inténtalo de nuevo más tarde" } } diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 868526c35..45f68cb3b 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -1317,6 +1317,7 @@ "215": "Échec de l'enregistrement : le fichier de destination est utilisé par un autre programme", "216": "Reconstruction de l'index des données du contenu des ressources, veuillez patienter...", "217": "[%d/%d] Création d'un index de données de contenu d'actif", - "218": "Trop d'instantanés du référentiel de données ont été détectés, ce qui ralentit le démarrage du programme et la synchronisation des données. Veuillez envisager d'exécuter [Paramètres - À propos - Purge du référentiel de données]" + "218": "Trop d'instantanés du référentiel de données ont été détectés, ce qui ralentit le démarrage du programme et la synchronisation des données. Veuillez envisager d'exécuter [Paramètres - À propos - Purge du référentiel de données]", + "219": "Le service de stockage cloud n'est pas disponible, veuillez réessayer plus tard" } } diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index d897a7936..b4a41bcd5 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -1317,6 +1317,7 @@ "215": "保存失敗：目標文件正在被其他程序佔用", "216": "正在重建資源文件內容數據索引，請稍等...", "217": "[%d/%d] 已經創建資源文件內容數據索引", - "218": "偵測到資料倉儲快照過多，降低了程式啟動和資料同步速度，請考慮執行 [設定 - 關於 - 資料倉儲清理]" + "218": "偵測到資料倉儲快照過多，降低了程式啟動和資料同步速度，請考慮執行 [設定 - 關於 - 資料倉儲清理]", + "219": "雲端儲存服務不可用，請稍後再試" } } diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 008e14a12..ff82789da 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -1317,6 +1317,7 @@ "215": "保存失败：目标文件并且正在被其他程序占用", "216": "正在重建资源文件内容数据索引，请稍等...", "217": "[%d/%d] 已经创建资源文件内容数据索引", - "218": "检测到数据仓库快照过多，降低了程序启动和数据同步速度，请考虑执行 [设置 - 关于 - 数据仓库清理]" + "218": "检测到数据仓库快照过多，降低了程序启动和数据同步速度，请考虑执行 [设置 - 关于 - 数据仓库清理]", + "219": "云端存储服务不可用，请稍后再试" } } diff --git a/kernel/go.mod b/kernel/go.mod index 895b8e8ed..e61f9018a 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -49,7 +49,7 @@ require ( github.com/radovskyb/watcher v1.0.7 github.com/sashabaranov/go-openai v1.17.5 github.com/shirou/gopsutil/v3 v3.23.10 - github.com/siyuan-note/dejavu v0.0.0-20231110140645-e32f6a32cffa + github.com/siyuan-note/dejavu v0.0.0-20231110145452-59dbd8bb6c2f github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 github.com/siyuan-note/filelock v0.0.0-20231107122348-6ed75b0b525a diff --git a/kernel/go.sum b/kernel/go.sum index 7c6c926b6..18f922297 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -354,6 +354,8 @@ github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05Ut github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= github.com/siyuan-note/dejavu v0.0.0-20231110140645-e32f6a32cffa h1:m7+UGC/nVyLKGy3NxWFJ1cT1sJDLUJ+D0sO/8cyN+xo= github.com/siyuan-note/dejavu v0.0.0-20231110140645-e32f6a32cffa/go.mod h1:6RdIofCDZP8vL0qYnVZfB12NhK7cUgh1ooUxRvRdlmI= +github.com/siyuan-note/dejavu v0.0.0-20231110145452-59dbd8bb6c2f h1:sq/GK4HappFB+VID9YphmZ1oVtj+Cc/4l6ZPSUy8r/g= +github.com/siyuan-note/dejavu v0.0.0-20231110145452-59dbd8bb6c2f/go.mod h1:6RdIofCDZP8vL0qYnVZfB12NhK7cUgh1ooUxRvRdlmI= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE= github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 h1:XghjHKJd+SiL0DkGYFVC+UGUDFtnR4v9gkAbPeh9Eq8= diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 51c73e91a..bf8f7ce2d 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -534,6 +534,8 @@ func formatRepoErrorMsg(err error) string { msg = Conf.Language(212) } else if errors.Is(err, cloud.ErrCloudCheckFailed) { msg = Conf.Language(213) + } else if errors.Is(err, cloud.ErrCloudServiceUnavailable) { + msg = Conf.language(219) } else { msgLowerCase := strings.ToLower(msg) if strings.Contains(msgLowerCase, "permission denied") || strings.Contains(msg, "access is denied") { From 7be06ed9556165cc5f23477123655a48b0f2a868 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 10 Nov 2023 23:13:57 +0800 Subject: [PATCH 4/4] :art: Improve database table view in super block Fix https://github.com/siyuan-note/siyuan/issues/9620 --- kernel/model/transaction.go | 43 ++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 460c23efc..cc7fada57 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -714,13 +714,24 @@ func (tx *Transaction) doDelete(operation *Operation) (ret *TxErr) { } syncDelete2AttributeView(node) - if ast.NodeAttributeView == node.Type { - avID := node.AttributeViewID - av.RemoveBlockRel(avID, node.ID) - } + removeAvBlockRel(node) return } +func removeAvBlockRel(node *ast.Node) { + ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering { + return ast.WalkContinue + } + + if ast.NodeAttributeView == n.Type { + avID := n.AttributeViewID + av.RemoveBlockRel(avID, n.ID) + } + return ast.WalkContinue + }) +} + func syncDelete2AttributeView(node *ast.Node) { avs := node.IALAttr(av.NodeAttrNameAvs) if "" == avs { @@ -904,10 +915,7 @@ func (tx *Transaction) doInsert(operation *Operation) (ret *TxErr) { return &TxErr{code: TxErrCodeWriteTree, msg: err.Error(), id: block.ID} } - if ast.NodeAttributeView == insertedNode.Type { - avID := insertedNode.AttributeViewID - av.UpsertBlockRel(avID, insertedNode.ID) - } + upsertAvBlockRel(insertedNode) operation.ID = insertedNode.ID operation.ParentID = insertedNode.Parent.ID @@ -998,13 +1006,24 @@ func (tx *Transaction) doUpdate(operation *Operation) (ret *TxErr) { return &TxErr{code: TxErrCodeWriteTree, msg: err.Error(), id: id} } - if ast.NodeAttributeView == updatedNode.Type { - avID := updatedNode.AttributeViewID - av.UpsertBlockRel(avID, updatedNode.ID) - } + upsertAvBlockRel(updatedNode) return } +func upsertAvBlockRel(node *ast.Node) { + ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering { + return ast.WalkContinue + } + + if ast.NodeAttributeView == n.Type { + avID := n.AttributeViewID + av.UpsertBlockRel(avID, n.ID) + } + return ast.WalkContinue + }) +} + func (tx *Transaction) doUpdateUpdated(operation *Operation) (ret *TxErr) { id := operation.ID tree, err := tx.loadTree(id)