From bc046e7339ccc4ba21946b390cb9db752375b107 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 13 Jul 2022 12:21:02 +0800 Subject: [PATCH 1/8] :art: Search ignore case supports Unicode character folding Fix https://github.com/siyuan-note/siyuan/issues/5398 --- kernel/go.mod | 2 +- kernel/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index b95aa4032..e8eecff4d 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -111,7 +111,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect ) -replace github.com/mattn/go-sqlite3 => github.com/88250/go-sqlite3 v1.14.13-0.20220713031108-18de142395e6 +replace github.com/mattn/go-sqlite3 => github.com/88250/go-sqlite3 v1.14.13-0.20220713041934-e2e29c01be36 //replace github.com/siyuan-note/dejavu => D:\88250\dejavu //replace github.com/siyuan-note/httpclient => D:\88250\httpclient diff --git a/kernel/go.sum b/kernel/go.sum index f7a0c42f2..18ca966d3 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -45,8 +45,8 @@ github.com/88250/css v0.1.2 h1:+AADhEwWoGZFbUjqIsBcdnq2xfj8fDFDAGRXhBUhUY8= github.com/88250/css v0.1.2/go.mod h1:XfcZHQ0YuUb9VncVBurQfVyw1ZQicsB5Gc9N7BK3/ig= github.com/88250/flock v0.8.2 h1:LLbRJw3hoYfjD4g7DiYsYcTCCFTxm8icn/WepLlxIg0= github.com/88250/flock v0.8.2/go.mod h1:k+PZxETAUe4vLZx3R39ykvQCIlwHhc7AI2P2NUQV6zw= -github.com/88250/go-sqlite3 v1.14.13-0.20220713031108-18de142395e6 h1:foyDAhSJzo12Ox/uu2LqL5pzdIO0lcME3ICBT5eiV3Y= -github.com/88250/go-sqlite3 v1.14.13-0.20220713031108-18de142395e6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/88250/go-sqlite3 v1.14.13-0.20220713041934-e2e29c01be36 h1:bFyBW46RX8+vXvMh8EzySRRjysSYuAiJ1A4kKCobA9I= +github.com/88250/go-sqlite3 v1.14.13-0.20220713041934-e2e29c01be36/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/88250/gulu v1.2.0/go.mod h1:ZhEJ98UjR2y7j2toGj4/b+1rRELcZFQAPq/Yjyin2yY= github.com/88250/gulu v1.2.3-0.20220623112232-c502d9016360 h1:afQ0cjIA/tzwvIDFy9Jf0jFCb1FvWwKuG1QidEMMi4M= github.com/88250/gulu v1.2.3-0.20220623112232-c502d9016360/go.mod h1:I1qBzsksFL2ciGSuqDE7R3XW4BUMrfDgOvSXEk7FsAI= From 8581f83e615bbe85c231e225f992cea19a2091d7 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 13 Jul 2022 17:14:54 +0800 Subject: [PATCH 2/8] =?UTF-8?q?:fire:=20=E7=A7=BB=E9=99=A4=E6=97=A7?= =?UTF-8?q?=E7=89=88=E4=BA=91=E7=AB=AF=E5=90=8C=E6=AD=A5=E5=92=8C=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8A=9F=E8=83=BD=E5=85=A5=E5=8F=A3=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/5405?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 13 ++-- app/appearance/langs/es_ES.json | 13 ++-- app/appearance/langs/fr_FR.json | 13 ++-- app/appearance/langs/zh_CHT.json | 13 ++-- app/appearance/langs/zh_CN.json | 13 ++-- kernel/api/router.go | 2 - kernel/api/sync.go | 19 ----- kernel/api/system.go | 71 ----------------- kernel/conf/sync.go | 22 +++--- kernel/model/conf.go | 26 ------- kernel/model/osssync.go | 31 -------- kernel/model/sync.go | 130 +++---------------------------- 12 files changed, 54 insertions(+), 312 deletions(-) diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 675455aa0..c83b71dc5 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -764,6 +764,7 @@ "8": "Check update failed", "9": "A new version is available, please browse the release announcement %s", "10": "Is the latest version", + "11": "TODO", "12": "Query asset failed [%s]", "13": "Cannot create a file starting with .", "14": "Export failed: %s", @@ -791,7 +792,7 @@ "36": "Please check the version update in the app store", "37": "Do not include spaces and special symbols in the name of the cloud sync directory", "38": "The number of mentioned keywords [%d] is too many, currently only supports up to [512] keywords", - "39": "E2EE password can not be blank", + "39": "TODO", "40": "Failed to decrypt data", "41": "Upload completed", "42": "The setting is complete, the application will be closed automatically, please restart later...", @@ -841,10 +842,10 @@ "86": "Please configure [Settings - About - Access authorization code]", "87": "Cannot move to this location", "88": "Finished parsing [%d] data files, remaining to be processed [%d]", - "89": "Local data will overwrite the data in the cloud sync directory %s", - "90": "Cloud synchronization directory %s data will overwrite local data", - "91": "The local data and the cloud synchronization directory %s have the same data", - "92": "The end-to-end encryption password is set", + "89": "TODO", + "90": "TODO", + "91": "TODO", + "92": "TODO", "93": "Download failed: %s", "94": "Upload failed: %s", "95": "Exiting...", @@ -854,7 +855,7 @@ "99": "Data cleaning is complete", "100": "Cleaning data...", "101": "Done setting reminder [%s]", - "102": "Setting end-to-end encryption password...", + "102": "TODO", "103": "[%d] data files have been downloaded, and [%d] remaining to be downloaded", "104": "[%d] data files have been uploaded, and [%d] remaining to be uploaded", "105": "Network transmission completed", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 752347d45..73248c04d 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -764,6 +764,7 @@ "8": "Comprobar la actualización falló", "9": "Una nueva versión está disponible, por favor, busque el anuncio de lanzamiento %s", "10": "Es la última versión", + "11": "TODO", "12": "Fallo en la consulta de activos [%s]", "13": "No se puede crear un archivo que empiece por .", "14": "Exportación fallida: %s", @@ -791,7 +792,7 @@ "36": "Por favor, comprueba la actualización de la versión en la tienda de aplicaciones", "37": "No incluyas espacios ni símbolos especiales en el nombre del directorio de sincronización con la nube", "38": "El número de palabras clave mencionadas [%d] son demasiados, actualmente solo admite hasta [512] palabras clave", - "39": "La contraseña de E2EE no puede estar en blanco", + "39": "TODO", "40": "Fallo en la desencriptación de datos", "41": "Carga completada", "42": "La configuración se ha completado, la aplicación se cerrará automáticamente, por favor reinicie más tarde...", @@ -841,10 +842,10 @@ "86": "Por favor, configure [Configuración - Acerca de - Código de autorización de acceso]", "87": "No se puede mover a esta ubicación", "88": "Se ha terminado de analizar [%d] archivos de datos, quedan por procesar [%d]", - "89": "Los datos locales sobrescribirán los datos del directorio de sincronización en la nube %s", - "90": "Los datos del directorio de sincronización en la nube %s sobrescribirán los datos locales", - "91": "Los datos locales y el directorio de sincronización en la nube %s tienen los mismos datos", - "92": "La contraseña de encriptación de extremo a extremo está establecida", + "89": "TODO", + "90": "TODO", + "91": "TODO", + "92": "TODO", "93": "Descarga fallida: %s", "94": "Carga fallida: %s", "95": "Saliendo...", @@ -854,7 +855,7 @@ "99": "La limpieza de datos ha finalizado", "100": "Limpieza de datos...", "101": "El recordatorio de configuración [%s] se ha completado", - "102": "Configurando la contraseña de cifrado de extremo a extremo...", + "102": "TODO", "103": "[%d] archivos de datos han sido descargados, y [%d] quedan por descargar", "104": "[%d] archivos de datos han sido cargados, y [%d] restantes por cargar", "105": "Transmisión de red completada", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 90ffb30a3..15d140172 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -764,6 +764,7 @@ "8": "La vérification de la mise à jour a échoué", "9": "Une nouvelle version est disponible, veuillez consulter l'annonce de la version %s", "10": "C'est la dernière version", + "11": "TODO", "12": "Échec de la requête asset [%s]", "13": "Impossible de créer un fichier commençant par .", "14": "L'exportation a échoué : %s", @@ -791,7 +792,7 @@ "36": "Veuillez vérifier la mise à jour de la version dans l'App Store", "37": "N'incluez pas d'espaces et de symboles spéciaux dans le nom du répertoire de synchronisation cloud", "38": "Le nombre de mots-clés mentionnés [%d] est trop élevé, ne prend actuellement en charge que jusqu'à [512] mots-clés", - "39": "Le mot de passe E2EE ne peut pas être vide", + "39": "TODO", "40": "Échec du décryptage des données", "41": "Transfert complété", "42": "Le paramétrage est terminé, l'application se fermera automatiquement, merci de redémarrer plus tard...", @@ -841,10 +842,10 @@ "86": "Veuillez configurer [Paramètres - A propos de - Code d'autorisation d'accès]", "87": "Impossible de se déplacer vers cet endroit", "88": "Fin de l'analyse des fichiers de données [%d], restant à traiter [%d]", - "89": "Les données locales écraseront les données du répertoire de synchronisation cloud %s", - "90": "Les données du répertoire de synchronisation cloud %s écraseront les données locales", - "91": "Les données locales et le répertoire de synchronisation cloud %s ont les mêmes données", - "92": "Le mot de passe de cryptage de bout en bout est défini", + "89": "TODO", + "90": "TODO", + "91": "TODO", + "92": "TODO", "93": "Le téléchargement a échoué : %s", "94": "Échec du téléchargement : %s", "95": "Quitter le programme...", @@ -854,7 +855,7 @@ "99": "Le nettoyage des données est terminé", "100": "Nettoyage des données...", "101": "Rappel de réglage terminé [%s]", - "102": "Définition du mot de passe de cryptage de bout en bout...", + "102": "TODO", "103": "[%d] fichiers de données ont été téléchargés, il reste à télécharger [%d]", "104": "[%d] fichiers de données ont été téléchargés, et [%d] reste à télécharger", "105": "Transmission réseau terminée", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index b0d7bcfd2..1c30b195f 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -764,6 +764,7 @@ "8": "檢查更新失敗", "9": "有新版本可用，請瀏覽發佈公告 %s", "10": "已是最新版", + "11": "TODO", "12": "查詢資料檔失敗 [%s]", "13": "無法創建 . 開頭的文件", "14": "匯出失敗：%s", @@ -791,7 +792,7 @@ "36": "請在應用商店中檢查版本更新", "37": "雲端同步目錄的名稱請勿包含空格和特殊符號", "38": "提及關鍵字數量 [%d] 過多，目前最多僅支援搜索 [512] 個關鍵字", - "39": "端到端加密密碼不能為空", + "39": "TODO", "40": "解密數據失敗", "41": "上傳完畢", "42": "設置完成，即將自動關閉應用，請稍後重新啟動...", @@ -841,10 +842,10 @@ "86": "請先配置 [設置 - 關於 - 存取授權碼]", "87": "無法移動到該位置", "88": "已完成解析 [%d] 個數據文件，剩餘待處理 [%d]", - "89": "本地資料將覆蓋雲端同步目錄 %s 資料", - "90": "雲端同步目錄 %s 資料將覆蓋本地資料", - "91": "本地資料和雲端同步目錄 %s 資料一致", - "92": "端到端加密密碼設置完畢", + "89": "TODO", + "90": "TODO", + "91": "TODO", + "92": "TODO", "93": "下載失敗：%s", "94": "上傳失敗：%s", "95": "正在退出...", @@ -854,7 +855,7 @@ "99": "清理數據完成", "100": "正在清理數據...", "101": "設置提醒完畢 [%s]", - "102": "正在設置端到端加密密碼...", + "102": "TODO", "103": "已下載 [%d] 個資料檔案，剩餘待下載 [%d]", "104": "已上傳 [%d] 個資料檔案，剩餘待上傳 [%d]", "105": "網絡傳輸完畢", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index e4c63b943..964f9ee17 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -765,6 +765,7 @@ "8": "检查更新失败", "9": "有新版本可用，请浏览发布公告 %s", "10": "已是最新版", + "11": "TODO", "12": "查询资源文件失败 [%s]", "13": "无法创建 . 开头的文件", "14": "导出失败：%s", @@ -792,7 +793,7 @@ "36": "请在应用商店中检查版本更新", "37": "云端同步目录的名称请勿包含空格和特殊符号", "38": "提及关键字数量 [%d] 过多，目前最多仅支持搜索 [512] 个关键字", - "39": "端到端加密密码不能为空", + "39": "TODO", "40": "解密数据失败", "41": "上传完毕", "42": "设置完成，即将自动关闭应用，请稍后重新启动...", @@ -842,10 +843,10 @@ "86": "请先配置 [设置 - 关于 - 访问授权码]", "87": "无法移动到该位置", "88": "已完成解析 [%d] 个数据文件，剩余待处理 [%d]", - "89": "本地数据将覆盖云端同步目录 %s 数据", - "90": "云端同步目录 %s 数据将覆盖本地数据", - "91": "本地数据和云端同步目录 %s 数据一致", - "92": "端到端加密密码设置完毕", + "89": "TODO", + "90": "TODO", + "91": "TODO", + "92": "TODO", "93": "下载失败：%s", "94": "上传失败：%s", "95": "正在退出...", @@ -855,7 +856,7 @@ "99": "清理数据完成", "100": "正在清理数据...", "101": "设置提醒完毕 [%s]", - "102": "正在设置端到端加密密码...", + "102": "TODO", "103": "已下载 [%d] 个数据文件，剩余待下载 [%d]", "104": "已上传 [%d] 个数据文件，剩余待上传 [%d]", "105": "网络传输完毕", diff --git a/kernel/api/router.go b/kernel/api/router.go index 7e9a697f1..78b533bd6 100644 --- a/kernel/api/router.go +++ b/kernel/api/router.go @@ -44,7 +44,6 @@ func ServeAPI(ginServer *gin.Engine) { ginServer.Handle("POST", "/api/system/listWorkspaceDirs", model.CheckAuth, listWorkspaceDirs) ginServer.Handle("POST", "/api/system/setAppearanceMode", model.CheckAuth, setAppearanceMode) ginServer.Handle("POST", "/api/system/getSysFonts", model.CheckAuth, getSysFonts) - ginServer.Handle("POST", "/api/system/setE2EEPasswd", model.CheckAuth, setE2EEPasswd) ginServer.Handle("POST", "/api/system/exit", model.CheckAuth, exit) ginServer.Handle("POST", "/api/system/setUILayout", model.CheckAuth, setUILayout) ginServer.Handle("POST", "/api/system/getConf", model.CheckAuth, getConf) @@ -167,7 +166,6 @@ func ServeAPI(ginServer *gin.Engine) { ginServer.Handle("POST", "/api/backup/removeCloudBackup", model.CheckAuth, model.CheckReadonly, removeCloudBackup) ginServer.Handle("POST", "/api/sync/setSyncEnable", model.CheckAuth, setSyncEnable) - ginServer.Handle("POST", "/api/sync/setSyncUseDataRepo", model.CheckAuth, setSyncUseDataRepo) ginServer.Handle("POST", "/api/sync/setSyncMode", model.CheckAuth, setSyncMode) ginServer.Handle("POST", "/api/sync/setCloudSyncDir", model.CheckAuth, setCloudSyncDir) ginServer.Handle("POST", "/api/sync/createCloudSyncDir", model.CheckAuth, model.CheckReadonly, createCloudSyncDir) diff --git a/kernel/api/sync.go b/kernel/api/sync.go index ecdbb542b..7dbf7d070 100644 --- a/kernel/api/sync.go +++ b/kernel/api/sync.go @@ -121,25 +121,6 @@ func createCloudSyncDir(c *gin.Context) { } } -func setSyncUseDataRepo(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - arg, ok := util.JsonArg(c, ret) - if !ok { - return - } - - enabled := arg["enabled"].(bool) - err := model.SetSyncUseDataRepo(enabled) - if nil != err { - ret.Code = 1 - ret.Msg = err.Error() - ret.Data = map[string]interface{}{"closeTimeout": 5000} - return - } -} - func setSyncEnable(c *gin.Context) { ret := gulu.Ret.NewResult() defer c.JSON(http.StatusOK, ret) diff --git a/kernel/api/system.go b/kernel/api/system.go index 18602f1d8..2e5b3a0d8 100644 --- a/kernel/api/system.go +++ b/kernel/api/system.go @@ -295,77 +295,6 @@ func setNetworkProxy(c *gin.Context) { time.Sleep(time.Second * 3) } -func setE2EEPasswd(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - arg, ok := util.JsonArg(c, ret) - if !ok { - return - } - - var passwd string - mode := int(arg["mode"].(float64)) - if 0 == mode { // 使用内建的密码生成 - passwd = model.GetBuiltInE2EEPasswd() - } else { // 使用自定义密码 - passwd = arg["e2eePasswd"].(string) - passwd = strings.TrimSpace(passwd) - } - - if "" == passwd { - ret.Code = -1 - ret.Msg = model.Conf.Language(39) - ret.Data = map[string]interface{}{"closeTimeout": 5000} - return - } - - newPasswd := util.AESEncrypt(passwd) - if model.Conf.E2EEPasswd == newPasswd { - util.PushMsg(model.Conf.Language(92), 3000) - return - } - - msgId := util.PushMsg(model.Conf.Language(102), 1000*7) - if err := os.RemoveAll(model.Conf.Backup.GetSaveDir()); nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } - if err := os.MkdirAll(model.Conf.Backup.GetSaveDir(), 0755); nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } - if err := os.RemoveAll(model.Conf.Sync.GetSaveDir()); nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } - if err := os.MkdirAll(model.Conf.Sync.GetSaveDir(), 0755); nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } - if err := os.RemoveAll(filepath.Join(util.TempDir, "incremental")); nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } - if err := os.MkdirAll(filepath.Join(util.TempDir, "incremental"), 0755); nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } - time.Sleep(1 * time.Second) - model.Conf.E2EEPasswd = newPasswd - model.Conf.E2EEPasswdMode = mode - model.Conf.Save() - util.PushUpdateMsg(msgId, model.Conf.Language(92), 3000) - time.Sleep(1 * time.Second) - model.SyncData(false, false, true) -} - func addUIProcess(c *gin.Context) { pid := c.Query("pid") util.UIProcessIDs.Store(pid, true) diff --git a/kernel/conf/sync.go b/kernel/conf/sync.go index a892c4ef4..47e4bb5fc 100644 --- a/kernel/conf/sync.go +++ b/kernel/conf/sync.go @@ -23,22 +23,20 @@ import ( ) type Sync struct { - CloudName string `json:"cloudName"` // 云端同步目录名称 - Enabled bool `json:"enabled"` // 是否开启同步 - Mode int `json:"mode"` // 同步模式，0：未设置（为兼容已有配置，initConf 函数中会转换为 1），1：自动，2：手动 https://github.com/siyuan-note/siyuan/issues/5089 - Uploaded int64 `json:"uploaded"` // 最近上传时间 - Downloaded int64 `json:"downloaded"` // 最近下载时间 - Synced int64 `json:"synced"` // 最近同步时间 - Stat string `json:"stat"` // 最近同步统计信息 - UseDataRepo bool `json:"useDataRepo"` // 是否使用数据仓库同步 + CloudName string `json:"cloudName"` // 云端同步目录名称 + Enabled bool `json:"enabled"` // 是否开启同步 + Mode int `json:"mode"` // 同步模式，0：未设置（为兼容已有配置，initConf 函数中会转换为 1），1：自动，2：手动 https://github.com/siyuan-note/siyuan/issues/5089 + Uploaded int64 `json:"uploaded"` // 最近上传时间 + Downloaded int64 `json:"downloaded"` // 最近下载时间 + Synced int64 `json:"synced"` // 最近同步时间 + Stat string `json:"stat"` // 最近同步统计信息 } func NewSync() *Sync { return &Sync{ - CloudName: "main", - Enabled: false, - Mode: 1, - UseDataRepo: true, + CloudName: "main", + Enabled: false, + Mode: 1, } } diff --git a/kernel/model/conf.go b/kernel/model/conf.go index 06f9df04f..c32ae7da1 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -18,9 +18,7 @@ package model import ( "bytes" - "crypto/sha256" "errors" - "fmt" "os" "path/filepath" "runtime" @@ -63,8 +61,6 @@ type AppConf struct { ReadOnly bool `json:"readonly"` // 是否是只读 LocalIPs []string `json:"localIPs"` // 本地 IP 列表 AccessAuthCode string `json:"accessAuthCode"` // 访问授权码 - E2EEPasswd string `json:"e2eePasswd"` // 端到端加密密码，用于备份和同步 - E2EEPasswdMode int `json:"e2eePasswdMode"` // 端到端加密密码生成方式，0：自动，1：自定义 System *conf.System `json:"system"` // 系统 Keymap *conf.Keymap `json:"keymap"` // 快捷键 Backup *conf.Backup `json:"backup"` // 备份配置 @@ -283,11 +279,6 @@ func InitConf() { Conf.AccessAuthCode = util.AccessAuthCode } - Conf.E2EEPasswdMode = 0 - if !isBuiltInE2EEPasswd() { - Conf.E2EEPasswdMode = 1 - } - Conf.LocalIPs = util.GetLocalIPs() Conf.Save() @@ -555,23 +546,6 @@ func IsSubscriber() bool { return nil != Conf.User && (-1 == Conf.User.UserSiYuanProExpireTime || 0 < Conf.User.UserSiYuanProExpireTime) && 0 == Conf.User.UserSiYuanSubscriptionStatus } -func isBuiltInE2EEPasswd() bool { - if nil == Conf || nil == Conf.User || "" == Conf.E2EEPasswd { - return true - } - - pwd := GetBuiltInE2EEPasswd() - return Conf.E2EEPasswd == util.AESEncrypt(pwd) -} - -func GetBuiltInE2EEPasswd() (ret string) { - part1 := Conf.User.UserId[:7] - part2 := Conf.User.UserId[7:] - ret = part2 + part1 - ret = fmt.Sprintf("%x", sha256.Sum256([]byte(ret)))[:7] - return -} - func clearWorkspaceTemp() { os.RemoveAll(filepath.Join(util.TempDir, "bazaar")) os.RemoveAll(filepath.Join(util.TempDir, "export")) diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index fef54db31..afac2ecab 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -105,37 +105,6 @@ func removeCloudDirPath(dirPath string) (err error) { return } -func createCloudSyncDirOSS(name string) (err error) { - result := map[string]interface{}{} - request := httpclient.NewCloudRequest() - resp, err := request. - SetResult(&result). - SetBody(map[string]string{"name": name, "token": Conf.User.UserToken}). - Post(util.AliyunServer + "/apis/siyuan/data/createSiYuanSyncDir") - if nil != err { - util.LogErrorf("create cloud sync dir failed: %s", err) - return ErrFailedToConnectCloudServer - } - - if 200 != resp.StatusCode { - if 401 == resp.StatusCode { - err = errors.New(Conf.Language(31)) - return - } - msg := fmt.Sprintf("create cloud sync dir failed: %d", resp.StatusCode) - util.LogErrorf(msg) - err = errors.New(msg) - return - } - - code := result["code"].(float64) - if 0 != code { - util.LogErrorf("create cloud sync dir failed: %s", result["msg"]) - return errors.New(result["msg"].(string)) - } - return -} - func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error) { result := map[string]interface{}{} request := httpclient.NewCloudRequest() diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 314073b55..9f877db84 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -84,12 +84,10 @@ func SyncData(boot, exit, byHand bool) { if exit { ExitSyncSucc = 0 } - if !IsSubscriber() || !Conf.Sync.Enabled || "" == Conf.Sync.CloudName || ("" == Conf.E2EEPasswd && !Conf.Sync.UseDataRepo) { + if !IsSubscriber() || !Conf.Sync.Enabled || "" == Conf.Sync.CloudName { if byHand { if "" == Conf.Sync.CloudName { util.PushMsg(Conf.Language(123), 5000) - } else if "" == Conf.E2EEPasswd { - util.PushMsg(Conf.Language(11), 5000) } else if !Conf.Sync.Enabled { util.PushMsg(Conf.Language(124), 5000) } @@ -513,15 +511,6 @@ func SetSyncEnable(b bool) (err error) { return } -func SetSyncUseDataRepo(b bool) (err error) { - syncLock.Lock() - defer syncLock.Unlock() - - Conf.Sync.UseDataRepo = b - Conf.Save() - return -} - func SetSyncMode(mode int) (err error) { syncLock.Lock() defer syncLock.Unlock() @@ -1009,83 +998,6 @@ func calcUnchangedSyncList() (ret map[string]bool, removes []string, err error) return } -// calcUnchangedDataList 计算 sync 和 data 一致（没有修改过）的文件列表 unchangedDataList，并删除 sync 中不存在于 data 中的多余文件 removeList。 -func calcUnchangedDataList(passwd string) (unchangedDataList map[string]bool, removeList map[string]bool, err error) { - syncDir := Conf.Sync.GetSaveDir() - meta := filepath.Join(syncDir, pathJSON) - if !gulu.File.IsExist(meta) { - return - } - data, err := os.ReadFile(meta) - if nil != err { - return - } - data, err = encryption.AESGCMDecryptBinBytes(data, passwd) - if nil != err { - err = errors.New(Conf.Language(40)) - return - } - - metaJSON := map[string]string{} - if err = gulu.JSON.UnmarshalJSON(data, &metaJSON); nil != err { - return - } - - unchangedDataList = map[string]bool{} - removeList = map[string]bool{} - filepath.Walk(syncDir, func(path string, info fs.FileInfo, _ error) error { - if syncDir == path || pathJSON == info.Name() || "index.json" == info.Name() || info.IsDir() { - return nil - } - - encryptedP := strings.TrimPrefix(path, syncDir+string(os.PathSeparator)) - encryptedP = filepath.ToSlash(encryptedP) - decryptedP := metaJSON[encryptedP] - if "" == decryptedP { - removeList[path] = true - if gulu.File.IsDir(path) { - return filepath.SkipDir - } - return nil - } - dataP := filepath.Join(util.DataDir, decryptedP) - dataP = filepath.FromSlash(dataP) - if !gulu.File.IsExist(dataP) { // data 已经删除的文件 - removeList[path] = true - if gulu.File.IsDir(path) { - return filepath.SkipDir - } - return nil - } - - stat, _ := os.Stat(dataP) - dataModTime := stat.ModTime() - if info.ModTime() == dataModTime { - unchangedDataList[dataP] = true - return nil - } - return nil - }) - - tmp := map[string]bool{} - // 在 sync 中删除 data 中已经删除的文件 - for remove, _ := range removeList { - if strings.HasSuffix(remove, "index.json") { - continue - } - - p := strings.TrimPrefix(remove, syncDir) - p = filepath.ToSlash(p) - tmp[p] = true - - if err = os.RemoveAll(remove); nil != err { - util.LogErrorf("remove [%s] failed: %s", remove, err) - } - } - removeList = tmp - return -} - func getWorkspaceDataConf() (conf *filesys.DataConf, err error) { conf = &filesys.DataConf{Updated: util.CurrentTimeMillis(), Device: Conf.System.ID} confPath := filepath.Join(Conf.Sync.GetSaveDir(), ".siyuan", "conf.json") @@ -1157,17 +1069,13 @@ func CreateCloudSyncDir(name string) (err error) { return errors.New(Conf.Language(37)) } - if Conf.Sync.UseDataRepo { - var cloudInfo *dejavu.CloudInfo - cloudInfo, err = buildCloudInfo() - if nil != err { - return - } - - err = dejavu.CreateCloudRepo(name, cloudInfo) - } else { - err = createCloudSyncDirOSS(name) + var cloudInfo *dejavu.CloudInfo + cloudInfo, err = buildCloudInfo() + if nil != err { + return } + + err = dejavu.CreateCloudRepo(name, cloudInfo) return } @@ -1376,28 +1284,8 @@ func GetSyncDirection(cloudDirName string) (code int, msg string) { // 0：失 return } - if Conf.Sync.UseDataRepo { - return 40, "" - } - - syncConf, err := getWorkspaceDataConf() - if nil != err { - msg = fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) - return - } - - cloudSyncVer, err := getCloudSyncVer(cloudDirName) - if nil != err { - msg = fmt.Sprintf(Conf.Language(24), err.Error()) - return - } - if cloudSyncVer < syncConf.SyncVer { - return 10, fmt.Sprintf(Conf.Language(89), cloudDirName) // 上传 - } - if cloudSyncVer > syncConf.SyncVer { - return 20, fmt.Sprintf(Conf.Language(90), cloudDirName) // 下载 - } - return 30, fmt.Sprintf(Conf.Language(91), cloudDirName) // 一致 + // TODO: 彻底移除方向判断 + return 40, "" } func IncWorkspaceDataVer() { From d96767b282880411f3990377fc3a83046722d46a Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 13 Jul 2022 17:44:28 +0800 Subject: [PATCH 3/8] =?UTF-8?q?:fire:=20=E7=A7=BB=E9=99=A4=E6=97=A7?= =?UTF-8?q?=E7=89=88=E4=BA=91=E7=AB=AF=E5=90=8C=E6=AD=A5=E5=92=8C=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8A=9F=E8=83=BD=E5=85=A5=E5=8F=A3=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/5405?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 24 +- app/appearance/langs/es_ES.json | 24 +- app/appearance/langs/fr_FR.json | 24 +- app/appearance/langs/zh_CHT.json | 24 +- app/appearance/langs/zh_CN.json | 24 +- kernel/go.mod | 2 +- kernel/go.sum | 4 +- kernel/model/osssync.go | 456 +----------------- kernel/model/sync.go | 785 +------------------------------ kernel/util/working.go | 9 - 10 files changed, 78 insertions(+), 1298 deletions(-) diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index c83b71dc5..2f3613494 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -777,11 +777,11 @@ "21": "Backup completed", "22": "Backuping, please wait...", "23": "Backup failed: %s", - "24": "Failed to obtain cloud sync info: %s", + "24": "TODO", "25": "The attribute name only supports English letters and digits", "26": "Please initialize the data repo key first in [Settings - About - Data repo key]", "27": "Data integrity check failed", - "28": "Incorrect end-to-end encryption password, unable to decrypt data", + "28": "TODO", "29": "This feature requires paid subscription (If you have subscribed, please refresh or log in again in settings - account)", "30": "Failed to obtain cloud backup info", "31": "Account authentication failed, please login again", @@ -793,7 +793,7 @@ "37": "Do not include spaces and special symbols in the name of the cloud sync directory", "38": "The number of mentioned keywords [%d] is too many, currently only supports up to [512] keywords", "39": "TODO", - "40": "Failed to decrypt data", + "40": "TODO", "41": "Upload completed", "42": "The setting is complete, the application will be closed automatically, please restart later...", "43": "The maximum storage capacity of cloud space [%s] has been exceeded, and data upload cannot continue", @@ -846,7 +846,7 @@ "90": "TODO", "91": "TODO", "92": "TODO", - "93": "Download failed: %s", + "93": "TODO", "94": "Upload failed: %s", "95": "Exiting...", "96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data", @@ -856,10 +856,10 @@ "100": "Cleaning data...", "101": "Done setting reminder [%s]", "102": "TODO", - "103": "[%d] data files have been downloaded, and [%d] remaining to be downloaded", + "103": "TODO", "104": "[%d] data files have been uploaded, and [%d] remaining to be uploaded", "105": "Network transmission completed", - "106": "Data download has been completed and decryption is in progress...", + "106": "TODO", "107": "Moving document [%s]", "108": "Cleaning obsolete indexes...", "109": "Remove reminder completed [%s]", @@ -882,13 +882,13 @@ "126": "Bookmark cannot be empty", "127": "There are [%d] days left before the subscription expires, after which the cloud data will be completely deleted. Please visit Here, if you don't need to renew, please log out of your account to close the reminder", "128": "Subscription has expired, cloud data will be completely deleted after expiration. To renew, please visit here , if you don't need to renew, please log out of your account to close the reminder", - "129": "Number of files transferred %d\nTotal bytes received %s\n", - "130": "Number of files transferred %d\nTotal bytes sent %s\n", - "131": "Downloaded in %.2fs", - "132": "Uploaded in %.2fs", - "133": "No changes to local data", + "129": "TODO", + "130": "TODO", + "131": "TODO", + "132": "TODO", + "133": "TODO", "134": "In order to prevent the newly restored data from being overwritten by synchronization, the data synchronization function has been automatically suspended", - "135": "Please make sure that all devices have been updated to the latest version, and then trigger synchronization after randomly changing a document on the main device, and finally trigger synchronization on other devices", + "135": "TODO", "136": "Initializing data repository key...", "137": "Failed to initialize data repository key: %s", "138": "Data repository key is set", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 73248c04d..68dfdccf0 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -777,11 +777,11 @@ "21": "Copia de seguridad completada", "22": "Haciendo copia de seguridad, por favor espere...", "23": "Copia de seguridad fallida: %s", - "24": "Fallo en la obtención de la información de sincronización con la nube: %s", + "24": "TODO", "25": "El nombre del atributo sólo admite letras y dígitos en inglés", "26": "Por favor, inicialice primero la clave de repositorio de datos en [Configuración - Acerca de - Clave de repositorio de datos]", "27": "Falló la comprobación de la integridad de los datos", - "28": "Contraseña de cifrado de extremo a extremo incorrecta, no se pueden descifrar los datos", + "28": "TODO", "29": "Esta función requiere una suscripción de pago (Si se ha suscrito, actualice o vuelva a conectarse en configuración - cuenta)", "30": "Fallo en la obtención de la información de la copia de seguridad en la nube", "31": "Falló la autentificación de la cuenta, por favor, inicie sesión de nuevo", @@ -793,7 +793,7 @@ "37": "No incluyas espacios ni símbolos especiales en el nombre del directorio de sincronización con la nube", "38": "El número de palabras clave mencionadas [%d] son demasiados, actualmente solo admite hasta [512] palabras clave", "39": "TODO", - "40": "Fallo en la desencriptación de datos", + "40": "TODO", "41": "Carga completada", "42": "La configuración se ha completado, la aplicación se cerrará automáticamente, por favor reinicie más tarde...", "43": "Se ha superado la capacidad máxima de almacenamiento del espacio en la nube [%s] y la carga de datos no puede continuar", @@ -846,7 +846,7 @@ "90": "TODO", "91": "TODO", "92": "TODO", - "93": "Descarga fallida: %s", + "93": "TODO", "94": "Carga fallida: %s", "95": "Saliendo...", "96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube", @@ -856,10 +856,10 @@ "100": "Limpieza de datos...", "101": "El recordatorio de configuración [%s] se ha completado", "102": "TODO", - "103": "[%d] archivos de datos han sido descargados, y [%d] quedan por descargar", + "103": "TODO", "104": "[%d] archivos de datos han sido cargados, y [%d] restantes por cargar", "105": "Transmisión de red completada", - "106": "La descarga de datos se ha completado y la desencriptación está en curso...", + "106": "TODO", "107": "Moviendo documento [%s]", "108": "Limpiando índices obsoletos...", "109": "Eliminación de recordatorios completada [%s]", @@ -882,13 +882,13 @@ "126": "El marcador no puede estar vacío", "127": "There are [%d] days left before the subscription expires, after which the cloud data will be completely deleted. Please visit Aquí para la renovación, si no necesita renovar, salga de su cuenta para cerrar el recordatorio", "128": "La suscripción ha caducado, los datos de la nube se eliminarán completamente después de la expiración. Para renovar, visite Aquí, si no necesita renovar, salga de su cuenta para cerrar el recordatorio", - "129": "Número de archivos transferidos %d\nTotal de bytes recibidos %s\n", - "130": "Número de archivos transferidos %d\nTotal de bytes enviados %s\n", - "131": "Descargado en %.2fs", - "132": "Cargado en %.2fs", - "133": "No hay cambios en los datos locales", + "129": "TODO", + "130": "TODO", + "131": "TODO", + "132": "TODO", + "133": "TODO", "134": "Para evitar que los datos recién restaurados sean sobrescritos por la sincronización, se ha suspendido automáticamente la función de sincronización de datos", - "135": "Por favor, asegúrese de que todos los dispositivos han sido actualizados a la última versión y, a continuación, active la sincronización después de cambiar aleatoriamente un documento en el dispositivo principal y, finalmente, active la sincronización en otros dispositivos", + "135": "TODO", "136": "Inicializando la clave del repositorio de datos...", "137": "Fallo en la inicialización de la clave del repositorio de datos: %s", "138": "La clave del repositorio de datos está configurada", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 15d140172..56a0ca5e4 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -777,11 +777,11 @@ "21": "Sauvegarde terminée", "22": "En cours de sauvegarde, veuillez patienter....", "23": "La sauvegarde a échoué : %s", - "24": "Impossible d'obtenir les informations de synchronisation du Cloud : %s", + "24": "TODO", "25": "Le nom de l'attribut ne supporte que les lettres et les chiffres anglais.", "26": "Veuillez d'abord initialiser la clé du référentiel de données dans [Paramètres - À propos - Clé du référentiel de données]", "27": "La vérification de l'intégrité des données a échoué", - "28": "Mot de passe de cryptage de bout en bout incorrect, impossible de décrypter les données", + "28": "TODO", "29": "Cette fonctionnalité nécessite un abonnement payant (Si vous êtes déjà abonné, Rafraîchissez ou connectez - vous à nouveau dans Paramètres - compte)", "30": "Impossible d'obtenir des informations sur la sauvegarde dans le Cloud.", "31": "L'authentification du compte a échoué, veuillez vous reconnecter", @@ -793,7 +793,7 @@ "37": "N'incluez pas d'espaces et de symboles spéciaux dans le nom du répertoire de synchronisation cloud", "38": "Le nombre de mots-clés mentionnés [%d] est trop élevé, ne prend actuellement en charge que jusqu'à [512] mots-clés", "39": "TODO", - "40": "Échec du décryptage des données", + "40": "TODO", "41": "Transfert complété", "42": "Le paramétrage est terminé, l'application se fermera automatiquement, merci de redémarrer plus tard...", "43": "La capacité de stockage maximale de l'espace cloud [%s] a été dépassée et le téléchargement des données ne peut pas continuer", @@ -846,7 +846,7 @@ "90": "TODO", "91": "TODO", "92": "TODO", - "93": "Le téléchargement a échoué : %s", + "93": "TODO", "94": "Échec du téléchargement : %s", "95": "Quitter le programme...", "96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud", @@ -856,10 +856,10 @@ "100": "Nettoyage des données...", "101": "Rappel de réglage terminé [%s]", "102": "TODO", - "103": "[%d] fichiers de données ont été téléchargés, il reste à télécharger [%d]", + "103": "TODO", "104": "[%d] fichiers de données ont été téléchargés, et [%d] reste à télécharger", "105": "Transmission réseau terminée", - "106": "Le téléchargement des données est terminé et le décryptage est en cours...", + "106": "TODO", "107": "Déplacement du document [%s]", "108": "Nettoyage des index obsolètes...", "109": "Supprimer le rappel terminé [%s]", @@ -882,13 +882,13 @@ "126": "Les signets ne peuvent pas être vides", "127": "Il reste [%d] jours avant l'expiration de l'abonnement, après quoi les données cloud seront complètement supprimées. Veuillez visiter ici, si vous n'avez pas besoin de renouveler, veuillez vous déconnecter de votre compte pour fermer le rappel", "128": "L'abonnement a expiré, les données cloud seront complètement supprimées après l'expiration. Pour renouveler, veuillez visiter ici , si vous n'avez pas besoin de renouveler, veuillez vous déconnecter de votre compte pour fermer le rappel", - "129": "Fichier transféré %d\noctets reçus %s\n", - "130": "Fichier transféré %d\noctets envoyés %s\n", - "131": "Temps de téléchargement %.2fs", - "132": "Le téléchargement a pris %.2fs", - "133": "Aucune modification des données locales", + "129": "TODO", + "130": "TODO", + "131": "TODO", + "132": "TODO", + "133": "TODO", "134": "Afin d'éviter que les données nouvellement restaurées ne soient écrasées par la synchronisation, la fonction de synchronisation des données a été automatiquement suspendue", - "135": "Assurez-vous que tous les appareils ont été mis à jour vers la dernière version, puis déclenchez la synchronisation après avoir modifié de manière aléatoire un document sur l'appareil principal, et enfin déclenchez la synchronisation sur d'autres appareils.", + "135": "TODO", "136": "Initialisation de la clé du référentiel de données...", "137": "Échec de l'initialisation de la clé du référentiel de données: %s", "138": "La clé du référentiel de données est définie", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 1c30b195f..b7d028e54 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -777,11 +777,11 @@ "21": "備份完畢", "22": "正在備份，請稍等...", "23": "備份失敗：%s", - "24": "獲取雲端同步資訊失敗：%s", + "24": "TODO", "25": "屬性名僅支援英文字母和阿拉伯數字", "26": "請先在 [設置 - 關於 - 數據倉庫密鑰] 中初始化數據倉庫密鑰", "27": "數據完整性校驗失敗", - "28": "端到端加密密碼不正確，無法解密數據", + "28": "TODO", "29": "該功能需要付費訂閱（如果你已經訂閱，請在設定-帳號中重繪或者重新登入）", "30": "獲取雲端備份資訊失敗", "31": "帳號鑒權失敗，請重新登入帳號", @@ -793,7 +793,7 @@ "37": "雲端同步目錄的名稱請勿包含空格和特殊符號", "38": "提及關鍵字數量 [%d] 過多，目前最多僅支援搜索 [512] 個關鍵字", "39": "TODO", - "40": "解密數據失敗", + "40": "TODO", "41": "上傳完畢", "42": "設置完成，即將自動關閉應用，請稍後重新啟動...", "43": "已超過雲端空間最大存儲容量 [%s]，無法繼續上傳數據", @@ -846,7 +846,7 @@ "90": "TODO", "91": "TODO", "92": "TODO", - "93": "下載失敗：%s", + "93": "TODO", "94": "上傳失敗：%s", "95": "正在退出...", "96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致", @@ -856,10 +856,10 @@ "100": "正在清理數據...", "101": "設置提醒完畢 [%s]", "102": "TODO", - "103": "已下載 [%d] 個資料檔案，剩餘待下載 [%d]", + "103": "TODO", "104": "已上傳 [%d] 個資料檔案，剩餘待上傳 [%d]", "105": "網絡傳輸完畢", - "106": "資料下載已經完成，正在進行解密...", + "106": "TODO", "107": "正在移動文檔 [%s]", "108": "正在清理已過時的索引...", "109": "移除提醒完畢 [%s]", @@ -881,13 +881,13 @@ "126": "書籤不能為空", "127": "訂閱距過期還剩 [%d] 天，過期後雲端數據會被徹底刪除。續訂請訪問這裡，如果不需要續訂，請登出賬號關閉該提醒", "128": "訂閱已經過期，過期後雲端數據會被徹底刪除。續訂請訪問這裡，如果不需要續訂，請登出賬號關閉該提醒", - "129": "已傳輸文件 %d\n接收字節數 %s\n", - "130": "已傳輸文件 %d\n發送字節數 %s\n", - "131": "下載耗時 %.2fs", - "132": "上傳耗時 %.2fs", - "133": "本地數據暫無變更", + "129": "TODO", + "130": "TODO", + "131": "TODO", + "132": "TODO", + "133": "TODO", "134": "為避免剛恢復的數據被同步覆蓋，數據同步功能已被自動暫停", - "135": "請確保所有設備已經更新到最新版，然後在主力設備上隨意更改一個文檔後觸發同步，最後再到其他設備觸發同步", + "135": "TODO", "136": "初始化數據倉庫密鑰...", "137": "初始化數據倉庫密鑰失敗：%s", "138": "數據倉庫密鑰設置完畢", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 964f9ee17..2db9a2e2c 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -778,11 +778,11 @@ "21": "备份完毕", "22": "正在备份，请稍等...", "23": "备份失败：%s", - "24": "获取云端同步信息失败：%s", + "24": "TODO", "25": "属性名仅支持英文字母和阿拉伯数字", "26": "请先在 [设置 - 关于 - 数据仓库密钥] 中初始化数据仓库密钥", "27": "数据完整性校验失败", - "28": "端到端加密密码不正确，无法解密数据", + "28": "TODO", "29": "该功能需要付费订阅（如果你已经订阅，请在 设置 - 账号中刷新或者重新登录）", "30": "获取云端备份信息失败", "31": "账号鉴权失败，请重新登录账号", @@ -794,7 +794,7 @@ "37": "云端同步目录的名称请勿包含空格和特殊符号", "38": "提及关键字数量 [%d] 过多，目前最多仅支持搜索 [512] 个关键字", "39": "TODO", - "40": "解密数据失败", + "40": "TODO", "41": "上传完毕", "42": "设置完成，即将自动关闭应用，请稍后重新启动...", "43": "已超过云端空间最大存储容量 [%s]，无法继续上传数据", @@ -847,7 +847,7 @@ "90": "TODO", "91": "TODO", "92": "TODO", - "93": "下载失败：%s", + "93": "TODO", "94": "上传失败：%s", "95": "正在退出...", "96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致", @@ -857,10 +857,10 @@ "100": "正在清理数据...", "101": "设置提醒完毕 [%s]", "102": "TODO", - "103": "已下载 [%d] 个数据文件，剩余待下载 [%d]", + "103": "TODO", "104": "已上传 [%d] 个数据文件，剩余待上传 [%d]", "105": "网络传输完毕", - "106": "数据下载已经完成，正在进行解密...", + "106": "TODO", "107": "正在移动文档 [%s]", "108": "正在清理已过时的索引...", "109": "移除提醒完毕 [%s]", @@ -883,13 +883,13 @@ "126": "书签不能为空", "127": "订阅距过期还剩 [%d] 天，过期后云端数据会被彻底删除。续订请访问这里，如果不需要续订，请登出账号关闭该提醒", "128": "订阅已经过期，过期后云端数据会被彻底删除。续订请访问这里，如果不需要续订，请登出账号关闭该提醒", - "129": "已传输文件 %d\n接收字节数 %s\n", - "130": "已传输文件 %d\n发送字节数 %s\n", - "131": "下载耗时 %.2fs", - "132": "上传耗时 %.2fs", - "133": "本地数据暂无变更", + "129": "TODO", + "130": "TODO", + "131": "TODO", + "132": "TODO", + "133": "TODO", "134": "为避免刚恢复的数据被同步覆盖，数据同步功能已被自动暂停", - "135": "请确保所有设备已经更新到最新版，然后在主力设备上随意更改一个文档后触发同步，最后再到其他设备触发同步", + "135": "TODO", "136": "初始化数据仓库密钥...", "137": "初始化数据仓库密钥失败：%s", "138": "数据仓库密钥设置完毕", diff --git a/kernel/go.mod b/kernel/go.mod index e8eecff4d..62685944b 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -42,7 +42,7 @@ require ( github.com/radovskyb/watcher v1.0.7 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/siyuan-note/dejavu v0.0.0-20220711060744-3fec84096399 - github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 + github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f github.com/siyuan-note/filelock v0.0.0-20220704090116-54dfb035283f github.com/siyuan-note/httpclient v0.0.0-20220709030145-2bfb50f28e73 diff --git a/kernel/go.sum b/kernel/go.sum index 18ca966d3..dd8e83865 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -418,8 +418,8 @@ github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJV github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/siyuan-note/dejavu v0.0.0-20220711060744-3fec84096399 h1:kg4BZwxn4A5d9YD9sx6GnyZ6o+Rn1IiuhrZ5qYrVXV0= github.com/siyuan-note/dejavu v0.0.0-20220711060744-3fec84096399/go.mod h1:cri+XyZAqmK5fJ98En9aOHB+YkuU8+XQcJdQ31EUhis= -github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 h1:QB9TjJQFhXhZ6dAtPpY02DlzHAQm1C+WqZq6OadG8mI= -github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= +github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE= +github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw= github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f h1:JMobMNZ7AqaKKyEK+WeWFhix/2TDQXgPZDajU00IybU= github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI= github.com/siyuan-note/filelock v0.0.0-20220704090116-54dfb035283f h1:IXZ4SWPjQLqMrBwDWcWYFE/SihUHRS9FYhk/0bnySok= diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go index afac2ecab..c69d648ec 100644 --- a/kernel/model/osssync.go +++ b/kernel/model/osssync.go @@ -20,19 +20,16 @@ import ( "context" "errors" "fmt" - "io" "io/fs" "os" "path" "path/filepath" - "sort" "strconv" "strings" "sync" "time" "github.com/88250/gulu" - "github.com/imroc/req/v3" "github.com/panjf2000/ants/v2" "github.com/qiniu/go-sdk/v7/storage" "github.com/siyuan-note/httpclient" @@ -41,20 +38,11 @@ import ( func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, err error) { result := map[string]interface{}{} - request := httpclient.NewCloudRequest() + resp, err := httpclient.NewCloudRequest(). + SetResult(&result). + SetBody(map[string]string{"token": Conf.User.UserToken}). + Post(util.AliyunServer + "/apis/siyuan/dejavu/getRepoStat?uid=" + Conf.User.UserId) - var resp *req.Response - if Conf.Sync.UseDataRepo { - resp, err = request. - SetResult(&result). - SetBody(map[string]string{"token": Conf.User.UserToken}). - Post(util.AliyunServer + "/apis/siyuan/dejavu/getRepoStat?uid=" + Conf.User.UserId) - } else { - resp, err = request. - SetResult(&result). - SetBody(map[string]string{"token": Conf.User.UserToken}). - Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanWorkspace?uid=" + Conf.User.UserId) - } if nil != err { util.LogErrorf("get cloud space failed: %s", err) err = ErrFailedToConnectCloudServer @@ -80,255 +68,6 @@ func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, e return } -func removeCloudDirPath(dirPath string) (err error) { - result := map[string]interface{}{} - request := httpclient.NewCloudRequest() - resp, err := request. - SetResult(&result). - SetBody(map[string]string{"dirPath": dirPath, "token": Conf.User.UserToken}). - Post(util.AliyunServer + "/apis/siyuan/data/removeSiYuanDirPath?uid=" + Conf.User.UserId) - if nil != err { - util.LogErrorf("create cloud sync dir failed: %s", err) - return ErrFailedToConnectCloudServer - } - - if 200 != resp.StatusCode { - if 401 == resp.StatusCode { - err = errors.New(Conf.Language(31)) - return - } - msg := fmt.Sprintf("remove cloud dir failed: %d", resp.StatusCode) - util.LogErrorf(msg) - err = errors.New(msg) - return - } - return -} - -func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error) { - result := map[string]interface{}{} - request := httpclient.NewCloudRequest() - resp, err := request. - SetBody(map[string]interface{}{"token": Conf.User.UserToken}). - SetResult(&result). - Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanSyncDirList?uid=" + Conf.User.UserId) - if nil != err { - util.LogErrorf("get cloud sync dirs failed: %s", err) - return nil, 0, ErrFailedToConnectCloudServer - } - - if 200 != resp.StatusCode { - if 401 == resp.StatusCode { - err = errors.New(Conf.Language(31)) - return - } - msg := fmt.Sprintf("get cloud sync dirs failed: %d", resp.StatusCode) - util.LogErrorf(msg) - err = errors.New(msg) - return - } - - code := result["code"].(float64) - if 0 != code { - util.LogErrorf("get cloud sync dirs failed: %s", result["msg"]) - return nil, 0, ErrFailedToConnectCloudServer - } - - data := result["data"].(map[string]interface{}) - dataDirs := data["dirs"].([]interface{}) - for _, d := range dataDirs { - dirs = append(dirs, d.(map[string]interface{})) - } - sort.Slice(dirs, func(i, j int) bool { return dirs[i]["name"].(string) < dirs[j]["name"].(string) }) - size = int64(data["size"].(float64)) - return -} - -func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles map[string]bool, err error) { - if !gulu.File.IsExist(localDirPath) { - return - } - - cloudFileList, err := getCloudFileListOSS(cloudDirPath) - if nil != err { - return - } - - if "backup" != cloudDirPath { - // 将云端索引文件临时保存一下，后面下载数据时如果部分成功，需要用索引文件恢复部分成功的文件 syncDirUpsertWorkspaceData() - - var data []byte - data, err = gulu.JSON.MarshalJSON(cloudFileList) - if nil != err { - return - } - tmpSyncDir := filepath.Join(util.TempDir, "sync") - err = os.MkdirAll(tmpSyncDir, 0755) - if nil != err { - return - } - tmpIndex := filepath.Join(tmpSyncDir, "index.json") - if err = os.WriteFile(tmpIndex, data, 0644); nil != err { - return - } - } - - localRemoves, cloudFetches, err := localUpsertRemoveListOSS(localDirPath, cloudFileList) - if nil != err { - return - } - - for _, localRemove := range localRemoves { - if err = os.RemoveAll(localRemove); nil != err { - util.LogErrorf("local remove [%s] failed: %s", localRemove, err) - return - } - } - - needPushProgress := 32 < len(cloudFetches) - waitGroup := &sync.WaitGroup{} - var downloadErr error - downloadedFilesLock := sync.Mutex{} - downloadedFiles = map[string]bool{} - poolSize := 4 - if poolSize > len(cloudFetches)-1 /* 不计入 /.siyuan/conf.json，配置文件最后单独下载 */ { - poolSize = len(cloudFetches) - } - p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) { - defer waitGroup.Done() - if nil != downloadErr { - return // 快速失败 - } - - fetch := arg.(string) - err = ossDownload0(localDirPath, cloudDirPath, fetch, &fetchedFilesCount, &transferSize, bootOrExit) - if nil != err { - downloadErr = err // 仅记录最后一次错误 - return - } - downloadedFilesLock.Lock() - downloadedFiles[fetch] = true - downloadedFilesLock.Unlock() - - if needPushProgress { - msg := fmt.Sprintf(Conf.Language(103), fetchedFilesCount, len(cloudFetches)-fetchedFilesCount) - util.PushProgress(util.PushProgressCodeProgressed, fetchedFilesCount, len(cloudFetches), msg) - } - if bootOrExit { - msg := fmt.Sprintf("Downloading data from the cloud %d/%d", fetchedFilesCount, len(cloudFetches)) - util.IncBootProgress(0, msg) - } - }) - for _, fetch := range cloudFetches { - if "/.siyuan/conf.json" == fetch { - // 同步下载可能会报错，为了确保本地数据版本号不变所以不能更新配置文件，配置文件最后单独下载 - continue - } - if "/"+pathJSON == fetch { - // 已经在前面验证解密的步骤中下载过了，目前位于 temp/sync/pathJSON - continue - } - - waitGroup.Add(1) - p.Invoke(fetch) - } - waitGroup.Wait() - p.Release() - if nil != downloadErr { - err = downloadErr - return - } - - if "backup" != cloudDirPath { - err = ossDownload0(localDirPath, cloudDirPath, "/.siyuan/conf.json", &fetchedFilesCount, &transferSize, bootOrExit) - if nil != err { - return - } - } - if needPushProgress { - util.ClearPushProgress(len(cloudFetches)) - util.PushMsg(Conf.Language(106), 1000*60*10) - } - if bootOrExit { - util.IncBootProgress(0, "Decrypting from sync to data...") - } - return -} - -func ossDownload0(localDirPath, cloudDirPath, fetch string, fetchedFiles *int, transferSize *uint64, bootORExit bool) (err error) { - localFilePath := filepath.Join(localDirPath, fetch) - remoteFileURL := path.Join(cloudDirPath, fetch) - var result map[string]interface{} - resp, err := httpclient.NewCloudRequest(). - SetResult(&result). - SetBody(map[string]interface{}{"token": Conf.User.UserToken, "path": remoteFileURL}). - Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanFile?uid=" + Conf.User.UserId) - if nil != err { - util.LogErrorf("download request [%s] failed: %s", remoteFileURL, err) - return errors.New(fmt.Sprintf(Conf.Language(93), err)) - } - - if 200 != resp.StatusCode { - if 401 == resp.StatusCode { - err = errors.New("account authentication failed, please login again") - return errors.New(fmt.Sprintf(Conf.Language(93), err)) - } - util.LogErrorf("download request status code [%d]", resp.StatusCode) - err = errors.New("download file URL failed") - return errors.New(fmt.Sprintf(Conf.Language(93), err)) - } - - code := result["code"].(float64) - if 0 != code { - msg := result["msg"].(string) - util.LogErrorf("download cloud file failed: %s", msg) - return errors.New(fmt.Sprintf(Conf.Language(93), msg)) - } - - resultData := result["data"].(map[string]interface{}) - downloadURL := resultData["url"].(string) - - if err = os.MkdirAll(filepath.Dir(localFilePath), 0755); nil != err { - return - } - os.Remove(localFilePath) - - if bootORExit { - resp, err = httpclient.NewCloudFileRequest15s().Get(downloadURL) - } else { - resp, err = httpclient.NewCloudFileRequest2m().Get(downloadURL) - } - if nil != err { - util.LogErrorf("download request [%s] failed: %s", downloadURL, err) - return errors.New(fmt.Sprintf(Conf.Language(93), err)) - } - if 200 != resp.StatusCode { - util.LogErrorf("download request [%s] status code [%d]", downloadURL, resp.StatusCode) - err = errors.New(fmt.Sprintf("download file failed [%d]", resp.StatusCode)) - if 404 == resp.StatusCode { - err = errors.New(Conf.Language(135)) - } - return errors.New(fmt.Sprintf(Conf.Language(93), err)) - } - - data, err := resp.ToBytes() - if nil != err { - util.LogErrorf("download read response body data failed: %s, %s", err, string(data)) - err = errors.New("download read data failed") - return errors.New(fmt.Sprintf(Conf.Language(93), err)) - } - size := int64(len(data)) - - if err = gulu.File.WriteFileSafer(localFilePath, data, 0644); nil != err { - util.LogErrorf("write file [%s] failed: %s", localFilePath, err) - return errors.New(fmt.Sprintf(Conf.Language(93), err)) - } - - *fetchedFiles++ - *transferSize += uint64(size) - return -} - func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, boot bool) (wroteFiles int, transferSize uint64, err error) { if !gulu.File.IsExist(localDirPath) { return @@ -530,90 +269,6 @@ func getOssUploadToken(filename, cloudDirPath string, length int64) (ret string, return } -func getCloudSyncVer(cloudDir string) (cloudSyncVer int64, err error) { - start := time.Now() - result := map[string]interface{}{} - request := httpclient.NewCloudRequest() - resp, err := request. - SetResult(&result). - SetBody(map[string]string{"syncDir": cloudDir, "token": Conf.User.UserToken}). - Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanWorkspaceSyncVer?uid=" + Conf.User.UserId) - if nil != err { - util.LogErrorf("get cloud sync ver failed: %s", err) - err = ErrFailedToConnectCloudServer - return - } - if 200 != resp.StatusCode { - if 401 == resp.StatusCode { - err = errors.New(Conf.Language(31)) - return - } - util.LogErrorf("get cloud sync ver failed: %d", resp.StatusCode) - err = ErrFailedToConnectCloudServer - return - } - - code := result["code"].(float64) - if 0 != code { - msg := result["msg"].(string) - util.LogErrorf("get cloud sync ver failed: %s", msg) - err = errors.New(msg) - return - } - - data := result["data"].(map[string]interface{}) - cloudSyncVer = int64(data["v"].(float64)) - - if elapsed := time.Now().Sub(start).Milliseconds(); 2000 < elapsed { - util.LogInfof("get cloud sync ver elapsed [%dms]", elapsed) - } - return -} - -func getCloudSync(cloudDir string) (assetSize, backupSize int64, device string, err error) { - start := time.Now() - result := map[string]interface{}{} - request := httpclient.NewCloudRequest() - resp, err := request. - SetResult(&result). - SetBody(map[string]string{"syncDir": cloudDir, "token": Conf.User.UserToken}). - Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanWorkspaceSync?uid=" + Conf.User.UserId) - if nil != err { - util.LogErrorf("get cloud sync info failed: %s", err) - err = ErrFailedToConnectCloudServer - return - } - if 200 != resp.StatusCode { - if 401 == resp.StatusCode { - err = errors.New(Conf.Language(31)) - return - } - util.LogErrorf("get cloud sync info failed: %d", resp.StatusCode) - err = ErrFailedToConnectCloudServer - return - } - - code := result["code"].(float64) - if 0 != code { - msg := result["msg"].(string) - util.LogErrorf("get cloud sync info failed: %s", msg) - err = errors.New(msg) - return - } - - data := result["data"].(map[string]interface{}) - assetSize = int64(data["assetSize"].(float64)) - backupSize = int64(data["backupSize"].(float64)) - if nil != data["d"] { - device = data["d"].(string) - } - - if elapsed := time.Now().Sub(start).Milliseconds(); 5000 < elapsed { - util.LogInfof("get cloud sync [%s] elapsed [%dms]", elapsed) - } - return -} - func getLocalFileListOSS(isBackup bool) (ret map[string]*CloudIndex, err error) { ret = map[string]*CloudIndex{} dir := "sync" @@ -636,109 +291,6 @@ func getLocalFileListOSS(isBackup bool) (ret map[string]*CloudIndex, err error) return } -func getCloudFileListOSS(cloudDirPath string) (ret map[string]*CloudIndex, err error) { - result := map[string]interface{}{} - request := httpclient.NewCloudRequest() - resp, err := request. - SetResult(&result). - SetBody(map[string]string{"dirPath": cloudDirPath, "token": Conf.User.UserToken}). - Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanFileListURL?uid=" + Conf.User.UserId) - if nil != err { - util.LogErrorf("get cloud file list failed: %s", err) - err = ErrFailedToConnectCloudServer - return - } - - if 401 == resp.StatusCode { - err = errors.New(Conf.Language(31)) - return - } - - code := result["code"].(float64) - if 0 != code { - util.LogErrorf("get cloud file list failed: %s", result["msg"]) - err = ErrFailedToConnectCloudServer - return - } - - retData := result["data"].(map[string]interface{}) - downloadURL := retData["url"].(string) - resp, err = httpclient.NewCloudFileRequest15s().Get(downloadURL) - if nil != err { - util.LogErrorf("download request [%s] failed: %s", downloadURL, err) - return - } - if 200 != resp.StatusCode { - util.LogErrorf("download request [%s] status code [%d]", downloadURL, resp.StatusCode) - err = errors.New(fmt.Sprintf("download file list failed [%d]", resp.StatusCode)) - return - } - - data, err := resp.ToBytes() - if err = gulu.JSON.UnmarshalJSON(data, &ret); nil != err { - util.LogErrorf("unmarshal index failed: %s", err) - err = errors.New(fmt.Sprintf("unmarshal index failed")) - return - } - return -} - -func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*CloudIndex) (localRemoves, cloudFetches []string, err error) { - unchanged := map[string]bool{} - - filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error { - if localDirPath == path { - return nil - } - - if info.IsDir() { - return nil - } - - relPath := filepath.ToSlash(strings.TrimPrefix(path, localDirPath)) - cloudIdx, ok := cloudFileList[relPath] - if !ok { - if util.CloudSingleFileMaxSizeLimit < info.Size() { - util.LogWarnf("file [%s] larger than 100MB, ignore removing it", path) - return nil - } - - localRemoves = append(localRemoves, path) - return nil - } - - if 0 < cloudIdx.Updated { - // 优先使用时间戳校验 - if localModTime := info.ModTime().Unix(); cloudIdx.Updated == localModTime { - unchanged[relPath] = true - } - return nil - } - - localHash, hashErr := util.GetEtag(path) - if nil != hashErr { - err = hashErr - return io.EOF - } - if cloudIdx.Hash == localHash { - unchanged[relPath] = true - } - return nil - }) - - for cloudPath, cloudIndex := range cloudFileList { - if _, ok := unchanged[cloudPath]; ok { - continue - } - if util.CloudSingleFileMaxSizeLimit < cloudIndex.Size { - util.LogWarnf("cloud file [%s] larger than 100MB, ignore fetching it", cloudPath) - continue - } - cloudFetches = append(cloudFetches, cloudPath) - } - return -} - func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList map[string]*CloudIndex, excludes map[string]bool) (localUpserts, cloudRemoves []string, err error) { localUpserts, cloudRemoves = []string{}, []string{} diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 9f877db84..a046e2250 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -17,13 +17,10 @@ package model import ( - "bytes" - "crypto/sha256" "errors" "fmt" "io" "io/fs" - "math" "os" "os/exec" "path/filepath" @@ -34,11 +31,8 @@ import ( "github.com/88250/gulu" "github.com/dustin/go-humanize" - gitignore "github.com/sabhiram/go-gitignore" "github.com/siyuan-note/dejavu" - "github.com/siyuan-note/encryption" "github.com/siyuan-note/filelock" - "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" @@ -126,98 +120,11 @@ func SyncData(boot, exit, byHand bool) { util.BroadcastByType("main", "syncing", 1, msg, nil) }() - Conf.Sync.Stat = Conf.Language(133) - syncLock.Lock() - defer syncLock.Unlock() - if Conf.Sync.UseDataRepo { syncRepo(boot, exit, byHand) return } - WaitForWritingFiles() - writingDataLock.Lock() - var err error - // 将 data 变更同步到 sync - if _, _, err = workspaceData2SyncDir(); nil != err { - msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) - Conf.Sync.Stat = msg - util.PushErrMsg(msg, 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - writingDataLock.Unlock() - return - } - - // 获取工作空间数据配置（数据版本） - dataConf, err := getWorkspaceDataConf() - if nil != err { - msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) - Conf.Sync.Stat = msg - util.PushErrMsg(msg, 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - writingDataLock.Unlock() - return - } - writingDataLock.Unlock() - - cloudSyncVer, err := getCloudSyncVer(Conf.Sync.CloudName) - if nil != err { - msg := fmt.Sprintf(Conf.Language(24), err.Error()) - Conf.Sync.Stat = msg - util.PushErrMsg(msg, 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - return - } - - //util.LogInfof("sync [cloud=%d, local=%d]", cloudSyncVer, dataConf.SyncVer) - if cloudSyncVer == dataConf.SyncVer { - BootSyncSucc = 0 - ExitSyncSucc = 0 - syncSameCount++ - if 10 < syncSameCount { - syncSameCount = 5 - } - if !byHand { - delay := time.Minute * time.Duration(int(math.Pow(2, float64(syncSameCount)))) - if fixSyncInterval.Minutes() > delay.Minutes() { - delay = time.Minute * 8 - } - planSyncAfter(delay) - } - - Conf.Sync.Stat = Conf.Language(133) - return - } - - cloudUsedAssetSize, cloudUsedBackupSize, device, err := getCloudSync(Conf.Sync.CloudName) - if nil != err { - msg := fmt.Sprintf(Conf.Language(24), err.Error()) - Conf.Sync.Stat = msg - util.PushErrMsg(msg, 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - return - } - localSyncDirPath := Conf.Sync.GetSaveDir() syncSameCount = 0 if cloudSyncVer < dataConf.SyncVer { @@ -280,159 +187,9 @@ func SyncData(boot, exit, byHand bool) { } return } - - // 下载 - - if !boot && !exit { - CloseWatchAssets() - defer WatchAssets() - } - - start := time.Now() - //util.LogInfof("sync [cloud=%d, local=%d] downloading...", cloudSyncVer, dataConf.SyncVer) - - // 使用路径映射文件进行解密验证 https://github.com/siyuan-note/siyuan/issues/3789 - var tmpFetchedFiles int - var tmpTransferSize uint64 - err = ossDownload0(util.TempDir+"/sync", "sync/"+Conf.Sync.CloudName, "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, boot || exit) - if nil != err { - util.PushClearProgress() - msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) - Conf.Sync.Stat = msg - util.PushErrMsg(msg, 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - syncDownloadErrCount++ - return - } - - tmpPathJSON := filepath.Join(util.TempDir, "/sync/"+pathJSON) - data, err := os.ReadFile(tmpPathJSON) - if nil != err { - return - } - data, err = encryption.AESGCMDecryptBinBytes(data, Conf.E2EEPasswd) - if nil != err { - util.PushClearProgress() - msg := Conf.Language(28) - Conf.Sync.Stat = msg - util.PushErrMsg(fmt.Sprintf(Conf.Language(80), msg), 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - Conf.Sync.Stat = msg - syncDownloadErrCount++ - return - } - - fetchedFilesCount, transferSize, downloadedFiles, err := ossDownload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit) - - // 加上前面的路径映射文件统计 - fetchedFilesCount += tmpFetchedFiles - transferSize += tmpTransferSize - - if nil != err { - util.PushClearProgress() - msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) - Conf.Sync.Stat = msg - util.PushErrMsg(msg, 7000) - - indexPath := filepath.Join(util.TempDir, "sync", "index.json") - _, err = syncDirUpsertWorkspaceData(tmpPathJSON, indexPath, downloadedFiles) - if nil != err { - util.LogErrorf("upsert partially downloaded files to workspace data failed: %s", err) - } - - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - syncDownloadErrCount++ - return - } - util.PushClearProgress() - - // 恢复 - var upsertFiles, removeFiles []string - if upsertFiles, removeFiles, err = syncDir2WorkspaceData(boot); nil != err { - msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) - Conf.Sync.Stat = msg - util.PushErrMsg(msg, 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - syncDownloadErrCount++ - return - } - - syncDownloadErrCount = 0 - - // 清理空文件夹 - clearEmptyDirs(util.DataDir) - - elapsed := time.Now().Sub(start).Seconds() - stat := fmt.Sprintf(Conf.Language(129), fetchedFilesCount, humanize.Bytes(transferSize)) + fmt.Sprintf(Conf.Language(131), elapsed) - util.LogInfof("sync [cloud=%d, local=%d, fetchedFiles=%d, transferSize=%s] downloaded in [%.2fs]", cloudSyncVer, dataConf.SyncVer, fetchedFilesCount, humanize.Bytes(transferSize), elapsed) - - Conf.Sync.Downloaded = now - Conf.Sync.Stat = stat - BootSyncSucc = 0 - ExitSyncSucc = 0 - if !byHand { - planSyncAfter(fixSyncInterval) - } - - if boot && gulu.File.IsExist(util.BlockTreePath) { - // 在 blocktree 存在的情况下不会重建索引，所以这里需要刷新 blocktree 和 database - treenode.InitBlockTree() - incReindex(upsertFiles, removeFiles) - return - } - - if !boot && !exit { - incReindex(upsertFiles, removeFiles) - cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939 - util.ReloadUI() - } return } -// TODO: 新版同步上线后移除 -// 清理 dir 下符合 ID 规则的空文件夹。 -// 因为是深度遍历，所以可能会清理不完全空文件夹，每次遍历仅能清理叶子节点。但是多次调用后，可以清理完全。 -func clearEmptyDirs(dir string) { - var emptyDirs []string - filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { - if nil != err || !info.IsDir() || dir == path { - return err - } - - if util.IsIDPattern(info.Name()) { - if files, readDirErr := os.ReadDir(path); nil == readDirErr && 0 == len(files) { - emptyDirs = append(emptyDirs, path) - } - } - return nil - }) - for _, emptyDir := range emptyDirs { - if err := os.RemoveAll(emptyDir); nil != err { - util.LogErrorf("clear empty dir [%s] failed [%s]", emptyDir, err.Error()) - } - } -} - // incReindex 增量重建索引。 func incReindex(upserts, removes []string) { needPushUpsertProgress := 32 < len(upserts) @@ -522,97 +279,6 @@ func SetSyncMode(mode int) (err error) { var syncLock = sync.Mutex{} -func syncDirUpsertWorkspaceData(metaPath, indexPath string, downloadedFiles map[string]bool) (upsertFiles []string, err error) { - start := time.Now() - - modified := map[string]bool{} - syncDir := Conf.Sync.GetSaveDir() - for file, _ := range downloadedFiles { - file = filepath.Join(syncDir, file) - modified[file] = true - } - - decryptedDataDir, upsertFiles, err := recoverSyncData(metaPath, indexPath, modified) - if nil != err { - util.LogErrorf("decrypt data dir failed: %s", err) - return - } - - dataDir := util.DataDir - if err = stableCopy(decryptedDataDir, dataDir); nil != err { - util.LogErrorf("copy decrypted data dir from [%s] to data dir [%s] failed: %s", decryptedDataDir, dataDir, err) - return - } - if elapsed := time.Since(start).Milliseconds(); 5000 < elapsed { - util.LogInfof("sync data to workspace data elapsed [%dms]", elapsed) - } - return -} - -// syncDir2WorkspaceData 将 sync 的数据更新到 data 中。 -// 1. 删除 data 中冗余的文件 -// 2. 将 sync 中新增/修改的文件解密后拷贝到 data 中 -func syncDir2WorkspaceData(boot bool) (upsertFiles, removeFiles []string, err error) { - start := time.Now() - unchanged, removeFiles, err := calcUnchangedSyncList() - if nil != err { - return - } - - modified := modifiedSyncList(unchanged) - metaPath := filepath.Join(util.TempDir, "sync", pathJSON) // 使用前面解密验证时下载的临时文件 - indexPath := filepath.Join(util.TempDir, "sync", "index.json") - decryptedDataDir, upsertFiles, err := recoverSyncData(metaPath, indexPath, modified) - if nil != err { - util.LogErrorf("decrypt data dir failed: %s", err) - return - } - - if boot { - util.IncBootProgress(0, "Copying decrypted data...") - } - - dataDir := util.DataDir - if err = stableCopy(decryptedDataDir, dataDir); nil != err { - util.LogErrorf("copy decrypted data dir from [%s] to data dir [%s] failed: %s", decryptedDataDir, dataDir, err) - return - } - if elapsed := time.Since(start).Milliseconds(); 5000 < elapsed { - util.LogInfof("sync data to workspace data elapsed [%dms]", elapsed) - } - return -} - -// workspaceData2SyncDir 将 data 的数据更新到 sync 中。 -// 1. 删除 sync 中多余的文件 -// 2. 将 data 中新增/修改的文件加密后拷贝到 sync 中 -func workspaceData2SyncDir() (removeList, upsertList map[string]bool, err error) { - start := time.Now() - filelock.ReleaseAllFileLocks() - - passwd := Conf.E2EEPasswd - unchangedDataList, removeList, err := calcUnchangedDataList(passwd) - if nil != err { - return - } - - encryptedDataDir, upsertList, err := prepareSyncData(passwd, unchangedDataList) - if nil != err { - util.LogErrorf("encrypt data dir failed: %s", err) - return - } - - syncDir := Conf.Sync.GetSaveDir() - if err = stableCopy(encryptedDataDir, syncDir); nil != err { - util.LogErrorf("copy encrypted data dir from [%s] to sync dir [%s] failed: %s", encryptedDataDir, syncDir, err) - return - } - if elapsed := time.Since(start).Milliseconds(); 5000 < elapsed { - util.LogInfof("workspace data to sync data elapsed [%dms]", elapsed) - } - return -} - type CloudIndex struct { Hash string `json:"hash"` Size int64 `json:"size"` @@ -664,340 +330,6 @@ func genCloudIndex(localDirPath string, excludes map[string]bool, calcHash bool) return } -func recoverSyncData(metaPath, indexPath string, modified map[string]bool) (decryptedDataDir string, upsertFiles []string, err error) { - passwd := Conf.E2EEPasswd - decryptedDataDir = filepath.Join(util.TempDir, "incremental", "sync-decrypt") - if err = os.RemoveAll(decryptedDataDir); nil != err { - return - } - if err = os.MkdirAll(decryptedDataDir, 0755); nil != err { - return - } - - syncDir := Conf.Sync.GetSaveDir() - data, err := os.ReadFile(metaPath) - if nil != err { - return - } - data, err = encryption.AESGCMDecryptBinBytes(data, passwd) - if nil != err { - err = errors.New(Conf.Language(40)) - return - } - - metaJSON := map[string]string{} - if err = gulu.JSON.UnmarshalJSON(data, &metaJSON); nil != err { - return - } - - index := map[string]*CloudIndex{} - data, err = os.ReadFile(indexPath) - if nil != err { - return - } - if err = gulu.JSON.UnmarshalJSON(data, &index); nil != err { - return - } - - now := time.Now().Format("2006-01-02-150405") - filepath.Walk(syncDir, func(path string, info fs.FileInfo, _ error) error { - if syncDir == path || pathJSON == info.Name() { - return nil - } - - // 如果不是新增或者修改则跳过 - if !modified[path] { - return nil - } - - encryptedP := strings.TrimPrefix(path, syncDir+string(os.PathSeparator)) - encryptedP = filepath.ToSlash(encryptedP) - if "" == metaJSON[encryptedP] { - return nil - } - - plainP := filepath.Join(decryptedDataDir, metaJSON[encryptedP]) - plainP = filepath.FromSlash(plainP) - - p := strings.TrimPrefix(plainP, decryptedDataDir+string(os.PathSeparator)) - upsertFiles = append(upsertFiles, p) - dataPath := filepath.Join(util.DataDir, p) - if gulu.File.IsExist(dataPath) && !gulu.File.IsDir(dataPath) { // 不是目录的话说明必定是已经存在的文件，这些文件被覆盖需要生成历史 - genSyncHistory(now, dataPath) - } - - if info.IsDir() { - if err = os.MkdirAll(plainP, 0755); nil != err { - return io.EOF - } - } else { - if err = os.MkdirAll(filepath.Dir(plainP), 0755); nil != err { - return io.EOF - } - - var err0 error - data, err0 = os.ReadFile(path) - if nil != err0 { - util.LogErrorf("read file [%s] failed: %s", path, err0) - err = err0 - return io.EOF - } - if !strings.HasPrefix(encryptedP, ".siyuan") { - data, err0 = encryption.AESGCMDecryptBinBytes(data, passwd) - if nil != err0 { - util.LogErrorf("decrypt file [%s] failed: %s", path, err0) - err = errors.New(Conf.Language(40)) - return io.EOF - } - } - - if err0 = gulu.File.WriteFileSafer(plainP, data, 0644); nil != err0 { - util.LogErrorf("write file [%s] failed: %s", plainP, err0) - err = err0 - return io.EOF - } - - var modTime int64 - idx := index["/"+encryptedP] - if nil == idx { - util.LogErrorf("index file [%s] not found", encryptedP) - modTime = info.ModTime().Unix() - } else { - modTime = idx.Updated - } - if err0 = os.Chtimes(plainP, time.Unix(modTime, 0), time.Unix(modTime, 0)); nil != err0 { - util.LogErrorf("change file [%s] time failed: %s", plainP, err0) - } - } - return nil - }) - return -} - -func prepareSyncData(passwd string, unchangedDataList map[string]bool) (encryptedDataDir string, upsertList map[string]bool, err error) { - encryptedDataDir = filepath.Join(util.TempDir, "incremental", "sync-encrypt") - if err = os.RemoveAll(encryptedDataDir); nil != err { - return - } - if err = os.MkdirAll(encryptedDataDir, 0755); nil != err { - return - } - - ctime := map[string]time.Time{} - meta := map[string]string{} - filepath.Walk(util.DataDir, func(path string, info fs.FileInfo, _ error) error { - if util.DataDir == path || nil == info { - return nil - } - - isDir := info.IsDir() - if isCloudSkipFile(path, info) { - if isDir { - return filepath.SkipDir - } - return nil - } - - plainP := strings.TrimPrefix(path, util.DataDir+string(os.PathSeparator)) - p := plainP - - if !strings.HasPrefix(plainP, ".siyuan") { // 配置目录下都用明文，其他文件需要映射文件名 - p = pathSha256Short(p, string(os.PathSeparator)) - } - if !isDir { - meta[filepath.ToSlash(p)] = filepath.ToSlash(plainP) - } - - // 如果不是新增或者修改则跳过 - if unchangedDataList[path] { - return nil - } - - p = encryptedDataDir + string(os.PathSeparator) + p - //util.LogInfof("update sync [%s] for data [%s]", p, path) - if isDir { - if err = os.MkdirAll(p, 0755); nil != err { - return io.EOF - } - } else { - if err = os.MkdirAll(filepath.Dir(p), 0755); nil != err { - return io.EOF - } - - data, err0 := filelock.NoLockFileRead(path) - if nil != err0 { - util.LogErrorf("read file [%s] failed: %s", path, err0) - err = err0 - return io.EOF - } - if !strings.HasPrefix(plainP, ".siyuan") { - data, err0 = encryption.AESGCMEncryptBinBytes(data, passwd) - if nil != err0 { - util.LogErrorf("encrypt file [%s] failed: %s", path, err0) - err = errors.New("encrypt file failed") - return io.EOF - } - } - - err0 = gulu.File.WriteFileSafer(p, data, 0644) - if nil != err0 { - util.LogErrorf("write file [%s] failed: %s", p, err0) - err = err0 - return io.EOF - } - - fi, err0 := os.Stat(path) - if nil != err0 { - util.LogErrorf("stat file [%s] failed: %s", path, err0) - err = err0 - return io.EOF - } - ctime[p] = fi.ModTime() - } - return nil - }) - if nil != err { - return - } - - for p, t := range ctime { - if err = os.Chtimes(p, t, t); nil != err { - return - } - } - - upsertList = map[string]bool{} - // 检查文件是否全部已经编入索引 - err = filepath.Walk(encryptedDataDir, func(path string, info fs.FileInfo, _ error) error { - if encryptedDataDir == path { - return nil - } - - if !info.IsDir() { - path = strings.TrimPrefix(path, encryptedDataDir+string(os.PathSeparator)) - path = filepath.ToSlash(path) - if _, ok := meta[path]; !ok { - util.LogErrorf("not found sync path in meta [%s]", path) - return errors.New(Conf.Language(27)) - } - - upsertList["/"+path] = true - } - return nil - }) - if nil != err { - return - } - - data, err := gulu.JSON.MarshalJSON(meta) - if nil != err { - return - } - data, err = encryption.AESGCMEncryptBinBytes(data, passwd) - if nil != err { - util.LogErrorf("encrypt file failed: %s", err) - return - } - if err = gulu.File.WriteFileSafer(filepath.Join(encryptedDataDir, pathJSON), data, 0644); nil != err { - return - } - return -} - -// modifiedSyncList 获取 sync 中新增和修改的文件列表。 -func modifiedSyncList(unchangedList map[string]bool) (ret map[string]bool) { - ret = map[string]bool{} - syncDir := Conf.Sync.GetSaveDir() - filepath.Walk(syncDir, func(path string, info fs.FileInfo, _ error) error { - if syncDir == path || pathJSON == info.Name() { - return nil - } - - if !unchangedList[path] { - ret[path] = true - } - return nil - }) - return -} - -// calcUnchangedSyncList 获取 data 和 sync 一致（没有修改过）的文件列表，并删除 data 中不存在于 sync 中的多余文件。 -func calcUnchangedSyncList() (ret map[string]bool, removes []string, err error) { - syncDir := Conf.Sync.GetSaveDir() - meta := filepath.Join(syncDir, pathJSON) - if !gulu.File.IsExist(meta) { - return - } - data, err := os.ReadFile(meta) - if nil != err { - return - } - passwd := Conf.E2EEPasswd - data, err = encryption.AESGCMDecryptBinBytes(data, passwd) - if nil != err { - err = errors.New(Conf.Language(40)) - return - } - - metaJSON := map[string]string{} - if err = gulu.JSON.UnmarshalJSON(data, &metaJSON); nil != err { - return - } - - excludes := getSyncExcludedList(syncDir) - ret = map[string]bool{} - sep := string(os.PathSeparator) - filepath.Walk(util.DataDir, func(path string, info fs.FileInfo, _ error) error { - if util.DataDir == path { - return nil - } - - if isCloudSkipFile(path, info) { - if info.IsDir() { - return filepath.SkipDir - } - return nil - } - - plainP := strings.TrimPrefix(path, util.DataDir+sep) - dataP := plainP - dataP = pathSha256Short(dataP, sep) - syncP := filepath.Join(syncDir, dataP) - - if excludes[syncP] { - return nil - } - - if !gulu.File.IsExist(syncP) { // sync 已经删除的文件 - removes = append(removes, path) - if gulu.File.IsDir(path) { - return filepath.SkipDir - } - return nil - } - - stat, _ := os.Stat(syncP) - syncModTime := stat.ModTime() - if info.ModTime() == syncModTime { - ret[syncP] = true - return nil - } - return nil - }) - - // 在 data 中删除 sync 已经删除的文件 - now := time.Now().Format("2006-01-02-150405") - for _, remove := range removes { - genSyncHistory(now, remove) - if ".siyuan" != filepath.Base(remove) { - if err = os.RemoveAll(remove); nil != err { - util.LogErrorf("remove [%s] failed: %s", remove, err) - } - } - } - return -} - func getWorkspaceDataConf() (conf *filesys.DataConf, err error) { conf = &filesys.DataConf{Updated: util.CurrentTimeMillis(), Device: Conf.System.ID} confPath := filepath.Join(Conf.Sync.GetSaveDir(), ".siyuan", "conf.json") @@ -1039,26 +371,6 @@ func incLocalSyncVer() { return } -func isCloudSkipFile(path string, info os.FileInfo) bool { - baseName := info.Name() - if strings.HasPrefix(baseName, ".") { - if ".siyuan" == baseName { - return false - } - return true - } - if "history" == baseName { - if strings.HasSuffix(path, ".siyuan"+string(os.PathSeparator)+"history") { - return true - } - } - - if (os.ModeSymlink == info.Mode()&os.ModeType) || (!strings.Contains(path, ".siyuan") && gulu.File.IsHidden(path)) { - return true - } - return false -} - func CreateCloudSyncDir(name string) (err error) { syncLock.Lock() defer syncLock.Unlock() @@ -1087,18 +399,13 @@ func RemoveCloudSyncDir(name string) (err error) { return } - if Conf.Sync.UseDataRepo { - var cloudInfo *dejavu.CloudInfo - cloudInfo, err = buildCloudInfo() - if nil != err { - return - } - - err = dejavu.RemoveCloudRepo(name, cloudInfo) - } else { - err = removeCloudDirPath("sync/" + name) + var cloudInfo *dejavu.CloudInfo + cloudInfo, err = buildCloudInfo() + if nil != err { + return } + err = dejavu.RemoveCloudRepo(name, cloudInfo) if nil != err { return } @@ -1115,17 +422,14 @@ func ListCloudSyncDir() (syncDirs []*Sync, hSize string, err error) { syncDirs = []*Sync{} var dirs []map[string]interface{} var size int64 - if Conf.Sync.UseDataRepo { - var cloudInfo *dejavu.CloudInfo - cloudInfo, err = buildCloudInfo() - if nil != err { - return - } - dirs, size, err = dejavu.GetCloudRepos(cloudInfo) - } else { - dirs, size, err = listCloudSyncDirOSS() + var cloudInfo *dejavu.CloudInfo + cloudInfo, err = buildCloudInfo() + if nil != err { + return } + + dirs, size, err = dejavu.GetCloudRepos(cloudInfo) if nil != err { return } @@ -1143,31 +447,6 @@ func ListCloudSyncDir() (syncDirs []*Sync, hSize string, err error) { return } -func genSyncHistory(now, p string) { - dir := strings.TrimPrefix(p, util.DataDir+string(os.PathSeparator)) - if strings.Contains(dir, string(os.PathSeparator)) { - dir = dir[:strings.Index(dir, string(os.PathSeparator))] - } - - if ".siyuan" == dir || ".siyuan" == filepath.Base(p) { - return - } - - historyDir, err := util.GetHistoryDirNow(now, "sync") - if nil != err { - util.LogErrorf("get history dir failed: %s", err) - return - } - - relativePath := strings.TrimPrefix(p, util.DataDir) - historyPath := filepath.Join(historyDir, relativePath) - filelock.ReleaseFileLocks(p) - if err = gulu.File.Copy(p, historyPath); nil != err { - util.LogErrorf("gen sync history failed: %s", err) - return - } -} - func formatErrorMsg(err error) string { msg := err.Error() if strings.Contains(msg, "Permission denied") || strings.Contains(msg, "Access is denied") { @@ -1199,36 +478,6 @@ func IsValidCloudDirName(cloudDirName string) bool { return true } -func getSyncExcludedList(localDirPath string) (ret map[string]bool) { - syncIgnoreList := getSyncIgnoreList() - ret = map[string]bool{} - for _, p := range syncIgnoreList { - relPath := p - relPath = pathSha256Short(relPath, "/") - relPath = filepath.Join(localDirPath, relPath) - ret[relPath] = true - } - return -} - -func getSyncIgnoreList() (ret []string) { - lines := getIgnoreLines() - if 1 > len(lines) { - return - } - - gi := gitignore.CompileIgnoreLines(lines...) - filepath.Walk(util.DataDir, func(p string, info os.FileInfo, err error) error { - p = strings.TrimPrefix(p, util.DataDir+string(os.PathSeparator)) - p = filepath.ToSlash(p) - if gi.MatchesPath(p) { - ret = append(ret, p) - } - return nil - }) - return -} - func getIgnoreLines() (ret []string) { ignore := filepath.Join(util.DataDir, ".siyuan", "syncignore") err := os.MkdirAll(filepath.Dir(ignore), 0755) @@ -1259,18 +508,6 @@ func getIgnoreLines() (ret []string) { return } -func pathSha256Short(p, sep string) string { - buf := bytes.Buffer{} - parts := strings.Split(p, sep) - for i, part := range parts { - buf.WriteString(fmt.Sprintf("%x", sha256.Sum256([]byte(part)))[:7]) - if i < len(parts)-1 { - buf.WriteString(sep) - } - } - return buf.String() -} - func GetSyncDirection(cloudDirName string) (code int, msg string) { // 0：失败，10：上传，20：下载，30：一致，40：使用数据仓库同步 if !IsSubscriber() { return diff --git a/kernel/util/working.go b/kernel/util/working.go index 81021f14a..293f4d051 100644 --- a/kernel/util/working.go +++ b/kernel/util/working.go @@ -139,15 +139,6 @@ func SetBooted() { LogInfof("kernel booted") } -func GetHistoryDirNow(now, suffix string) (ret string, err error) { - ret = filepath.Join(HistoryDir, now+"-"+suffix) - if err = os.MkdirAll(ret, 0755); nil != err { - LogErrorf("make history dir failed: %s", err) - return - } - return -} - func GetHistoryDir(suffix string) (ret string, err error) { ret = filepath.Join(HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix) if err = os.MkdirAll(ret, 0755); nil != err { From e538824819ad3147ca60f98233d7d889f0b15e57 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 13 Jul 2022 20:03:48 +0800 Subject: [PATCH 4/8] =?UTF-8?q?:fire:=20=E7=A7=BB=E9=99=A4=E6=97=A7?= =?UTF-8?q?=E7=89=88=E4=BA=91=E7=AB=AF=E5=90=8C=E6=AD=A5=E5=92=8C=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8A=9F=E8=83=BD=E5=85=A5=E5=8F=A3=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/5405?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 4 +- app/appearance/langs/es_ES.json | 4 +- app/appearance/langs/fr_FR.json | 4 +- app/appearance/langs/zh_CHT.json | 4 +- app/appearance/langs/zh_CN.json | 4 +- kernel/model/osssync.go | 353 ------------------------------- kernel/model/repository.go | 33 +++ kernel/model/sync.go | 121 +---------- 8 files changed, 44 insertions(+), 483 deletions(-) delete mode 100644 kernel/model/osssync.go diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index 2f3613494..c7580deac 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -857,8 +857,8 @@ "101": "Done setting reminder [%s]", "102": "TODO", "103": "TODO", - "104": "[%d] data files have been uploaded, and [%d] remaining to be uploaded", - "105": "Network transmission completed", + "104": "TODO", + "105": "TODO", "106": "TODO", "107": "Moving document [%s]", "108": "Cleaning obsolete indexes...", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 68dfdccf0..5934de329 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -857,8 +857,8 @@ "101": "El recordatorio de configuración [%s] se ha completado", "102": "TODO", "103": "TODO", - "104": "[%d] archivos de datos han sido cargados, y [%d] restantes por cargar", - "105": "Transmisión de red completada", + "104": "TODO", + "105": "TODO", "106": "TODO", "107": "Moviendo documento [%s]", "108": "Limpiando índices obsoletos...", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index 56a0ca5e4..d6ed6803e 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -857,8 +857,8 @@ "101": "Rappel de réglage terminé [%s]", "102": "TODO", "103": "TODO", - "104": "[%d] fichiers de données ont été téléchargés, et [%d] reste à télécharger", - "105": "Transmission réseau terminée", + "104": "TODO", + "105": "TODO", "106": "TODO", "107": "Déplacement du document [%s]", "108": "Nettoyage des index obsolètes...", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index b7d028e54..434a278ed 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -857,8 +857,8 @@ "101": "設置提醒完畢 [%s]", "102": "TODO", "103": "TODO", - "104": "已上傳 [%d] 個資料檔案，剩餘待上傳 [%d]", - "105": "網絡傳輸完畢", + "104": "TODO", + "105": "TODO", "106": "TODO", "107": "正在移動文檔 [%s]", "108": "正在清理已過時的索引...", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 2db9a2e2c..633784d05 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -858,8 +858,8 @@ "101": "设置提醒完毕 [%s]", "102": "TODO", "103": "TODO", - "104": "已上传 [%d] 个数据文件，剩余待上传 [%d]", - "105": "网络传输完毕", + "104": "TODO", + "105": "TODO", "106": "TODO", "107": "正在移动文档 [%s]", "108": "正在清理已过时的索引...", diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go deleted file mode 100644 index c69d648ec..000000000 --- a/kernel/model/osssync.go +++ /dev/null @@ -1,353 +0,0 @@ -// SiYuan - Build Your Eternal Digital Garden -// Copyright (c) 2020-present, b3log.org -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package model - -import ( - "context" - "errors" - "fmt" - "io/fs" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/88250/gulu" - "github.com/panjf2000/ants/v2" - "github.com/qiniu/go-sdk/v7/storage" - "github.com/siyuan-note/httpclient" - "github.com/siyuan-note/siyuan/kernel/util" -) - -func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, err error) { - result := map[string]interface{}{} - resp, err := httpclient.NewCloudRequest(). - SetResult(&result). - SetBody(map[string]string{"token": Conf.User.UserToken}). - Post(util.AliyunServer + "/apis/siyuan/dejavu/getRepoStat?uid=" + Conf.User.UserId) - - if nil != err { - util.LogErrorf("get cloud space failed: %s", err) - err = ErrFailedToConnectCloudServer - return - } - - if 401 == resp.StatusCode { - err = errors.New(Conf.Language(31)) - return - } - - code := result["code"].(float64) - if 0 != code { - util.LogErrorf("get cloud space failed: %s", result["msg"]) - err = errors.New(result["msg"].(string)) - return - } - - data := result["data"].(map[string]interface{}) - sync = data["sync"].(map[string]interface{}) - backup = data["backup"].(map[string]interface{}) - assetSize = int64(data["assetSize"].(float64)) - return -} - -func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, boot bool) (wroteFiles int, transferSize uint64, err error) { - if !gulu.File.IsExist(localDirPath) { - return - } - - localDevice := Conf.System.ID - var localFileList, cloudFileList map[string]*CloudIndex - if "" != localDevice && localDevice == cloudDevice && !isBackup { - // 同一台设备连续上传，使用上一次的本地索引作为云端索引 - cloudFileList, err = getLocalFileListOSS(isBackup) - } else { - cloudFileList, err = getCloudFileListOSS(cloudDirPath) - } - if nil != err { - return - } - - calcHash := false - if 0 < len(cloudFileList) { - if idx := cloudFileList["/index.json"]; nil != idx { - calcHash = 0 == idx.Updated - } - } - - excludes := getSyncExcludedList(localDirPath) - localFileList, err = genCloudIndex(localDirPath, excludes, calcHash) - if nil != err { - return - } - - var localUpserts, cloudRemoves []string - localUpserts, cloudRemoves, err = cloudUpsertRemoveListOSS(localDirPath, cloudFileList, localFileList, excludes) - if nil != err { - return - } - - err = ossRemove0(cloudDirPath, cloudRemoves) - if nil != err { - return - } - - needPushProgress := 32 < len(localUpserts) - waitGroup := &sync.WaitGroup{} - var uploadErr error - - poolSize := 4 - if poolSize > len(localUpserts) { - poolSize = len(localUpserts) - } - msgId := gulu.Rand.String(7) - p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) { - defer waitGroup.Done() - if nil != uploadErr { - return // 快速失败 - } - localUpsert := arg.(string) - err = ossUpload0(localDirPath, cloudDirPath, localUpsert, &wroteFiles, &transferSize) - if nil != err { - uploadErr = err - return - } - if needPushProgress { - util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(104), wroteFiles, len(localUpserts)-wroteFiles), 1000*60) - } - if boot { - msg := fmt.Sprintf("Uploading data to the cloud %d/%d", wroteFiles, len(localUpserts)) - util.IncBootProgress(0, msg) - } - }) - index := filepath.Join(localDirPath, "index.json") - meta := filepath.Join(localDirPath, pathJSON) - for _, localUpsert := range localUpserts { - if index == localUpsert || meta == localUpsert { - // 同步过程中断导致的一致性问题 https://github.com/siyuan-note/siyuan/issues/4912 - // index 和路径映射文件最后单独上传 - continue - } - - waitGroup.Add(1) - p.Invoke(localUpsert) - } - waitGroup.Wait() - p.Release() - if nil != uploadErr { - err = uploadErr - return - } - - // 单独上传 index 和路径映射 - if uploadErr = ossUpload0(localDirPath, cloudDirPath, index, &wroteFiles, &transferSize); nil != uploadErr { - err = uploadErr - return - } - if uploadErr = ossUpload0(localDirPath, cloudDirPath, meta, &wroteFiles, &transferSize); nil != uploadErr { - err = uploadErr - return - } - - if needPushProgress { - util.PushMsg(Conf.Language(105), 3000) - util.PushClearMsg(msgId) - } - return -} - -func ossRemove0(cloudDirPath string, removes []string) (err error) { - if 1 > len(removes) { - return - } - - request := httpclient.NewCloudRequest() - resp, err := request. - SetBody(map[string]interface{}{"token": Conf.User.UserToken, "dirPath": cloudDirPath, "paths": removes}). - Post(util.AliyunServer + "/apis/siyuan/data/removeSiYuanFile?uid=" + Conf.User.UserId) - if nil != err { - util.LogErrorf("remove cloud file failed: %s", err) - err = ErrFailedToConnectCloudServer - return - } - - if 401 == resp.StatusCode { - err = errors.New(Conf.Language(31)) - return - } - - if 200 != resp.StatusCode { - msg := fmt.Sprintf("remove cloud file failed [sc=%d]", resp.StatusCode) - util.LogErrorf(msg) - err = errors.New(msg) - return - } - return -} - -func ossUpload0(localDirPath, cloudDirPath, localUpsert string, wroteFiles *int, transferSize *uint64) (err error) { - info, statErr := os.Stat(localUpsert) - if nil != statErr { - util.LogErrorf("stat file [%s] failed: %s", localUpsert, statErr) - err = statErr - return - } - - filename := filepath.ToSlash(strings.TrimPrefix(localUpsert, localDirPath)) - upToken, err := getOssUploadToken(filename, cloudDirPath, info.Size()) - if nil != err { - return - } - - key := path.Join("siyuan", Conf.User.UserId, cloudDirPath, filename) - if err = putFileToCloud(localUpsert, key, upToken); nil != err { - util.LogErrorf("put file [%s] to cloud failed: %s", localUpsert, err) - return errors.New(fmt.Sprintf(Conf.Language(94), err)) - } - - //util.LogInfof("cloud wrote [%s], size [%d]", filename, info.Size()) - *wroteFiles++ - *transferSize += uint64(info.Size()) - return -} - -func getOssUploadToken(filename, cloudDirPath string, length int64) (ret string, err error) { - // 因为需要指定 key，所以每次上传文件都必须在云端生成 Token，否则有安全隐患 - - var result map[string]interface{} - req := httpclient.NewCloudRequest(). - SetResult(&result) - req.SetBody(map[string]interface{}{ - "token": Conf.User.UserToken, - "dirPath": cloudDirPath, - "name": filename, - "length": length}) - resp, err := req.Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanFileUploadToken?uid=" + Conf.User.UserId) - if nil != err { - util.LogErrorf("get file [%s] upload token failed: %+v", filename, err) - err = errors.New(fmt.Sprintf(Conf.Language(94), err)) - return - } - - if 200 != resp.StatusCode { - if 401 == resp.StatusCode { - err = errors.New(fmt.Sprintf(Conf.Language(94), Conf.Language(31))) - return - } - util.LogErrorf("get file [%s] upload token failed [sc=%d]", filename, resp.StatusCode) - err = errors.New(fmt.Sprintf(Conf.Language(94), strconv.Itoa(resp.StatusCode))) - return - } - - code := result["code"].(float64) - if 0 != code { - msg := result["msg"].(string) - util.LogErrorf("get file [%s] upload token failed: %s", filename, msg) - err = errors.New(fmt.Sprintf(Conf.Language(93), msg)) - return - } - - resultData := result["data"].(map[string]interface{}) - ret = resultData["token"].(string) - return -} - -func getLocalFileListOSS(isBackup bool) (ret map[string]*CloudIndex, err error) { - ret = map[string]*CloudIndex{} - dir := "sync" - if isBackup { - dir = "backup" - } - - localDirPath := filepath.Join(util.WorkspaceDir, dir) - indexPath := filepath.Join(localDirPath, "index.json") - if !gulu.File.IsExist(indexPath) { - return - } - - data, err := os.ReadFile(indexPath) - if nil != err { - return - } - - err = gulu.JSON.UnmarshalJSON(data, &ret) - return -} - -func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList map[string]*CloudIndex, excludes map[string]bool) (localUpserts, cloudRemoves []string, err error) { - localUpserts, cloudRemoves = []string{}, []string{} - - unchanged := map[string]bool{} - for cloudFile, cloudIdx := range cloudFileList { - localIdx := localFileList[cloudFile] - if nil == localIdx { - cloudRemoves = append(cloudRemoves, cloudFile) - continue - } - if 0 < cloudIdx.Updated { - // 优先使用时间戳校验 - if localIdx.Updated == cloudIdx.Updated { - unchanged[filepath.Join(localDirPath, cloudFile)] = true - } - continue - } - - if localIdx.Hash == cloudIdx.Hash { - unchanged[filepath.Join(localDirPath, cloudFile)] = true - continue - } - } - - filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error { - if localDirPath == path || info.IsDir() { - return nil - } - - if !unchanged[path] { - if excludes[path] { - return nil - } - if util.CloudSingleFileMaxSizeLimit < info.Size() { - util.LogWarnf("file [%s] larger than 100MB, ignore uploading it", path) - return nil - } - localUpserts = append(localUpserts, path) - return nil - } - return nil - }) - return -} - -func putFileToCloud(filePath, key, upToken string) (err error) { - formUploader := storage.NewFormUploader(&storage.Config{UseHTTPS: true}) - ret := storage.PutRet{} - err = formUploader.PutFile(context.Background(), &ret, upToken, key, filePath, nil) - if nil != err { - util.LogWarnf("put file [%s] to cloud failed [%s], retry it after 3s", filePath, err) - time.Sleep(3 * time.Second) - err = formUploader.PutFile(context.Background(), &ret, upToken, key, filePath, nil) - if nil != err { - return - } - util.LogInfof("put file [%s] to cloud retry success", filePath) - } - return -} diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 0c3b5a18c..5565e8e1a 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -32,6 +32,7 @@ import ( "github.com/siyuan-note/encryption" "github.com/siyuan-note/eventbus" "github.com/siyuan-note/filelock" + "github.com/siyuan-note/httpclient" "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/util" @@ -738,3 +739,35 @@ func buildCloudInfo() (ret *dejavu.CloudInfo, err error) { } return } + +func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, err error) { + result := map[string]interface{}{} + resp, err := httpclient.NewCloudRequest(). + SetResult(&result). + SetBody(map[string]string{"token": Conf.User.UserToken}). + Post(util.AliyunServer + "/apis/siyuan/dejavu/getRepoStat?uid=" + Conf.User.UserId) + + if nil != err { + util.LogErrorf("get cloud space failed: %s", err) + err = ErrFailedToConnectCloudServer + return + } + + if 401 == resp.StatusCode { + err = errors.New(Conf.Language(31)) + return + } + + code := result["code"].(float64) + if 0 != code { + util.LogErrorf("get cloud space failed: %s", result["msg"]) + err = errors.New(result["msg"].(string)) + return + } + + data := result["data"].(map[string]interface{}) + sync = data["sync"].(map[string]interface{}) + backup = data["backup"].(map[string]interface{}) + assetSize = int64(data["assetSize"].(float64)) + return +} diff --git a/kernel/model/sync.go b/kernel/model/sync.go index a046e2250..9edcd8963 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -19,8 +19,6 @@ package model import ( "errors" "fmt" - "io" - "io/fs" "os" "os/exec" "path/filepath" @@ -120,73 +118,7 @@ func SyncData(boot, exit, byHand bool) { util.BroadcastByType("main", "syncing", 1, msg, nil) }() - if Conf.Sync.UseDataRepo { - syncRepo(boot, exit, byHand) - return - } - - localSyncDirPath := Conf.Sync.GetSaveDir() - syncSameCount = 0 - if cloudSyncVer < dataConf.SyncVer { - // 上传 - - if -1 == cloudSyncVer { - // 初次上传 - IncWorkspaceDataVer() - incLocalSyncVer() - } - - start := time.Now() - //util.LogInfof("sync [cloud=%d, local=%d] uploading...", cloudSyncVer, dataConf.SyncVer) - syncSize, err := util.SizeOfDirectory(localSyncDirPath, false) - if nil != err { - util.PushErrMsg(fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)), 7000) - return - } - - leftSyncSize := int64(Conf.User.UserSiYuanRepoSize) - cloudUsedAssetSize - cloudUsedBackupSize - if leftSyncSize < syncSize { - util.PushErrMsg(fmt.Sprintf(Conf.Language(43), byteCountSI(int64(Conf.User.UserSiYuanRepoSize))), 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - return - } - - wroteFiles, transferSize, err := ossUpload(false, localSyncDirPath, "sync/"+Conf.Sync.CloudName, device, boot) - if nil != err { - util.PushClearProgress() - IncWorkspaceDataVer() // 上传失败的话提升本地版本，以备下次上传 - - msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) - Conf.Sync.Stat = msg - util.PushErrMsg(msg, 7000) - if boot { - BootSyncSucc = 1 - } - if exit { - ExitSyncSucc = 1 - } - return - } - - util.PushClearProgress() - elapsed := time.Now().Sub(start).Seconds() - stat := fmt.Sprintf(Conf.Language(130), wroteFiles, humanize.Bytes(transferSize)) + fmt.Sprintf(Conf.Language(132), elapsed) - util.LogInfof("sync [cloud=%d, local=%d, wroteFiles=%d, transferSize=%s] uploaded in [%.2fs]", cloudSyncVer, dataConf.SyncVer, wroteFiles, humanize.Bytes(transferSize), elapsed) - - Conf.Sync.Uploaded = now - Conf.Sync.Stat = stat - BootSyncSucc = 0 - ExitSyncSucc = 0 - if !byHand { - planSyncAfter(fixSyncInterval) - } - return - } + syncRepo(boot, exit, byHand) return } @@ -279,57 +211,6 @@ func SetSyncMode(mode int) (err error) { var syncLock = sync.Mutex{} -type CloudIndex struct { - Hash string `json:"hash"` - Size int64 `json:"size"` - Updated int64 `json:"updated"` // Unix timestamp 秒 -} - -// genCloudIndex 生成云端索引文件。 -func genCloudIndex(localDirPath string, excludes map[string]bool, calcHash bool) (cloudIndex map[string]*CloudIndex, err error) { - cloudIndex = map[string]*CloudIndex{} - err = filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error { - if nil != err { - return err - } - if localDirPath == path || info.IsDir() || excludes[path] { - return nil - } - - if util.CloudSingleFileMaxSizeLimit < info.Size() { - return nil - } - - p := strings.TrimPrefix(path, localDirPath) - p = filepath.ToSlash(p) - hash := "" - if calcHash { - var hashErr error - hash, hashErr = util.GetEtag(path) - if nil != hashErr { - err = hashErr - return io.EOF - } - } - cloudIndex[p] = &CloudIndex{Hash: hash, Size: info.Size(), Updated: info.ModTime().Unix()} - return nil - }) - if nil != err { - util.LogErrorf("walk sync dir [%s] failed: %s", localDirPath, err) - return - } - data, err := gulu.JSON.MarshalJSON(cloudIndex) - if nil != err { - util.LogErrorf("marshal sync cloud index failed: %s", err) - return - } - if err = gulu.File.WriteFileSafer(filepath.Join(localDirPath, "index.json"), data, 0644); nil != err { - util.LogErrorf("write sync cloud index failed: %s", err) - return - } - return -} - func getWorkspaceDataConf() (conf *filesys.DataConf, err error) { conf = &filesys.DataConf{Updated: util.CurrentTimeMillis(), Device: Conf.System.ID} confPath := filepath.Join(Conf.Sync.GetSaveDir(), ".siyuan", "conf.json") From 2863e32cff359664126ce29d081a9a6d45bbf2d4 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 13 Jul 2022 20:23:04 +0800 Subject: [PATCH 5/8] =?UTF-8?q?:fire:=20=E7=A7=BB=E9=99=A4=E6=97=A7?= =?UTF-8?q?=E7=89=88=E4=BA=91=E7=AB=AF=E5=90=8C=E6=AD=A5=E5=92=8C=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8A=9F=E8=83=BD=E5=85=A5=E5=8F=A3=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/5405?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/appearance/langs/en_US.json | 14 +- app/appearance/langs/es_ES.json | 14 +- app/appearance/langs/fr_FR.json | 14 +- app/appearance/langs/zh_CHT.json | 14 +- app/appearance/langs/zh_CN.json | 14 +- kernel/api/backup.go | 129 -------- kernel/api/repo.go | 27 ++ kernel/api/router.go | 7 - kernel/model/backup.go | 514 ------------------------------- kernel/model/box.go | 24 -- kernel/model/export.go | 2 +- 11 files changed, 63 insertions(+), 710 deletions(-) delete mode 100644 kernel/api/backup.go diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json index c7580deac..4721efc41 100644 --- a/app/appearance/langs/en_US.json +++ b/app/appearance/langs/en_US.json @@ -774,16 +774,16 @@ "18": "Get community user account failed", "19": "User information has expired, please log in again", "20": "Cannot be converted to heading when including sub-documents", - "21": "Backup completed", - "22": "Backuping, please wait...", - "23": "Backup failed: %s", + "21": "TODO", + "22": "TODO", + "23": "TODO", "24": "TODO", "25": "The attribute name only supports English letters and digits", "26": "Please initialize the data repo key first in [Settings - About - Data repo key]", "27": "Data integrity check failed", "28": "TODO", "29": "This feature requires paid subscription (If you have subscribed, please refresh or log in again in settings - account)", - "30": "Failed to obtain cloud backup info", + "30": "Failed to obtain cloud info", "31": "Account authentication failed, please login again", "32": "Failed to remove cloud notebook", "33": "Insufficient permissions to read and write files or access to the network, please check the permissions of the workspace folder and the settings of the anti-virus software/firewall. If you have run SiYuan as an administrator before, please consider switching to a new workspace directory, and do not run it as an administrator in the future (the current workspace directory may no longer be accessible by ordinary users)", @@ -814,14 +814,14 @@ "58": "After the index is rebuilt, the interface will be automatically refreshed later...", "59": "Failed to set sync ignore list", "60": "Failed to get the update package: %s", - "61": "Uploading, please wait...", - "62": "The recovery is complete, and the index will be rebuilt...", + "61": "TODO", + "62": "TODO", "63": "Recovering, please wait...", "64": "There are [%d] files in total, it will take some time to index, please wait...", "65": "Exporting data...", "66": "Data file [%s] created", "67": "Uploaded at %s, downloaded at %s", - "68": "Downloading, please wait...", + "68": "TODO", "69": "Download completed", "70": "Copy notebook [%s] file [%s] failed: %s", "71": "Failed to insert asset file, please reopen the document", diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json index 5934de329..b47f1d367 100644 --- a/app/appearance/langs/es_ES.json +++ b/app/appearance/langs/es_ES.json @@ -774,16 +774,16 @@ "18": "Falló la obtención de la cuenta de usuario de la comunidad", "19": "La información del usuario ha caducado, por favor, inicie sesión de nuevo", "20": "No se puede convertir en título al incluir subdocumentos", - "21": "Copia de seguridad completada", - "22": "Haciendo copia de seguridad, por favor espere...", - "23": "Copia de seguridad fallida: %s", + "21": "TODO", + "22": "TODO", + "23": "TODO", "24": "TODO", "25": "El nombre del atributo sólo admite letras y dígitos en inglés", "26": "Por favor, inicialice primero la clave de repositorio de datos en [Configuración - Acerca de - Clave de repositorio de datos]", "27": "Falló la comprobación de la integridad de los datos", "28": "TODO", "29": "Esta función requiere una suscripción de pago (Si se ha suscrito, actualice o vuelva a conectarse en configuración - cuenta)", - "30": "Fallo en la obtención de la información de la copia de seguridad en la nube", + "30": "No se pudo obtener la información de la nube", "31": "Falló la autentificación de la cuenta, por favor, inicie sesión de nuevo", "32": "Fallo en la eliminación de la libreta en la nube", "33": "Permisos insuficientes para leer y escribir archivos o acceso a la red, por favor comprueba los permisos de la carpeta del espacio de trabajo y la configuración del software antivirus/firewall. Si has ejecutado SiYuan como administrador antes, por favor considera cambiar a un nuevo directorio de espacio de trabajo, y no lo ejecutes como administrador en el futuro (el directorio de espacio de trabajo actual puede que ya no sea accesible por los usuarios ordinarios)", @@ -814,14 +814,14 @@ "58": "Después de reconstruir el índice, la interfaz se actualizará automáticamente más tarde...", "59": " Falló la configuración de sincronización de la lista de ignorados", "60": "Fallo al obtener el paquete de actualización: %s", - "61": "Cargando, por favor espere...", - "62": "La recuperación se ha completado y el índice se reconstruirá...", + "61": "TODO", + "62": "TODO", "63": "Recuperando, por favor espere...", "64": "Hay [%d] archivos en total, tardará un tiempo en indexarse, por favor espere...", "65": "Exportando datos...", "66": "Archivo de datos [%s] creado", "67": "Cargado en %s, descargado en %s", - "68": "Descargando, por favor espere...", + "68": "TODO", "69": "Descarga completada", "70": "Error en la copia del cuaderno [%s] del archivo [%s]: %s", "71": "Fallo en la inserción del archivo de activos, por favor reabra el documento", diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json index d6ed6803e..ff6da4a64 100644 --- a/app/appearance/langs/fr_FR.json +++ b/app/appearance/langs/fr_FR.json @@ -774,16 +774,16 @@ "18": "Échec de la récupération du compte utilisateur communautaire", "19": "Les informations de l'utilisateur ont expiré, veuillez vous connecter à nouveau.", "20": "Ne peut pas être converti en titre lorsque des sous-documents sont inclus.", - "21": "Sauvegarde terminée", - "22": "En cours de sauvegarde, veuillez patienter....", - "23": "La sauvegarde a échoué : %s", + "21": "TODO", + "22": "TODO", + "23": "TODO", "24": "TODO", "25": "Le nom de l'attribut ne supporte que les lettres et les chiffres anglais.", "26": "Veuillez d'abord initialiser la clé du référentiel de données dans [Paramètres - À propos - Clé du référentiel de données]", "27": "La vérification de l'intégrité des données a échoué", "28": "TODO", "29": "Cette fonctionnalité nécessite un abonnement payant (Si vous êtes déjà abonné, Rafraîchissez ou connectez - vous à nouveau dans Paramètres - compte)", - "30": "Impossible d'obtenir des informations sur la sauvegarde dans le Cloud.", + "30": "Échec de l'obtention des informations sur le cloud", "31": "L'authentification du compte a échoué, veuillez vous reconnecter", "32": "Échec de la suppression de carnet de notes du Cloud", "33": "Autorisations insuffisantes pour lire et écrire des fichiers ou accéder au réseau, veuillez vérifier les autorisations du dossier de l'espace de travail et les paramètres du logiciel anti-virus/pare-feu. Si vous avez déjà exécuté SiYuan en tant qu'administrateur, envisagez de passer à un nouveau répertoire d'espace de travail et ne l'exécutez plus en tant qu'administrateur à l'avenir (le répertoire d'espace de travail actuel peut ne plus être accessible aux utilisateurs ordinaires) ", @@ -814,14 +814,14 @@ "58": "Une fois l'index reconstruit, l'interface sera automatiquement rafraîchie ultérieurement...", "59": "Échec de la définition de la liste des ignores de synchronisation", "60": "Échec de la récupération du paquet de mise à jour : %s", - "61": "En cours de transfert, veuillez patienter...", - "62": "La récupération est terminée, et l'index sera reconstruit...", + "61": "TODO", + "62": "TODO", "63": "Récupération, veuillez patienter...", "64": "Il y a [%d] fichiers au total, l'indexation prendra un certain temps, veuillez patienter...", "65": "Exportation des données...", "66": "Fichier de données [%s] créé", "67": "Transféré à %s, téléchargé à %s", - "68": "En cours de téléchargement, veuillez patienter...", + "68": "TODO", "69": "Téléchargement terminé", "70": "La copie du carnet de notes [%s] du fichier [%s] a échoué : %s", "71": "L'insertion du fichier asset a échoué, veuillez rouvrir le document.", diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json index 434a278ed..c6e2d8188 100644 --- a/app/appearance/langs/zh_CHT.json +++ b/app/appearance/langs/zh_CHT.json @@ -774,16 +774,16 @@ "18": "獲取社區用戶帳號失敗", "19": "使用者資訊已過期，請重新登入帳號", "20": "包含子文檔時無法轉換為標題", - "21": "備份完畢", - "22": "正在備份，請稍等...", - "23": "備份失敗：%s", + "21": "TODO", + "22": "TODO", + "23": "TODO", "24": "TODO", "25": "屬性名僅支援英文字母和阿拉伯數字", "26": "請先在 [設置 - 關於 - 數據倉庫密鑰] 中初始化數據倉庫密鑰", "27": "數據完整性校驗失敗", "28": "TODO", "29": "該功能需要付費訂閱（如果你已經訂閱，請在設定-帳號中重繪或者重新登入）", - "30": "獲取雲端備份資訊失敗", + "30": "獲取雲端資訊失敗", "31": "帳號鑒權失敗，請重新登入帳號", "32": "刪除雲端筆記本失敗", "33": "讀寫檔或存取網路權限不足，請檢查工作空間資料夾權限和防毒軟體/防火牆的設置。如果你曾經使用管理員身份運行過思源，請考慮切換到新的工作空間目錄，後續請勿使用管理員身份運行（當前的工作空間目錄可能已經無法使用普通用戶存取）", @@ -814,14 +814,14 @@ "58": "重建索引完畢，稍後將自動重新整理介面...", "59": "設置同步忽略列表失敗", "60": "獲取更新包失敗：%s", - "61": "上傳中，請稍等...", - "62": "恢復完畢，即將重建索引...", + "61": "TODO", + "62": "TODO", "63": "正在恢復，請稍等...", "64": "共有檔 [%d] 個，需要一些時間進行索引，請稍等...", "65": "導出數據中...", "66": "已創建資料檔案 [%s]", "67": "上傳於 %s，下載於 %s", - "68": "下載中，請稍等...", + "68": "TODO", "69": "下載完畢", "70": "複製筆記本 [%s] 下的檔 [%s] 失敗：%s", "71": "插入資料檔失敗，請重新打開文檔", diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json index 633784d05..f3cf1eecd 100644 --- a/app/appearance/langs/zh_CN.json +++ b/app/appearance/langs/zh_CN.json @@ -775,16 +775,16 @@ "18": "获取社区用户账号失败", "19": "用户信息已过期，请重新登录账号", "20": "包含子文档时无法转换为标题", - "21": "备份完毕", - "22": "正在备份，请稍等...", - "23": "备份失败：%s", + "21": "TODO", + "22": "TODO", + "23": "TODO", "24": "TODO", "25": "属性名仅支持英文字母和阿拉伯数字", "26": "请先在 [设置 - 关于 - 数据仓库密钥] 中初始化数据仓库密钥", "27": "数据完整性校验失败", "28": "TODO", "29": "该功能需要付费订阅（如果你已经订阅，请在 设置 - 账号中刷新或者重新登录）", - "30": "获取云端备份信息失败", + "30": "获取云端信息失败", "31": "账号鉴权失败，请重新登录账号", "32": "删除云端笔记本失败", "33": "读写文件或访问网络权限不足，请检查工作空间文件夹权限和杀毒软件/防火墙的设置。如果你曾经使用管理员身份运行过思源，请考虑切换到新的工作空间目录，后续请勿使用管理员身份运行（当前的工作空间目录可能已经无法使用普通用户访问）", @@ -815,14 +815,14 @@ "58": "重建索引完毕，稍后将自动刷新界面...", "59": "设置同步忽略列表失败", "60": "获取更新包失败：%s", - "61": "上传中，请稍等...", - "62": "恢复完毕，即将重建索引...", + "61": "TODO", + "62": "TODO", "63": "正在恢复，请稍等...", "64": "共有文件 [%d] 个，需要一些时间进行索引，请稍等...", "65": "导出数据中...", "66": "已创建数据文件 [%s]", "67": "上传于 %s，下载于 %s", - "68": "下载中，请稍等...", + "68": "TODO", "69": "下载完毕", "70": "复制笔记本 [%s] 下的文件 [%s] 失败：%s", "71": "插入资源文件失败，请重新打开文档", diff --git a/kernel/api/backup.go b/kernel/api/backup.go deleted file mode 100644 index 0ea8dc613..000000000 --- a/kernel/api/backup.go +++ /dev/null @@ -1,129 +0,0 @@ -// SiYuan - Build Your Eternal Digital Garden -// Copyright (c) 2020-present, b3log.org -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package api - -import ( - "net/http" - - "github.com/88250/gulu" - "github.com/dustin/go-humanize" - "github.com/gin-gonic/gin" - "github.com/siyuan-note/siyuan/kernel/model" - "github.com/siyuan-note/siyuan/kernel/util" -) - -func removeCloudBackup(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - err := model.RemoveCloudBackup() - if nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } -} - -func downloadCloudBackup(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - err := model.DownloadBackup() - if nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } -} - -func uploadLocalBackup(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - err := model.UploadBackup() - if nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } -} - -func recoverLocalBackup(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - err := model.RecoverLocalBackup() - if nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } -} - -func createLocalBackup(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - err := model.CreateLocalBackup() - if nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } -} - -func getLocalBackup(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - backup, err := model.GetLocalBackup() - if nil != err { - ret.Code = -1 - ret.Msg = err.Error() - return - } - - ret.Data = map[string]interface{}{ - "backup": backup, - } -} - -func getCloudSpace(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - sync, backup, size, assetSize, totalSize, err := model.GetCloudSpace() - if nil != err { - ret.Code = 1 - ret.Msg = err.Error() - util.PushErrMsg(err.Error(), 3000) - return - } - - hTrafficUploadSize := humanize.Bytes(uint64(model.Conf.User.UserTrafficUpload)) - hTrafficDownloadSize := humanize.Bytes(uint64(model.Conf.User.UserTrafficDownload)) - - ret.Data = map[string]interface{}{ - "sync": sync, - "backup": backup, - "hAssetSize": assetSize, - "hSize": size, - "hTotalSize": totalSize, - "hTrafficUploadSize": hTrafficUploadSize, - "hTrafficDownloadSize": hTrafficDownloadSize, - } -} diff --git a/kernel/api/repo.go b/kernel/api/repo.go index 0d528da42..41bcf21ab 100644 --- a/kernel/api/repo.go +++ b/kernel/api/repo.go @@ -21,11 +21,38 @@ import ( "net/http" "github.com/88250/gulu" + "github.com/dustin/go-humanize" "github.com/gin-gonic/gin" "github.com/siyuan-note/siyuan/kernel/model" "github.com/siyuan-note/siyuan/kernel/util" ) +func getCloudSpace(c *gin.Context) { + ret := gulu.Ret.NewResult() + defer c.JSON(http.StatusOK, ret) + + sync, backup, size, assetSize, totalSize, err := model.GetCloudSpace() + if nil != err { + ret.Code = 1 + ret.Msg = err.Error() + util.PushErrMsg(err.Error(), 3000) + return + } + + hTrafficUploadSize := humanize.Bytes(uint64(model.Conf.User.UserTrafficUpload)) + hTrafficDownloadSize := humanize.Bytes(uint64(model.Conf.User.UserTrafficDownload)) + + ret.Data = map[string]interface{}{ + "sync": sync, + "backup": backup, + "hAssetSize": assetSize, + "hSize": size, + "hTotalSize": totalSize, + "hTrafficUploadSize": hTrafficUploadSize, + "hTrafficDownloadSize": hTrafficDownloadSize, + } +} + func checkoutRepo(c *gin.Context) { ret := gulu.Ret.NewResult() defer c.JSON(http.StatusOK, ret) diff --git a/kernel/api/router.go b/kernel/api/router.go index 78b533bd6..2ef8f34fa 100644 --- a/kernel/api/router.go +++ b/kernel/api/router.go @@ -158,13 +158,6 @@ func ServeAPI(ginServer *gin.Engine) { ginServer.Handle("POST", "/api/cloud/getCloudSpace", model.CheckAuth, getCloudSpace) - ginServer.Handle("POST", "/api/backup/getLocalBackup", model.CheckAuth, getLocalBackup) - ginServer.Handle("POST", "/api/backup/createLocalBackup", model.CheckAuth, model.CheckReadonly, createLocalBackup) - ginServer.Handle("POST", "/api/backup/recoverLocalBackup", model.CheckAuth, model.CheckReadonly, recoverLocalBackup) - ginServer.Handle("POST", "/api/backup/uploadLocalBackup", model.CheckAuth, model.CheckReadonly, uploadLocalBackup) - ginServer.Handle("POST", "/api/backup/downloadCloudBackup", model.CheckAuth, model.CheckReadonly, downloadCloudBackup) - ginServer.Handle("POST", "/api/backup/removeCloudBackup", model.CheckAuth, model.CheckReadonly, removeCloudBackup) - ginServer.Handle("POST", "/api/sync/setSyncEnable", model.CheckAuth, setSyncEnable) ginServer.Handle("POST", "/api/sync/setSyncMode", model.CheckAuth, setSyncMode) ginServer.Handle("POST", "/api/sync/setCloudSyncDir", model.CheckAuth, setCloudSyncDir) diff --git a/kernel/model/backup.go b/kernel/model/backup.go index fb4d08e3b..6022f3eeb 100644 --- a/kernel/model/backup.go +++ b/kernel/model/backup.go @@ -17,25 +17,10 @@ package model import ( - "bytes" - "crypto/md5" - "crypto/sha256" - "encoding/hex" "errors" "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" - "time" - "github.com/88250/gulu" "github.com/dustin/go-humanize" - "github.com/siyuan-note/encryption" - "github.com/siyuan-note/filelock" - "github.com/siyuan-note/siyuan/kernel/sql" - "github.com/siyuan-note/siyuan/kernel/util" ) type Backup struct { @@ -53,25 +38,6 @@ type Sync struct { SaveDir string `json:"saveDir"` // 本地同步数据存放目录路径 } -func RemoveCloudBackup() (err error) { - err = removeCloudDirPath("backup") - return -} - -func getCloudAvailableBackupSize() (size int64, err error) { - sync, _, assetSize, err := getCloudSpaceOSS() - if nil != err { - return - } - - var syncSize int64 - if nil != sync { - syncSize = int64(sync["size"].(float64)) - } - size = int64(Conf.User.UserSiYuanRepoSize) - syncSize - assetSize - return -} - func GetCloudSpace() (s *Sync, b *Backup, hSize, hAssetSize, hTotalSize string, err error) { sync, backup, assetSize, err := getCloudSpaceOSS() if nil != err { @@ -119,483 +85,3 @@ func byteCountSI(b int64) string { } return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) } - -func GetLocalBackup() (ret *Backup, err error) { - backupDir := Conf.Backup.GetSaveDir() - if err = os.MkdirAll(backupDir, 0755); nil != err { - return - } - - backup, err := os.Stat(backupDir) - ret = &Backup{ - Updated: backup.ModTime().Format("2006-01-02 15:04:05"), - SaveDir: Conf.Backup.GetSaveDir(), - } - return -} - -func RecoverLocalBackup() (err error) { - if "" == Conf.E2EEPasswd { - return errors.New(Conf.Language(11)) - } - - writingDataLock.Lock() - defer writingDataLock.Unlock() - - err = filelock.ReleaseAllFileLocks() - if nil != err { - return - } - sql.WaitForWritingDatabase() - - CloseWatchAssets() - defer WatchAssets() - - // 使用备份恢复时自动暂停同步，避免刚刚恢复后的数据又被同步覆盖 https://github.com/siyuan-note/siyuan/issues/4773 - syncEnabled := Conf.Sync.Enabled - Conf.Sync.Enabled = false - Conf.Save() - - util.PushEndlessProgress(Conf.Language(63)) - util.LogInfof("starting recovery...") - start := time.Now() - data := util.AESDecrypt(Conf.E2EEPasswd) - data, _ = hex.DecodeString(string(data)) - passwd := string(data) - decryptedDataDir, err := decryptDataDir(passwd) - if nil != err { - util.ClearPushProgress(100) - return - } - newDataDir := filepath.Join(util.WorkspaceDir, "data.new") - os.RemoveAll(newDataDir) - if err = os.MkdirAll(newDataDir, 0755); nil != err { - util.ClearPushProgress(100) - return - } - - if err = stableCopy(decryptedDataDir, newDataDir); nil != err { - util.ClearPushProgress(100) - return - } - - oldDataDir := filepath.Join(util.WorkspaceDir, "data.old") - if err = os.RemoveAll(oldDataDir); nil != err { - util.ClearPushProgress(100) - return - } - - // 备份恢复时生成历史 https://github.com/siyuan-note/siyuan/issues/4752 - if gulu.File.IsExist(util.DataDir) { - var historyDir string - historyDir, err = util.GetHistoryDir("backup") - if nil != err { - util.LogErrorf("get history dir failed: %s", err) - util.ClearPushProgress(100) - return - } - - var dirs []os.DirEntry - dirs, err = os.ReadDir(util.DataDir) - if nil != err { - util.LogErrorf("read dir [%s] failed: %s", util.DataDir, err) - util.ClearPushProgress(100) - return - } - for _, dir := range dirs { - from := filepath.Join(util.DataDir, dir.Name()) - to := filepath.Join(historyDir, dir.Name()) - if err = os.Rename(from, to); nil != err { - util.LogErrorf("rename [%s] to [%s] failed: %s", from, to, err) - util.ClearPushProgress(100) - return - } - } - } - - if gulu.File.IsExist(util.DataDir) { - if err = os.RemoveAll(util.DataDir); nil != err { - util.LogErrorf("remove [%s] failed: %s", util.DataDir, err) - util.ClearPushProgress(100) - return - } - } - - if err = os.Rename(newDataDir, util.DataDir); nil != err { - util.ClearPushProgress(100) - util.LogErrorf("rename data dir from [%s] to [%s] failed: %s", newDataDir, util.DataDir, err) - return - } - - elapsed := time.Now().Sub(start).Seconds() - size, _ := util.SizeOfDirectory(util.DataDir, false) - sizeStr := humanize.Bytes(uint64(size)) - util.LogInfof("recovered backup [size=%s] in [%.2fs]", sizeStr, elapsed) - - util.PushEndlessProgress(Conf.Language(62)) - time.Sleep(2 * time.Second) - RefreshFileTree() - if syncEnabled { - func() { - time.Sleep(5 * time.Second) - util.PushMsg(Conf.Language(134), 0) - }() - } - return -} - -func CreateLocalBackup() (err error) { - if "" == Conf.E2EEPasswd { - return errors.New(Conf.Language(11)) - } - - defer util.ClearPushProgress(100) - util.PushEndlessProgress(Conf.Language(22)) - - writingDataLock.Lock() - defer writingDataLock.Unlock() - WaitForWritingFiles() - sql.WaitForWritingDatabase() - err = filelock.ReleaseAllFileLocks() - if nil != err { - return - } - - util.LogInfof("creating backup...") - start := time.Now() - data := util.AESDecrypt(Conf.E2EEPasswd) - data, _ = hex.DecodeString(string(data)) - passwd := string(data) - encryptedDataDir, err := encryptDataDir(passwd) - if nil != err { - util.LogErrorf("encrypt data dir failed: %s", err) - err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err))) - return - } - - newBackupDir := Conf.Backup.GetSaveDir() + ".new" - os.RemoveAll(newBackupDir) - if err = os.MkdirAll(newBackupDir, 0755); nil != err { - err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err))) - return - } - - if err = stableCopy(encryptedDataDir, newBackupDir); nil != err { - util.LogErrorf("copy encrypted data dir from [%s] to [%s] failed: %s", encryptedDataDir, newBackupDir, err) - err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err))) - return - } - - _, err = genCloudIndex(newBackupDir, map[string]bool{}, true) - if nil != err { - return - } - - conf := map[string]interface{}{"updated": time.Now().UnixMilli()} - data, err = gulu.JSON.MarshalJSON(conf) - if nil != err { - util.LogErrorf("marshal backup conf.json failed: %s", err) - } else { - confPath := filepath.Join(newBackupDir, "conf.json") - if err = gulu.File.WriteFileSafer(confPath, data, 0644); nil != err { - util.LogErrorf("write backup conf.json [%s] failed: %s", confPath, err) - } - } - - oldBackupDir := Conf.Backup.GetSaveDir() + ".old" - os.RemoveAll(oldBackupDir) - - backupDir := Conf.Backup.GetSaveDir() - if gulu.File.IsExist(backupDir) { - if err = os.Rename(backupDir, oldBackupDir); nil != err { - util.LogErrorf("rename backup dir from [%s] to [%s] failed: %s", backupDir, oldBackupDir, err) - err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err))) - return - } - } - if err = os.Rename(newBackupDir, backupDir); nil != err { - util.LogErrorf("rename backup dir from [%s] to [%s] failed: %s", newBackupDir, backupDir, err) - err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err))) - return - } - os.RemoveAll(oldBackupDir) - elapsed := time.Now().Sub(start).Seconds() - size, _ := util.SizeOfDirectory(backupDir, false) - sizeStr := humanize.Bytes(uint64(size)) - util.LogInfof("created backup [size=%s] in [%.2fs]", sizeStr, elapsed) - - util.PushEndlessProgress(Conf.Language(21)) - time.Sleep(2 * time.Second) - return -} - -func DownloadBackup() (err error) { - // 使用路径映射文件进行解密验证 https://github.com/siyuan-note/siyuan/issues/3789 - var tmpFetchedFiles int - var tmpTransferSize uint64 - err = ossDownload0(util.TempDir+"/backup", "backup", "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, false) - if nil != err { - return - } - data, err := os.ReadFile(filepath.Join(util.TempDir, "/backup/"+pathJSON)) - if nil != err { - return - } - passwdData, _ := hex.DecodeString(string(util.AESDecrypt(Conf.E2EEPasswd))) - passwd := string(passwdData) - data, err = encryption.AESGCMDecryptBinBytes(data, passwd) - if nil != err { - err = errors.New(Conf.Language(28)) - return - } - - localDirPath := Conf.Backup.GetSaveDir() - util.PushEndlessProgress(Conf.Language(68)) - start := time.Now() - fetchedFilesCount, transferSize, _, err := ossDownload(localDirPath, "backup", false) - if nil == err { - elapsed := time.Now().Sub(start).Seconds() - util.LogInfof("downloaded backup [fetchedFiles=%d, transferSize=%s] in [%.2fs]", fetchedFilesCount, humanize.Bytes(transferSize), elapsed) - util.PushEndlessProgress(Conf.Language(69)) - } - return -} - -func UploadBackup() (err error) { - defer util.ClearPushProgress(100) - - if err = checkUploadBackup(); nil != err { - return - } - - localDirPath := Conf.Backup.GetSaveDir() - util.PushEndlessProgress(Conf.Language(61)) - util.LogInfof("uploading backup...") - start := time.Now() - wroteFiles, transferSize, err := ossUpload(true, localDirPath, "backup", "not exist", false) - if nil == err { - elapsed := time.Now().Sub(start).Seconds() - util.LogInfof("uploaded backup [wroteFiles=%d, transferSize=%s] in [%.2fs]", wroteFiles, humanize.Bytes(transferSize), elapsed) - util.PushEndlessProgress(Conf.Language(41)) - time.Sleep(2 * time.Second) - return - } - err = errors.New(formatErrorMsg(err)) - return -} - -var pathJSON = fmt.Sprintf("%x", md5.Sum([]byte("paths.json"))) // 6952277a5a37c17aa6a7c6d86cd507b1 - -func encryptDataDir(passwd string) (encryptedDataDir string, err error) { - encryptedDataDir = filepath.Join(util.TempDir, "incremental", "backup-encrypt") - if err = os.RemoveAll(encryptedDataDir); nil != err { - return - } - if err = os.MkdirAll(encryptedDataDir, 0755); nil != err { - return - } - - ctime := map[string]time.Time{} - metaJSON := map[string]string{} - filepath.Walk(util.DataDir, func(path string, info fs.FileInfo, _ error) error { - if util.DataDir == path { - return nil - } - - if isCloudSkipFile(path, info) { - if info.IsDir() { - return filepath.SkipDir - } - return nil - } - - plainP := strings.TrimPrefix(path, util.DataDir+string(os.PathSeparator)) - p := plainP - parts := strings.Split(p, string(os.PathSeparator)) - buf := bytes.Buffer{} - for i, part := range parts { - buf.WriteString(fmt.Sprintf("%x", sha256.Sum256([]byte(part)))[:7]) - if i < len(parts)-1 { - buf.WriteString(string(os.PathSeparator)) - } - } - p = buf.String() - metaJSON[filepath.ToSlash(p)] = filepath.ToSlash(plainP) - p = encryptedDataDir + string(os.PathSeparator) + p - - if info.IsDir() { - if err = os.MkdirAll(p, 0755); nil != err { - return io.EOF - } - if fi, err0 := os.Stat(path); nil == err0 { - ctime[p] = fi.ModTime() - } - } else { - if err = os.MkdirAll(filepath.Dir(p), 0755); nil != err { - return io.EOF - } - - data, err0 := filelock.NoLockFileRead(path) - if nil != err0 { - util.LogErrorf("read file [%s] failed: %s", path, err0) - err = err0 - return io.EOF - } - data, err0 = encryption.AESGCMEncryptBinBytes(data, passwd) - if nil != err0 { - util.LogErrorf("encrypt file [%s] failed: %s", path, err0) - err = errors.New("encrypt file failed") - return io.EOF - } - - if err0 = gulu.File.WriteFileSafer(p, data, 0644); nil != err0 { - util.LogErrorf("write file [%s] failed: %s", p, err0) - err = err0 - return io.EOF - } - - fi, err0 := os.Stat(path) - if nil != err0 { - util.LogErrorf("stat file [%s] failed: %s", path, err0) - err = err0 - return io.EOF - } - ctime[p] = fi.ModTime() - } - return nil - }) - if nil != err { - return - } - - for p, t := range ctime { - if err = os.Chtimes(p, t, t); nil != err { - return - } - } - - // 检查文件是否全部已经编入索引 - err = filepath.Walk(encryptedDataDir, func(path string, info fs.FileInfo, _ error) error { - if encryptedDataDir == path { - return nil - } - - path = strings.TrimPrefix(path, encryptedDataDir+string(os.PathSeparator)) - path = filepath.ToSlash(path) - if _, ok := metaJSON[path]; !ok { - util.LogErrorf("not found backup path in meta [%s]", path) - return errors.New(Conf.Language(27)) - } - return nil - }) - - if nil != err { - return - } - - data, err := gulu.JSON.MarshalJSON(metaJSON) - if nil != err { - return - } - data, err = encryption.AESGCMEncryptBinBytes(data, passwd) - if nil != err { - return "", errors.New("encrypt file failed") - } - meta := filepath.Join(encryptedDataDir, pathJSON) - if err = gulu.File.WriteFileSafer(meta, data, 0644); nil != err { - return - } - return -} - -func decryptDataDir(passwd string) (decryptedDataDir string, err error) { - decryptedDataDir = filepath.Join(util.TempDir, "incremental", "backup-decrypt") - if err = os.RemoveAll(decryptedDataDir); nil != err { - return - } - - backupDir := Conf.Backup.GetSaveDir() - meta := filepath.Join(util.TempDir, "backup", pathJSON) - data, err := os.ReadFile(meta) - if nil != err { - return - } - data, err = encryption.AESGCMDecryptBinBytes(data, passwd) - if nil != err { - return "", errors.New(Conf.Language(40)) - } - metaJSON := map[string]string{} - if err = gulu.JSON.UnmarshalJSON(data, &metaJSON); nil != err { - return - } - - index := map[string]*CloudIndex{} - data, err = os.ReadFile(filepath.Join(backupDir, "index.json")) - if nil != err { - return - } - if err = gulu.JSON.UnmarshalJSON(data, &index); nil != err { - return - } - - err = filepath.Walk(backupDir, func(path string, info fs.FileInfo, _ error) error { - if backupDir == path || pathJSON == info.Name() || strings.HasSuffix(info.Name(), ".json") { - return nil - } - - encryptedP := strings.TrimPrefix(path, backupDir+string(os.PathSeparator)) - encryptedP = filepath.ToSlash(encryptedP) - decryptedP := metaJSON[encryptedP] - if "" == decryptedP { - if gulu.File.IsDir(path) { - return filepath.SkipDir - } - return nil - } - plainP := filepath.Join(decryptedDataDir, decryptedP) - plainP = filepath.FromSlash(plainP) - - if info.IsDir() { - if err = os.MkdirAll(plainP, 0755); nil != err { - return io.EOF - } - } else { - if err = os.MkdirAll(filepath.Dir(plainP), 0755); nil != err { - return io.EOF - } - - var err0 error - data, err0 = os.ReadFile(path) - if nil != err0 { - util.LogErrorf("read file [%s] failed: %s", path, err0) - err = err0 - return io.EOF - } - data, err0 = encryption.AESGCMDecryptBinBytes(data, passwd) - if nil != err0 { - util.LogErrorf("decrypt file [%s] failed: %s", path, err0) - err = errors.New(Conf.Language(40)) - return io.EOF - } - if err0 = gulu.File.WriteFileSafer(plainP, data, 0644); nil != err0 { - util.LogErrorf("write file [%s] failed: %s", plainP, err0) - err = err0 - return io.EOF - } - - var modTime int64 - idx := index["/"+encryptedP] - if nil == idx { - util.LogErrorf("index file [%s] not found", encryptedP) - modTime = info.ModTime().Unix() - } else { - modTime = idx.Updated - } - if err0 = os.Chtimes(plainP, time.Unix(modTime, 0), time.Unix(modTime, 0)); nil != err0 { - util.LogErrorf("change file [%s] time failed: %s", plainP, err0) - } - } - return nil - }) - return -} diff --git a/kernel/model/box.go b/kernel/model/box.go index c4c7a94bd..329eb9dc9 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -349,30 +349,6 @@ func isSkipFile(filename string) bool { return strings.HasPrefix(filename, ".") || "node_modules" == filename || "dist" == filename || "target" == filename } -func checkUploadBackup() (err error) { - if !IsSubscriber() { - if "ios" == util.Container { - return errors.New(Conf.Language(122)) - } - return errors.New(Conf.Language(29)) - } - - backupDir := Conf.Backup.GetSaveDir() - backupSize, err := util.SizeOfDirectory(backupDir, false) - if nil != err { - return - } - - cloudAvailableBackupSize, err := getCloudAvailableBackupSize() - if nil != err { - return - } - if cloudAvailableBackupSize < backupSize { - return errors.New(fmt.Sprintf(Conf.Language(43), byteCountSI(int64(Conf.User.UserSiYuanRepoSize)))) - } - return nil -} - func (box *Box) renameSubTrees(tree *parse.Tree) { subFiles := box.ListFiles(tree.Path) totals := len(subFiles) + 3 diff --git a/kernel/model/export.go b/kernel/model/export.go index df5c2f09e..5d9abd59e 100644 --- a/kernel/model/export.go +++ b/kernel/model/export.go @@ -120,7 +120,7 @@ func exportData(exportFolder string) (err error) { data := filepath.Join(util.WorkspaceDir, "data") if err = stableCopy(data, exportFolder); nil != err { util.LogErrorf("copy data dir from [%s] to [%s] failed: %s", data, baseFolderName, err) - err = errors.New(fmt.Sprintf(Conf.Language(23), formatErrorMsg(err))) + err = errors.New(fmt.Sprintf(Conf.Language(14), formatErrorMsg(err))) return } From e936b9cf0b26538bb778a54e20911642ec173e45 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 13 Jul 2022 20:39:10 +0800 Subject: [PATCH 6/8] =?UTF-8?q?:fire:=20=E7=A7=BB=E9=99=A4=E6=97=A7?= =?UTF-8?q?=E7=89=88=E4=BA=91=E7=AB=AF=E5=90=8C=E6=AD=A5=E5=92=8C=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8A=9F=E8=83=BD=E5=85=A5=E5=8F=A3=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/5405?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/router.go | 1 - kernel/api/sync.go | 13 ------ kernel/model/backup.go | 87 -------------------------------------- kernel/model/repository.go | 59 ++++++++++++++++++++++++-- kernel/model/sync.go | 59 -------------------------- 5 files changed, 55 insertions(+), 164 deletions(-) delete mode 100644 kernel/model/backup.go diff --git a/kernel/api/router.go b/kernel/api/router.go index 2ef8f34fa..8c9a32ef9 100644 --- a/kernel/api/router.go +++ b/kernel/api/router.go @@ -167,7 +167,6 @@ func ServeAPI(ginServer *gin.Engine) { ginServer.Handle("POST", "/api/sync/performSync", model.CheckAuth, performSync) ginServer.Handle("POST", "/api/sync/performBootSync", model.CheckAuth, performBootSync) ginServer.Handle("POST", "/api/sync/getBootSync", model.CheckAuth, getBootSync) - ginServer.Handle("POST", "/api/sync/getSyncDirection", model.CheckAuth, getSyncDirection) ginServer.Handle("POST", "/api/inbox/getShorthands", model.CheckAuth, getShorthands) ginServer.Handle("POST", "/api/inbox/removeShorthands", model.CheckAuth, removeShorthands) diff --git a/kernel/api/sync.go b/kernel/api/sync.go index 7dbf7d070..aedb50a81 100644 --- a/kernel/api/sync.go +++ b/kernel/api/sync.go @@ -25,19 +25,6 @@ import ( "github.com/siyuan-note/siyuan/kernel/util" ) -func getSyncDirection(c *gin.Context) { - ret := gulu.Ret.NewResult() - defer c.JSON(http.StatusOK, ret) - - arg, ok := util.JsonArg(c, ret) - if !ok { - return - } - - cloudDirName := arg["name"].(string) - ret.Code, ret.Msg = model.GetSyncDirection(cloudDirName) -} - func getBootSync(c *gin.Context) { ret := gulu.Ret.NewResult() defer c.JSON(http.StatusOK, ret) diff --git a/kernel/model/backup.go b/kernel/model/backup.go deleted file mode 100644 index 6022f3eeb..000000000 --- a/kernel/model/backup.go +++ /dev/null @@ -1,87 +0,0 @@ -// SiYuan - Build Your Eternal Digital Garden -// Copyright (c) 2020-present, b3log.org -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package model - -import ( - "errors" - "fmt" - - "github.com/dustin/go-humanize" -) - -type Backup struct { - Size int64 `json:"size"` - HSize string `json:"hSize"` - Updated string `json:"updated"` - SaveDir string `json:"saveDir"` // 本地备份数据存放目录路径 -} - -type Sync struct { - Size int64 `json:"size"` - HSize string `json:"hSize"` - Updated string `json:"updated"` - CloudName string `json:"cloudName"` // 云端同步数据存放目录名 - SaveDir string `json:"saveDir"` // 本地同步数据存放目录路径 -} - -func GetCloudSpace() (s *Sync, b *Backup, hSize, hAssetSize, hTotalSize string, err error) { - sync, backup, assetSize, err := getCloudSpaceOSS() - if nil != err { - err = errors.New(Conf.Language(30) + " " + err.Error()) - return - } - - var totalSize, syncSize, backupSize int64 - var syncUpdated, backupUpdated string - if nil != sync { - syncSize = int64(sync["size"].(float64)) - syncUpdated = sync["updated"].(string) - } - s = &Sync{ - Size: syncSize, - HSize: humanize.Bytes(uint64(syncSize)), - Updated: syncUpdated, - } - - if nil != backup { - backupSize = int64(backup["size"].(float64)) - backupUpdated = backup["updated"].(string) - } - b = &Backup{ - Size: backupSize, - HSize: humanize.Bytes(uint64(backupSize)), - Updated: backupUpdated, - } - totalSize = syncSize + backupSize + assetSize - hAssetSize = humanize.Bytes(uint64(assetSize)) - hSize = humanize.Bytes(uint64(totalSize)) - hTotalSize = byteCountSI(int64(Conf.User.UserSiYuanRepoSize)) - return -} - -func byteCountSI(b int64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) - } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) -} diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 5565e8e1a..9db36b5d8 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -27,6 +27,7 @@ import ( "time" "github.com/88250/gulu" + "github.com/dustin/go-humanize" "github.com/siyuan-note/dejavu" "github.com/siyuan-note/dejavu/entity" "github.com/siyuan-note/encryption" @@ -221,7 +222,7 @@ func DownloadCloudSnapshot(tag, id string) (err error) { if nil != err { return } - msg := fmt.Sprintf(Conf.Language(153), downloadFileCount, downloadChunkCount, byteCountSI(downloadBytes)) + msg := fmt.Sprintf(Conf.Language(153), downloadFileCount, downloadChunkCount, humanize.Bytes(uint64(downloadBytes))) util.PushMsg(msg, 5000) util.PushStatusBar(msg) return @@ -252,7 +253,7 @@ func UploadCloudSnapshot(tag, id string) (err error) { } return } - msg := fmt.Sprintf(Conf.Language(152), uploadFileCount, uploadChunkCount, byteCountSI(uploadBytes)) + msg := fmt.Sprintf(Conf.Language(152), uploadFileCount, uploadChunkCount, humanize.Bytes(uint64(uploadBytes))) util.PushMsg(msg, 5000) util.PushStatusBar(msg) return @@ -458,7 +459,7 @@ func syncRepo(boot, exit, byHand bool) { util.LogErrorf("sync data repo failed: %s", err) msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err)) if errors.Is(err, dejavu.ErrCloudStorageSizeExceeded) { - msg = fmt.Sprintf(Conf.Language(43), byteCountSI(int64(Conf.User.UserSiYuanRepoSize))) + msg = fmt.Sprintf(Conf.Language(43), humanize.Bytes(uint64(Conf.User.UserSiYuanRepoSize))) } Conf.Sync.Stat = msg util.PushStatusBar(msg) @@ -474,7 +475,7 @@ func syncRepo(boot, exit, byHand bool) { } util.PushStatusBar(fmt.Sprintf(Conf.Language(149), elapsed.Seconds())) Conf.Sync.Synced = util.CurrentTimeMillis() - msg := fmt.Sprintf(Conf.Language(150), trafficStat.UploadFileCount, trafficStat.DownloadFileCount, trafficStat.UploadChunkCount, trafficStat.DownloadChunkCount, byteCountSI(trafficStat.UploadBytes), byteCountSI(trafficStat.DownloadBytes)) + msg := fmt.Sprintf(Conf.Language(150), trafficStat.UploadFileCount, trafficStat.DownloadFileCount, trafficStat.UploadChunkCount, trafficStat.DownloadChunkCount, humanize.Bytes(uint64(trafficStat.UploadBytes)), humanize.Bytes(uint64(trafficStat.DownloadBytes))) Conf.Sync.Stat = msg if 1 > len(mergeResult.Upserts) && 1 > len(mergeResult.Removes) { // 没有数据变更 @@ -740,6 +741,56 @@ func buildCloudInfo() (ret *dejavu.CloudInfo, err error) { return } +type Backup struct { + Size int64 `json:"size"` + HSize string `json:"hSize"` + Updated string `json:"updated"` + SaveDir string `json:"saveDir"` // 本地备份数据存放目录路径 +} + +type Sync struct { + Size int64 `json:"size"` + HSize string `json:"hSize"` + Updated string `json:"updated"` + CloudName string `json:"cloudName"` // 云端同步数据存放目录名 + SaveDir string `json:"saveDir"` // 本地同步数据存放目录路径 +} + +func GetCloudSpace() (s *Sync, b *Backup, hSize, hAssetSize, hTotalSize string, err error) { + sync, backup, assetSize, err := getCloudSpaceOSS() + if nil != err { + err = errors.New(Conf.Language(30) + " " + err.Error()) + return + } + + var totalSize, syncSize, backupSize int64 + var syncUpdated, backupUpdated string + if nil != sync { + syncSize = int64(sync["size"].(float64)) + syncUpdated = sync["updated"].(string) + } + s = &Sync{ + Size: syncSize, + HSize: humanize.Bytes(uint64(syncSize)), + Updated: syncUpdated, + } + + if nil != backup { + backupSize = int64(backup["size"].(float64)) + backupUpdated = backup["updated"].(string) + } + b = &Backup{ + Size: backupSize, + HSize: humanize.Bytes(uint64(backupSize)), + Updated: backupUpdated, + } + totalSize = syncSize + backupSize + assetSize + hAssetSize = humanize.Bytes(uint64(assetSize)) + hSize = humanize.Bytes(uint64(totalSize)) + hTotalSize = humanize.Bytes(uint64(Conf.User.UserSiYuanRepoSize)) + return +} + func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, err error) { result := map[string]interface{}{} resp, err := httpclient.NewCloudRequest(). diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 9edcd8963..26ad4626c 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -30,7 +30,6 @@ import ( "github.com/88250/gulu" "github.com/dustin/go-humanize" "github.com/siyuan-note/dejavu" - "github.com/siyuan-note/filelock" "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/sql" "github.com/siyuan-note/siyuan/kernel/treenode" @@ -211,47 +210,6 @@ func SetSyncMode(mode int) (err error) { var syncLock = sync.Mutex{} -func getWorkspaceDataConf() (conf *filesys.DataConf, err error) { - conf = &filesys.DataConf{Updated: util.CurrentTimeMillis(), Device: Conf.System.ID} - confPath := filepath.Join(Conf.Sync.GetSaveDir(), ".siyuan", "conf.json") - if !gulu.File.IsExist(confPath) { - os.MkdirAll(filepath.Dir(confPath), 0755) - data, _ := gulu.JSON.MarshalIndentJSON(conf, "", " ") - if err = filelock.NoLockFileWrite(confPath, data); nil != err { - util.LogErrorf("save sync conf [%s] failed: %s", confPath, err) - } - return - } - - data, err := filelock.NoLockFileRead(confPath) - if nil != err { - util.LogErrorf("read sync conf [%s] failed: %s", confPath, err) - return - } - if err = gulu.JSON.UnmarshalJSON(data, conf); nil != err { - filesys.IncWorkspaceDataVer(false, Conf.System.ID) // 尝试恢复 data/.siyuan/conf.json - util.LogErrorf("unmarshal sync conf [%s] failed: %s", confPath, err) - err = errors.New(Conf.Language(84)) - return - } - return -} - -func incLocalSyncVer() { - conf, err := getWorkspaceDataConf() - if nil != err { - return - } - - conf.SyncVer++ - data, _ := gulu.JSON.MarshalIndentJSON(conf, "", " ") - confPath := filepath.Join(Conf.Sync.GetSaveDir(), ".siyuan", "conf.json") - if err = gulu.File.WriteFileSafer(confPath, data, 0644); nil != err { - util.LogErrorf("save sync conf [%s] failed: %s", confPath, err) - } - return -} - func CreateCloudSyncDir(name string) (err error) { syncLock.Lock() defer syncLock.Unlock() @@ -389,23 +347,6 @@ func getIgnoreLines() (ret []string) { return } -func GetSyncDirection(cloudDirName string) (code int, msg string) { // 0：失败，10：上传，20：下载，30：一致，40：使用数据仓库同步 - if !IsSubscriber() { - return - } - - if "" == cloudDirName { - return - } - - if !IsValidCloudDirName(cloudDirName) { - return - } - - // TODO: 彻底移除方向判断 - return 40, "" -} - func IncWorkspaceDataVer() { filesys.IncWorkspaceDataVer(true, Conf.System.ID) syncSameCount = 0 From 816b5e842f35883ecd90e9e9e8ce75f82db3dcd8 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 13 Jul 2022 20:48:26 +0800 Subject: [PATCH 7/8] :art: Search ignore case supports Unicode character folding Fix https://github.com/siyuan-note/siyuan/issues/5398 --- kernel/go.mod | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 62685944b..e554e9771 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -38,9 +38,7 @@ require ( github.com/mssola/user_agent v0.5.3 github.com/panjf2000/ants/v2 v2.5.0 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/qiniu/go-sdk/v7 v7.13.0 github.com/radovskyb/watcher v1.0.7 - github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/siyuan-note/dejavu v0.0.0-20220711060744-3fec84096399 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f @@ -93,7 +91,9 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/qiniu/go-sdk/v7 v7.13.0 // indirect github.com/restic/chunker v0.4.0 // indirect + github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/ugorji/go/codec v1.2.7 // indirect @@ -111,7 +111,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect ) -replace github.com/mattn/go-sqlite3 => github.com/88250/go-sqlite3 v1.14.13-0.20220713041934-e2e29c01be36 +replace github.com/mattn/go-sqlite3 => github.com/88250/go-sqlite3 v1.14.13-0.20220713124603-951e48d11239 //replace github.com/siyuan-note/dejavu => D:\88250\dejavu //replace github.com/siyuan-note/httpclient => D:\88250\httpclient From f665b9aa252fdebb37c387ce55bd4957e36521f5 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 13 Jul 2022 20:48:33 +0800 Subject: [PATCH 8/8] :art: Search ignore case supports Unicode character folding Fix https://github.com/siyuan-note/siyuan/issues/5398 --- kernel/go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/go.sum b/kernel/go.sum index dd8e83865..5dab35e34 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -45,8 +45,8 @@ github.com/88250/css v0.1.2 h1:+AADhEwWoGZFbUjqIsBcdnq2xfj8fDFDAGRXhBUhUY8= github.com/88250/css v0.1.2/go.mod h1:XfcZHQ0YuUb9VncVBurQfVyw1ZQicsB5Gc9N7BK3/ig= github.com/88250/flock v0.8.2 h1:LLbRJw3hoYfjD4g7DiYsYcTCCFTxm8icn/WepLlxIg0= github.com/88250/flock v0.8.2/go.mod h1:k+PZxETAUe4vLZx3R39ykvQCIlwHhc7AI2P2NUQV6zw= -github.com/88250/go-sqlite3 v1.14.13-0.20220713041934-e2e29c01be36 h1:bFyBW46RX8+vXvMh8EzySRRjysSYuAiJ1A4kKCobA9I= -github.com/88250/go-sqlite3 v1.14.13-0.20220713041934-e2e29c01be36/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/88250/go-sqlite3 v1.14.13-0.20220713124603-951e48d11239 h1:x/Nrh630VbTneCf5AM4mA/1pB4e+lpTJXOoJeBsm+C0= +github.com/88250/go-sqlite3 v1.14.13-0.20220713124603-951e48d11239/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/88250/gulu v1.2.0/go.mod h1:ZhEJ98UjR2y7j2toGj4/b+1rRELcZFQAPq/Yjyin2yY= github.com/88250/gulu v1.2.3-0.20220623112232-c502d9016360 h1:afQ0cjIA/tzwvIDFy9Jf0jFCb1FvWwKuG1QidEMMi4M= github.com/88250/gulu v1.2.3-0.20220623112232-c502d9016360/go.mod h1:I1qBzsksFL2ciGSuqDE7R3XW4BUMrfDgOvSXEk7FsAI=