diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json
index c83b71dc5..2f3613494 100644
--- a/app/appearance/langs/en_US.json
+++ b/app/appearance/langs/en_US.json
@@ -777,11 +777,11 @@
"21": "Backup completed",
"22": "Backuping, please wait...",
"23": "Backup failed: %s",
- "24": "Failed to obtain cloud sync info: %s",
+ "24": "TODO",
"25": "The attribute name only supports English letters and digits",
"26": "Please initialize the data repo key first in [Settings - About - Data repo key]",
"27": "Data integrity check failed",
- "28": "Incorrect end-to-end encryption password, unable to decrypt data",
+ "28": "TODO",
"29": "This feature requires paid subscription (If you have subscribed, please refresh or log in again in settings - account)",
"30": "Failed to obtain cloud backup info",
"31": "Account authentication failed, please login again",
@@ -793,7 +793,7 @@
"37": "Do not include spaces and special symbols in the name of the cloud sync directory",
"38": "The number of mentioned keywords [%d] is too many, currently only supports up to [512] keywords",
"39": "TODO",
- "40": "Failed to decrypt data",
+ "40": "TODO",
"41": "Upload completed",
"42": "The setting is complete, the application will be closed automatically, please restart later...",
"43": "The maximum storage capacity of cloud space [%s] has been exceeded, and data upload cannot continue",
@@ -846,7 +846,7 @@
"90": "TODO",
"91": "TODO",
"92": "TODO",
- "93": "Download failed: %s",
+ "93": "TODO",
"94": "Upload failed: %s",
"95": "Exiting...",
"96": "Synchronization failed when exiting. Please manually perform a synchronization to ensure that the local data is consistent with the cloud data",
@@ -856,10 +856,10 @@
"100": "Cleaning data...",
"101": "Done setting reminder [%s]",
"102": "TODO",
- "103": "[%d] data files have been downloaded, and [%d] remaining to be downloaded",
+ "103": "TODO",
"104": "[%d] data files have been uploaded, and [%d] remaining to be uploaded",
"105": "Network transmission completed",
- "106": "Data download has been completed and decryption is in progress...",
+ "106": "TODO",
"107": "Moving document [%s]",
"108": "Cleaning obsolete indexes...",
"109": "Remove reminder completed [%s]",
@@ -882,13 +882,13 @@
"126": "Bookmark cannot be empty",
"127": "There are [%d] days left before the subscription expires, after which the cloud data will be completely deleted. Please visit Here, if you don't need to renew, please log out of your account to close the reminder",
"128": "Subscription has expired, cloud data will be completely deleted after expiration. To renew, please visit here , if you don't need to renew, please log out of your account to close the reminder",
- "129": "Number of files transferred %d\nTotal bytes received %s\n",
- "130": "Number of files transferred %d\nTotal bytes sent %s\n",
- "131": "Downloaded in %.2fs",
- "132": "Uploaded in %.2fs",
- "133": "No changes to local data",
+ "129": "TODO",
+ "130": "TODO",
+ "131": "TODO",
+ "132": "TODO",
+ "133": "TODO",
"134": "In order to prevent the newly restored data from being overwritten by synchronization, the data synchronization function has been automatically suspended",
- "135": "Please make sure that all devices have been updated to the latest version, and then trigger synchronization after randomly changing a document on the main device, and finally trigger synchronization on other devices",
+ "135": "TODO",
"136": "Initializing data repository key...",
"137": "Failed to initialize data repository key: %s",
"138": "Data repository key is set",
diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json
index 73248c04d..68dfdccf0 100644
--- a/app/appearance/langs/es_ES.json
+++ b/app/appearance/langs/es_ES.json
@@ -777,11 +777,11 @@
"21": "Copia de seguridad completada",
"22": "Haciendo copia de seguridad, por favor espere...",
"23": "Copia de seguridad fallida: %s",
- "24": "Fallo en la obtención de la información de sincronización con la nube: %s",
+ "24": "TODO",
"25": "El nombre del atributo sólo admite letras y dígitos en inglés",
"26": "Por favor, inicialice primero la clave de repositorio de datos en [Configuración - Acerca de - Clave de repositorio de datos]",
"27": "Falló la comprobación de la integridad de los datos",
- "28": "Contraseña de cifrado de extremo a extremo incorrecta, no se pueden descifrar los datos",
+ "28": "TODO",
"29": "Esta función requiere una suscripción de pago (Si se ha suscrito, actualice o vuelva a conectarse en configuración - cuenta)",
"30": "Fallo en la obtención de la información de la copia de seguridad en la nube",
"31": "Falló la autentificación de la cuenta, por favor, inicie sesión de nuevo",
@@ -793,7 +793,7 @@
"37": "No incluyas espacios ni símbolos especiales en el nombre del directorio de sincronización con la nube",
"38": "El número de palabras clave mencionadas [%d] son demasiados, actualmente solo admite hasta [512] palabras clave",
"39": "TODO",
- "40": "Fallo en la desencriptación de datos",
+ "40": "TODO",
"41": "Carga completada",
"42": "La configuración se ha completado, la aplicación se cerrará automáticamente, por favor reinicie más tarde...",
"43": "Se ha superado la capacidad máxima de almacenamiento del espacio en la nube [%s] y la carga de datos no puede continuar",
@@ -846,7 +846,7 @@
"90": "TODO",
"91": "TODO",
"92": "TODO",
- "93": "Descarga fallida: %s",
+ "93": "TODO",
"94": "Carga fallida: %s",
"95": "Saliendo...",
"96": "La sincronización falló al salir. Por favor, realice manualmente una sincronización para asegurarse de que los datos locales son coherentes con los datos de la nube",
@@ -856,10 +856,10 @@
"100": "Limpieza de datos...",
"101": "El recordatorio de configuración [%s] se ha completado",
"102": "TODO",
- "103": "[%d] archivos de datos han sido descargados, y [%d] quedan por descargar",
+ "103": "TODO",
"104": "[%d] archivos de datos han sido cargados, y [%d] restantes por cargar",
"105": "Transmisión de red completada",
- "106": "La descarga de datos se ha completado y la desencriptación está en curso...",
+ "106": "TODO",
"107": "Moviendo documento [%s]",
"108": "Limpiando índices obsoletos...",
"109": "Eliminación de recordatorios completada [%s]",
@@ -882,13 +882,13 @@
"126": "El marcador no puede estar vacío",
"127": "There are [%d] days left before the subscription expires, after which the cloud data will be completely deleted. Please visit Aquí para la renovación, si no necesita renovar, salga de su cuenta para cerrar el recordatorio",
"128": "La suscripción ha caducado, los datos de la nube se eliminarán completamente después de la expiración. Para renovar, visite Aquí, si no necesita renovar, salga de su cuenta para cerrar el recordatorio",
- "129": "Número de archivos transferidos %d\nTotal de bytes recibidos %s\n",
- "130": "Número de archivos transferidos %d\nTotal de bytes enviados %s\n",
- "131": "Descargado en %.2fs",
- "132": "Cargado en %.2fs",
- "133": "No hay cambios en los datos locales",
+ "129": "TODO",
+ "130": "TODO",
+ "131": "TODO",
+ "132": "TODO",
+ "133": "TODO",
"134": "Para evitar que los datos recién restaurados sean sobrescritos por la sincronización, se ha suspendido automáticamente la función de sincronización de datos",
- "135": "Por favor, asegúrese de que todos los dispositivos han sido actualizados a la última versión y, a continuación, active la sincronización después de cambiar aleatoriamente un documento en el dispositivo principal y, finalmente, active la sincronización en otros dispositivos",
+ "135": "TODO",
"136": "Inicializando la clave del repositorio de datos...",
"137": "Fallo en la inicialización de la clave del repositorio de datos: %s",
"138": "La clave del repositorio de datos está configurada",
diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json
index 15d140172..56a0ca5e4 100644
--- a/app/appearance/langs/fr_FR.json
+++ b/app/appearance/langs/fr_FR.json
@@ -777,11 +777,11 @@
"21": "Sauvegarde terminée",
"22": "En cours de sauvegarde, veuillez patienter....",
"23": "La sauvegarde a échoué : %s",
- "24": "Impossible d'obtenir les informations de synchronisation du Cloud : %s",
+ "24": "TODO",
"25": "Le nom de l'attribut ne supporte que les lettres et les chiffres anglais.",
"26": "Veuillez d'abord initialiser la clé du référentiel de données dans [Paramètres - À propos - Clé du référentiel de données]",
"27": "La vérification de l'intégrité des données a échoué",
- "28": "Mot de passe de cryptage de bout en bout incorrect, impossible de décrypter les données",
+ "28": "TODO",
"29": "Cette fonctionnalité nécessite un abonnement payant (Si vous êtes déjà abonné, Rafraîchissez ou connectez - vous à nouveau dans Paramètres - compte)",
"30": "Impossible d'obtenir des informations sur la sauvegarde dans le Cloud.",
"31": "L'authentification du compte a échoué, veuillez vous reconnecter",
@@ -793,7 +793,7 @@
"37": "N'incluez pas d'espaces et de symboles spéciaux dans le nom du répertoire de synchronisation cloud",
"38": "Le nombre de mots-clés mentionnés [%d] est trop élevé, ne prend actuellement en charge que jusqu'à [512] mots-clés",
"39": "TODO",
- "40": "Échec du décryptage des données",
+ "40": "TODO",
"41": "Transfert complété",
"42": "Le paramétrage est terminé, l'application se fermera automatiquement, merci de redémarrer plus tard...",
"43": "La capacité de stockage maximale de l'espace cloud [%s] a été dépassée et le téléchargement des données ne peut pas continuer",
@@ -846,7 +846,7 @@
"90": "TODO",
"91": "TODO",
"92": "TODO",
- "93": "Le téléchargement a échoué : %s",
+ "93": "TODO",
"94": "Échec du téléchargement : %s",
"95": "Quitter le programme...",
"96": "La synchronisation a échoué lors de la sortie. Veuillez effectuer une synchronisation manuellement pour vous assurer que les données locales sont cohérentes avec les données du cloud",
@@ -856,10 +856,10 @@
"100": "Nettoyage des données...",
"101": "Rappel de réglage terminé [%s]",
"102": "TODO",
- "103": "[%d] fichiers de données ont été téléchargés, il reste à télécharger [%d]",
+ "103": "TODO",
"104": "[%d] fichiers de données ont été téléchargés, et [%d] reste à télécharger",
"105": "Transmission réseau terminée",
- "106": "Le téléchargement des données est terminé et le décryptage est en cours...",
+ "106": "TODO",
"107": "Déplacement du document [%s]",
"108": "Nettoyage des index obsolètes...",
"109": "Supprimer le rappel terminé [%s]",
@@ -882,13 +882,13 @@
"126": "Les signets ne peuvent pas être vides",
"127": "Il reste [%d] jours avant l'expiration de l'abonnement, après quoi les données cloud seront complètement supprimées. Veuillez visiter ici, si vous n'avez pas besoin de renouveler, veuillez vous déconnecter de votre compte pour fermer le rappel",
"128": "L'abonnement a expiré, les données cloud seront complètement supprimées après l'expiration. Pour renouveler, veuillez visiter ici , si vous n'avez pas besoin de renouveler, veuillez vous déconnecter de votre compte pour fermer le rappel",
- "129": "Fichier transféré %d\noctets reçus %s\n",
- "130": "Fichier transféré %d\noctets envoyés %s\n",
- "131": "Temps de téléchargement %.2fs",
- "132": "Le téléchargement a pris %.2fs",
- "133": "Aucune modification des données locales",
+ "129": "TODO",
+ "130": "TODO",
+ "131": "TODO",
+ "132": "TODO",
+ "133": "TODO",
"134": "Afin d'éviter que les données nouvellement restaurées ne soient écrasées par la synchronisation, la fonction de synchronisation des données a été automatiquement suspendue",
- "135": "Assurez-vous que tous les appareils ont été mis à jour vers la dernière version, puis déclenchez la synchronisation après avoir modifié de manière aléatoire un document sur l'appareil principal, et enfin déclenchez la synchronisation sur d'autres appareils.",
+ "135": "TODO",
"136": "Initialisation de la clé du référentiel de données...",
"137": "Échec de l'initialisation de la clé du référentiel de données: %s",
"138": "La clé du référentiel de données est définie",
diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json
index 1c30b195f..b7d028e54 100644
--- a/app/appearance/langs/zh_CHT.json
+++ b/app/appearance/langs/zh_CHT.json
@@ -777,11 +777,11 @@
"21": "備份完畢",
"22": "正在備份，請稍等...",
"23": "備份失敗：%s",
- "24": "獲取雲端同步資訊失敗：%s",
+ "24": "TODO",
"25": "屬性名僅支援英文字母和阿拉伯數字",
"26": "請先在 [設置 - 關於 - 數據倉庫密鑰] 中初始化數據倉庫密鑰",
"27": "數據完整性校驗失敗",
- "28": "端到端加密密碼不正確，無法解密數據",
+ "28": "TODO",
"29": "該功能需要付費訂閱（如果你已經訂閱，請在設定-帳號中重繪或者重新登入）",
"30": "獲取雲端備份資訊失敗",
"31": "帳號鑒權失敗，請重新登入帳號",
@@ -793,7 +793,7 @@
"37": "雲端同步目錄的名稱請勿包含空格和特殊符號",
"38": "提及關鍵字數量 [%d] 過多，目前最多僅支援搜索 [512] 個關鍵字",
"39": "TODO",
- "40": "解密數據失敗",
+ "40": "TODO",
"41": "上傳完畢",
"42": "設置完成，即將自動關閉應用，請稍後重新啟動...",
"43": "已超過雲端空間最大存儲容量 [%s]，無法繼續上傳數據",
@@ -846,7 +846,7 @@
"90": "TODO",
"91": "TODO",
"92": "TODO",
- "93": "下載失敗：%s",
+ "93": "TODO",
"94": "上傳失敗：%s",
"95": "正在退出...",
"96": "退出時同步失敗，請手動執行一次同步以確保本地資料和雲端資料一致",
@@ -856,10 +856,10 @@
"100": "正在清理數據...",
"101": "設置提醒完畢 [%s]",
"102": "TODO",
- "103": "已下載 [%d] 個資料檔案，剩餘待下載 [%d]",
+ "103": "TODO",
"104": "已上傳 [%d] 個資料檔案，剩餘待上傳 [%d]",
"105": "網絡傳輸完畢",
- "106": "資料下載已經完成，正在進行解密...",
+ "106": "TODO",
"107": "正在移動文檔 [%s]",
"108": "正在清理已過時的索引...",
"109": "移除提醒完畢 [%s]",
@@ -881,13 +881,13 @@
"126": "書籤不能為空",
"127": "訂閱距過期還剩 [%d] 天，過期後雲端數據會被徹底刪除。續訂請訪問這裡，如果不需要續訂，請登出賬號關閉該提醒",
"128": "訂閱已經過期，過期後雲端數據會被徹底刪除。續訂請訪問這裡，如果不需要續訂，請登出賬號關閉該提醒",
- "129": "已傳輸文件 %d\n接收字節數 %s\n",
- "130": "已傳輸文件 %d\n發送字節數 %s\n",
- "131": "下載耗時 %.2fs",
- "132": "上傳耗時 %.2fs",
- "133": "本地數據暫無變更",
+ "129": "TODO",
+ "130": "TODO",
+ "131": "TODO",
+ "132": "TODO",
+ "133": "TODO",
"134": "為避免剛恢復的數據被同步覆蓋，數據同步功能已被自動暫停",
- "135": "請確保所有設備已經更新到最新版，然後在主力設備上隨意更改一個文檔後觸發同步，最後再到其他設備觸發同步",
+ "135": "TODO",
"136": "初始化數據倉庫密鑰...",
"137": "初始化數據倉庫密鑰失敗：%s",
"138": "數據倉庫密鑰設置完畢",
diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json
index 964f9ee17..2db9a2e2c 100644
--- a/app/appearance/langs/zh_CN.json
+++ b/app/appearance/langs/zh_CN.json
@@ -778,11 +778,11 @@
"21": "备份完毕",
"22": "正在备份，请稍等...",
"23": "备份失败：%s",
- "24": "获取云端同步信息失败：%s",
+ "24": "TODO",
"25": "属性名仅支持英文字母和阿拉伯数字",
"26": "请先在 [设置 - 关于 - 数据仓库密钥] 中初始化数据仓库密钥",
"27": "数据完整性校验失败",
- "28": "端到端加密密码不正确，无法解密数据",
+ "28": "TODO",
"29": "该功能需要付费订阅（如果你已经订阅，请在 设置 - 账号中刷新或者重新登录）",
"30": "获取云端备份信息失败",
"31": "账号鉴权失败，请重新登录账号",
@@ -794,7 +794,7 @@
"37": "云端同步目录的名称请勿包含空格和特殊符号",
"38": "提及关键字数量 [%d] 过多，目前最多仅支持搜索 [512] 个关键字",
"39": "TODO",
- "40": "解密数据失败",
+ "40": "TODO",
"41": "上传完毕",
"42": "设置完成，即将自动关闭应用，请稍后重新启动...",
"43": "已超过云端空间最大存储容量 [%s]，无法继续上传数据",
@@ -847,7 +847,7 @@
"90": "TODO",
"91": "TODO",
"92": "TODO",
- "93": "下载失败：%s",
+ "93": "TODO",
"94": "上传失败：%s",
"95": "正在退出...",
"96": "退出时同步失败，请手动执行一次同步以确保本地数据和云端数据一致",
@@ -857,10 +857,10 @@
"100": "正在清理数据...",
"101": "设置提醒完毕 [%s]",
"102": "TODO",
- "103": "已下载 [%d] 个数据文件，剩余待下载 [%d]",
+ "103": "TODO",
"104": "已上传 [%d] 个数据文件，剩余待上传 [%d]",
"105": "网络传输完毕",
- "106": "数据下载已经完成，正在进行解密...",
+ "106": "TODO",
"107": "正在移动文档 [%s]",
"108": "正在清理已过时的索引...",
"109": "移除提醒完毕 [%s]",
@@ -883,13 +883,13 @@
"126": "书签不能为空",
"127": "订阅距过期还剩 [%d] 天，过期后云端数据会被彻底删除。续订请访问这里，如果不需要续订，请登出账号关闭该提醒",
"128": "订阅已经过期，过期后云端数据会被彻底删除。续订请访问这里，如果不需要续订，请登出账号关闭该提醒",
- "129": "已传输文件 %d\n接收字节数 %s\n",
- "130": "已传输文件 %d\n发送字节数 %s\n",
- "131": "下载耗时 %.2fs",
- "132": "上传耗时 %.2fs",
- "133": "本地数据暂无变更",
+ "129": "TODO",
+ "130": "TODO",
+ "131": "TODO",
+ "132": "TODO",
+ "133": "TODO",
"134": "为避免刚恢复的数据被同步覆盖，数据同步功能已被自动暂停",
- "135": "请确保所有设备已经更新到最新版，然后在主力设备上随意更改一个文档后触发同步，最后再到其他设备触发同步",
+ "135": "TODO",
"136": "初始化数据仓库密钥...",
"137": "初始化数据仓库密钥失败：%s",
"138": "数据仓库密钥设置完毕",
diff --git a/kernel/go.mod b/kernel/go.mod
index e8eecff4d..62685944b 100644
--- a/kernel/go.mod
+++ b/kernel/go.mod
@@ -42,7 +42,7 @@ require (
github.com/radovskyb/watcher v1.0.7
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/siyuan-note/dejavu v0.0.0-20220711060744-3fec84096399
- github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676
+ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f
github.com/siyuan-note/filelock v0.0.0-20220704090116-54dfb035283f
github.com/siyuan-note/httpclient v0.0.0-20220709030145-2bfb50f28e73
diff --git a/kernel/go.sum b/kernel/go.sum
index 18ca966d3..dd8e83865 100644
--- a/kernel/go.sum
+++ b/kernel/go.sum
@@ -418,8 +418,8 @@ github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJV
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/siyuan-note/dejavu v0.0.0-20220711060744-3fec84096399 h1:kg4BZwxn4A5d9YD9sx6GnyZ6o+Rn1IiuhrZ5qYrVXV0=
github.com/siyuan-note/dejavu v0.0.0-20220711060744-3fec84096399/go.mod h1:cri+XyZAqmK5fJ98En9aOHB+YkuU8+XQcJdQ31EUhis=
-github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676 h1:QB9TjJQFhXhZ6dAtPpY02DlzHAQm1C+WqZq6OadG8mI=
-github.com/siyuan-note/encryption v0.0.0-20220612074546-f1dd94fe8676/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
+github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29LW+Fm0cHc0J1NO1cZqyJwljSWVmfOqVZgaE=
+github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f h1:JMobMNZ7AqaKKyEK+WeWFhix/2TDQXgPZDajU00IybU=
github.com/siyuan-note/eventbus v0.0.0-20220624162334-ca7c06dc771f/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/filelock v0.0.0-20220704090116-54dfb035283f h1:IXZ4SWPjQLqMrBwDWcWYFE/SihUHRS9FYhk/0bnySok=
diff --git a/kernel/model/osssync.go b/kernel/model/osssync.go
index afac2ecab..c69d648ec 100644
--- a/kernel/model/osssync.go
+++ b/kernel/model/osssync.go
@@ -20,19 +20,16 @@ import (
"context"
"errors"
"fmt"
- "io"
"io/fs"
"os"
"path"
"path/filepath"
- "sort"
"strconv"
"strings"
"sync"
"time"
"github.com/88250/gulu"
- "github.com/imroc/req/v3"
"github.com/panjf2000/ants/v2"
"github.com/qiniu/go-sdk/v7/storage"
"github.com/siyuan-note/httpclient"
@@ -41,20 +38,11 @@ import (
func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, err error) {
result := map[string]interface{}{}
- request := httpclient.NewCloudRequest()
+ resp, err := httpclient.NewCloudRequest().
+ SetResult(&result).
+ SetBody(map[string]string{"token": Conf.User.UserToken}).
+ Post(util.AliyunServer + "/apis/siyuan/dejavu/getRepoStat?uid=" + Conf.User.UserId)
- var resp *req.Response
- if Conf.Sync.UseDataRepo {
- resp, err = request.
- SetResult(&result).
- SetBody(map[string]string{"token": Conf.User.UserToken}).
- Post(util.AliyunServer + "/apis/siyuan/dejavu/getRepoStat?uid=" + Conf.User.UserId)
- } else {
- resp, err = request.
- SetResult(&result).
- SetBody(map[string]string{"token": Conf.User.UserToken}).
- Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanWorkspace?uid=" + Conf.User.UserId)
- }
if nil != err {
util.LogErrorf("get cloud space failed: %s", err)
err = ErrFailedToConnectCloudServer
@@ -80,255 +68,6 @@ func getCloudSpaceOSS() (sync, backup map[string]interface{}, assetSize int64, e
return
}
-func removeCloudDirPath(dirPath string) (err error) {
- result := map[string]interface{}{}
- request := httpclient.NewCloudRequest()
- resp, err := request.
- SetResult(&result).
- SetBody(map[string]string{"dirPath": dirPath, "token": Conf.User.UserToken}).
- Post(util.AliyunServer + "/apis/siyuan/data/removeSiYuanDirPath?uid=" + Conf.User.UserId)
- if nil != err {
- util.LogErrorf("create cloud sync dir failed: %s", err)
- return ErrFailedToConnectCloudServer
- }
-
- if 200 != resp.StatusCode {
- if 401 == resp.StatusCode {
- err = errors.New(Conf.Language(31))
- return
- }
- msg := fmt.Sprintf("remove cloud dir failed: %d", resp.StatusCode)
- util.LogErrorf(msg)
- err = errors.New(msg)
- return
- }
- return
-}
-
-func listCloudSyncDirOSS() (dirs []map[string]interface{}, size int64, err error) {
- result := map[string]interface{}{}
- request := httpclient.NewCloudRequest()
- resp, err := request.
- SetBody(map[string]interface{}{"token": Conf.User.UserToken}).
- SetResult(&result).
- Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanSyncDirList?uid=" + Conf.User.UserId)
- if nil != err {
- util.LogErrorf("get cloud sync dirs failed: %s", err)
- return nil, 0, ErrFailedToConnectCloudServer
- }
-
- if 200 != resp.StatusCode {
- if 401 == resp.StatusCode {
- err = errors.New(Conf.Language(31))
- return
- }
- msg := fmt.Sprintf("get cloud sync dirs failed: %d", resp.StatusCode)
- util.LogErrorf(msg)
- err = errors.New(msg)
- return
- }
-
- code := result["code"].(float64)
- if 0 != code {
- util.LogErrorf("get cloud sync dirs failed: %s", result["msg"])
- return nil, 0, ErrFailedToConnectCloudServer
- }
-
- data := result["data"].(map[string]interface{})
- dataDirs := data["dirs"].([]interface{})
- for _, d := range dataDirs {
- dirs = append(dirs, d.(map[string]interface{}))
- }
- sort.Slice(dirs, func(i, j int) bool { return dirs[i]["name"].(string) < dirs[j]["name"].(string) })
- size = int64(data["size"].(float64))
- return
-}
-
-func ossDownload(localDirPath, cloudDirPath string, bootOrExit bool) (fetchedFilesCount int, transferSize uint64, downloadedFiles map[string]bool, err error) {
- if !gulu.File.IsExist(localDirPath) {
- return
- }
-
- cloudFileList, err := getCloudFileListOSS(cloudDirPath)
- if nil != err {
- return
- }
-
- if "backup" != cloudDirPath {
- // 将云端索引文件临时保存一下，后面下载数据时如果部分成功，需要用索引文件恢复部分成功的文件 syncDirUpsertWorkspaceData()
-
- var data []byte
- data, err = gulu.JSON.MarshalJSON(cloudFileList)
- if nil != err {
- return
- }
- tmpSyncDir := filepath.Join(util.TempDir, "sync")
- err = os.MkdirAll(tmpSyncDir, 0755)
- if nil != err {
- return
- }
- tmpIndex := filepath.Join(tmpSyncDir, "index.json")
- if err = os.WriteFile(tmpIndex, data, 0644); nil != err {
- return
- }
- }
-
- localRemoves, cloudFetches, err := localUpsertRemoveListOSS(localDirPath, cloudFileList)
- if nil != err {
- return
- }
-
- for _, localRemove := range localRemoves {
- if err = os.RemoveAll(localRemove); nil != err {
- util.LogErrorf("local remove [%s] failed: %s", localRemove, err)
- return
- }
- }
-
- needPushProgress := 32 < len(cloudFetches)
- waitGroup := &sync.WaitGroup{}
- var downloadErr error
- downloadedFilesLock := sync.Mutex{}
- downloadedFiles = map[string]bool{}
- poolSize := 4
- if poolSize > len(cloudFetches)-1 /* 不计入 /.siyuan/conf.json，配置文件最后单独下载 */ {
- poolSize = len(cloudFetches)
- }
- p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) {
- defer waitGroup.Done()
- if nil != downloadErr {
- return // 快速失败
- }
-
- fetch := arg.(string)
- err = ossDownload0(localDirPath, cloudDirPath, fetch, &fetchedFilesCount, &transferSize, bootOrExit)
- if nil != err {
- downloadErr = err // 仅记录最后一次错误
- return
- }
- downloadedFilesLock.Lock()
- downloadedFiles[fetch] = true
- downloadedFilesLock.Unlock()
-
- if needPushProgress {
- msg := fmt.Sprintf(Conf.Language(103), fetchedFilesCount, len(cloudFetches)-fetchedFilesCount)
- util.PushProgress(util.PushProgressCodeProgressed, fetchedFilesCount, len(cloudFetches), msg)
- }
- if bootOrExit {
- msg := fmt.Sprintf("Downloading data from the cloud %d/%d", fetchedFilesCount, len(cloudFetches))
- util.IncBootProgress(0, msg)
- }
- })
- for _, fetch := range cloudFetches {
- if "/.siyuan/conf.json" == fetch {
- // 同步下载可能会报错，为了确保本地数据版本号不变所以不能更新配置文件，配置文件最后单独下载
- continue
- }
- if "/"+pathJSON == fetch {
- // 已经在前面验证解密的步骤中下载过了，目前位于 temp/sync/pathJSON
- continue
- }
-
- waitGroup.Add(1)
- p.Invoke(fetch)
- }
- waitGroup.Wait()
- p.Release()
- if nil != downloadErr {
- err = downloadErr
- return
- }
-
- if "backup" != cloudDirPath {
- err = ossDownload0(localDirPath, cloudDirPath, "/.siyuan/conf.json", &fetchedFilesCount, &transferSize, bootOrExit)
- if nil != err {
- return
- }
- }
- if needPushProgress {
- util.ClearPushProgress(len(cloudFetches))
- util.PushMsg(Conf.Language(106), 1000*60*10)
- }
- if bootOrExit {
- util.IncBootProgress(0, "Decrypting from sync to data...")
- }
- return
-}
-
-func ossDownload0(localDirPath, cloudDirPath, fetch string, fetchedFiles *int, transferSize *uint64, bootORExit bool) (err error) {
- localFilePath := filepath.Join(localDirPath, fetch)
- remoteFileURL := path.Join(cloudDirPath, fetch)
- var result map[string]interface{}
- resp, err := httpclient.NewCloudRequest().
- SetResult(&result).
- SetBody(map[string]interface{}{"token": Conf.User.UserToken, "path": remoteFileURL}).
- Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanFile?uid=" + Conf.User.UserId)
- if nil != err {
- util.LogErrorf("download request [%s] failed: %s", remoteFileURL, err)
- return errors.New(fmt.Sprintf(Conf.Language(93), err))
- }
-
- if 200 != resp.StatusCode {
- if 401 == resp.StatusCode {
- err = errors.New("account authentication failed, please login again")
- return errors.New(fmt.Sprintf(Conf.Language(93), err))
- }
- util.LogErrorf("download request status code [%d]", resp.StatusCode)
- err = errors.New("download file URL failed")
- return errors.New(fmt.Sprintf(Conf.Language(93), err))
- }
-
- code := result["code"].(float64)
- if 0 != code {
- msg := result["msg"].(string)
- util.LogErrorf("download cloud file failed: %s", msg)
- return errors.New(fmt.Sprintf(Conf.Language(93), msg))
- }
-
- resultData := result["data"].(map[string]interface{})
- downloadURL := resultData["url"].(string)
-
- if err = os.MkdirAll(filepath.Dir(localFilePath), 0755); nil != err {
- return
- }
- os.Remove(localFilePath)
-
- if bootORExit {
- resp, err = httpclient.NewCloudFileRequest15s().Get(downloadURL)
- } else {
- resp, err = httpclient.NewCloudFileRequest2m().Get(downloadURL)
- }
- if nil != err {
- util.LogErrorf("download request [%s] failed: %s", downloadURL, err)
- return errors.New(fmt.Sprintf(Conf.Language(93), err))
- }
- if 200 != resp.StatusCode {
- util.LogErrorf("download request [%s] status code [%d]", downloadURL, resp.StatusCode)
- err = errors.New(fmt.Sprintf("download file failed [%d]", resp.StatusCode))
- if 404 == resp.StatusCode {
- err = errors.New(Conf.Language(135))
- }
- return errors.New(fmt.Sprintf(Conf.Language(93), err))
- }
-
- data, err := resp.ToBytes()
- if nil != err {
- util.LogErrorf("download read response body data failed: %s, %s", err, string(data))
- err = errors.New("download read data failed")
- return errors.New(fmt.Sprintf(Conf.Language(93), err))
- }
- size := int64(len(data))
-
- if err = gulu.File.WriteFileSafer(localFilePath, data, 0644); nil != err {
- util.LogErrorf("write file [%s] failed: %s", localFilePath, err)
- return errors.New(fmt.Sprintf(Conf.Language(93), err))
- }
-
- *fetchedFiles++
- *transferSize += uint64(size)
- return
-}
-
func ossUpload(isBackup bool, localDirPath, cloudDirPath, cloudDevice string, boot bool) (wroteFiles int, transferSize uint64, err error) {
if !gulu.File.IsExist(localDirPath) {
return
@@ -530,90 +269,6 @@ func getOssUploadToken(filename, cloudDirPath string, length int64) (ret string,
return
}
-func getCloudSyncVer(cloudDir string) (cloudSyncVer int64, err error) {
- start := time.Now()
- result := map[string]interface{}{}
- request := httpclient.NewCloudRequest()
- resp, err := request.
- SetResult(&result).
- SetBody(map[string]string{"syncDir": cloudDir, "token": Conf.User.UserToken}).
- Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanWorkspaceSyncVer?uid=" + Conf.User.UserId)
- if nil != err {
- util.LogErrorf("get cloud sync ver failed: %s", err)
- err = ErrFailedToConnectCloudServer
- return
- }
- if 200 != resp.StatusCode {
- if 401 == resp.StatusCode {
- err = errors.New(Conf.Language(31))
- return
- }
- util.LogErrorf("get cloud sync ver failed: %d", resp.StatusCode)
- err = ErrFailedToConnectCloudServer
- return
- }
-
- code := result["code"].(float64)
- if 0 != code {
- msg := result["msg"].(string)
- util.LogErrorf("get cloud sync ver failed: %s", msg)
- err = errors.New(msg)
- return
- }
-
- data := result["data"].(map[string]interface{})
- cloudSyncVer = int64(data["v"].(float64))
-
- if elapsed := time.Now().Sub(start).Milliseconds(); 2000 < elapsed {
- util.LogInfof("get cloud sync ver elapsed [%dms]", elapsed)
- }
- return
-}
-
-func getCloudSync(cloudDir string) (assetSize, backupSize int64, device string, err error) {
- start := time.Now()
- result := map[string]interface{}{}
- request := httpclient.NewCloudRequest()
- resp, err := request.
- SetResult(&result).
- SetBody(map[string]string{"syncDir": cloudDir, "token": Conf.User.UserToken}).
- Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanWorkspaceSync?uid=" + Conf.User.UserId)
- if nil != err {
- util.LogErrorf("get cloud sync info failed: %s", err)
- err = ErrFailedToConnectCloudServer
- return
- }
- if 200 != resp.StatusCode {
- if 401 == resp.StatusCode {
- err = errors.New(Conf.Language(31))
- return
- }
- util.LogErrorf("get cloud sync info failed: %d", resp.StatusCode)
- err = ErrFailedToConnectCloudServer
- return
- }
-
- code := result["code"].(float64)
- if 0 != code {
- msg := result["msg"].(string)
- util.LogErrorf("get cloud sync info failed: %s", msg)
- err = errors.New(msg)
- return
- }
-
- data := result["data"].(map[string]interface{})
- assetSize = int64(data["assetSize"].(float64))
- backupSize = int64(data["backupSize"].(float64))
- if nil != data["d"] {
- device = data["d"].(string)
- }
-
- if elapsed := time.Now().Sub(start).Milliseconds(); 5000 < elapsed {
- util.LogInfof("get cloud sync [%s] elapsed [%dms]", elapsed)
- }
- return
-}
-
func getLocalFileListOSS(isBackup bool) (ret map[string]*CloudIndex, err error) {
ret = map[string]*CloudIndex{}
dir := "sync"
@@ -636,109 +291,6 @@ func getLocalFileListOSS(isBackup bool) (ret map[string]*CloudIndex, err error)
return
}
-func getCloudFileListOSS(cloudDirPath string) (ret map[string]*CloudIndex, err error) {
- result := map[string]interface{}{}
- request := httpclient.NewCloudRequest()
- resp, err := request.
- SetResult(&result).
- SetBody(map[string]string{"dirPath": cloudDirPath, "token": Conf.User.UserToken}).
- Post(util.AliyunServer + "/apis/siyuan/data/getSiYuanFileListURL?uid=" + Conf.User.UserId)
- if nil != err {
- util.LogErrorf("get cloud file list failed: %s", err)
- err = ErrFailedToConnectCloudServer
- return
- }
-
- if 401 == resp.StatusCode {
- err = errors.New(Conf.Language(31))
- return
- }
-
- code := result["code"].(float64)
- if 0 != code {
- util.LogErrorf("get cloud file list failed: %s", result["msg"])
- err = ErrFailedToConnectCloudServer
- return
- }
-
- retData := result["data"].(map[string]interface{})
- downloadURL := retData["url"].(string)
- resp, err = httpclient.NewCloudFileRequest15s().Get(downloadURL)
- if nil != err {
- util.LogErrorf("download request [%s] failed: %s", downloadURL, err)
- return
- }
- if 200 != resp.StatusCode {
- util.LogErrorf("download request [%s] status code [%d]", downloadURL, resp.StatusCode)
- err = errors.New(fmt.Sprintf("download file list failed [%d]", resp.StatusCode))
- return
- }
-
- data, err := resp.ToBytes()
- if err = gulu.JSON.UnmarshalJSON(data, &ret); nil != err {
- util.LogErrorf("unmarshal index failed: %s", err)
- err = errors.New(fmt.Sprintf("unmarshal index failed"))
- return
- }
- return
-}
-
-func localUpsertRemoveListOSS(localDirPath string, cloudFileList map[string]*CloudIndex) (localRemoves, cloudFetches []string, err error) {
- unchanged := map[string]bool{}
-
- filepath.Walk(localDirPath, func(path string, info fs.FileInfo, err error) error {
- if localDirPath == path {
- return nil
- }
-
- if info.IsDir() {
- return nil
- }
-
- relPath := filepath.ToSlash(strings.TrimPrefix(path, localDirPath))
- cloudIdx, ok := cloudFileList[relPath]
- if !ok {
- if util.CloudSingleFileMaxSizeLimit < info.Size() {
- util.LogWarnf("file [%s] larger than 100MB, ignore removing it", path)
- return nil
- }
-
- localRemoves = append(localRemoves, path)
- return nil
- }
-
- if 0 < cloudIdx.Updated {
- // 优先使用时间戳校验
- if localModTime := info.ModTime().Unix(); cloudIdx.Updated == localModTime {
- unchanged[relPath] = true
- }
- return nil
- }
-
- localHash, hashErr := util.GetEtag(path)
- if nil != hashErr {
- err = hashErr
- return io.EOF
- }
- if cloudIdx.Hash == localHash {
- unchanged[relPath] = true
- }
- return nil
- })
-
- for cloudPath, cloudIndex := range cloudFileList {
- if _, ok := unchanged[cloudPath]; ok {
- continue
- }
- if util.CloudSingleFileMaxSizeLimit < cloudIndex.Size {
- util.LogWarnf("cloud file [%s] larger than 100MB, ignore fetching it", cloudPath)
- continue
- }
- cloudFetches = append(cloudFetches, cloudPath)
- }
- return
-}
-
func cloudUpsertRemoveListOSS(localDirPath string, cloudFileList, localFileList map[string]*CloudIndex, excludes map[string]bool) (localUpserts, cloudRemoves []string, err error) {
localUpserts, cloudRemoves = []string{}, []string{}
diff --git a/kernel/model/sync.go b/kernel/model/sync.go
index 9f877db84..a046e2250 100644
--- a/kernel/model/sync.go
+++ b/kernel/model/sync.go
@@ -17,13 +17,10 @@
package model
import (
- "bytes"
- "crypto/sha256"
"errors"
"fmt"
"io"
"io/fs"
- "math"
"os"
"os/exec"
"path/filepath"
@@ -34,11 +31,8 @@ import (
"github.com/88250/gulu"
"github.com/dustin/go-humanize"
- gitignore "github.com/sabhiram/go-gitignore"
"github.com/siyuan-note/dejavu"
- "github.com/siyuan-note/encryption"
"github.com/siyuan-note/filelock"
- "github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
@@ -126,98 +120,11 @@ func SyncData(boot, exit, byHand bool) {
util.BroadcastByType("main", "syncing", 1, msg, nil)
}()
- Conf.Sync.Stat = Conf.Language(133)
- syncLock.Lock()
- defer syncLock.Unlock()
-
if Conf.Sync.UseDataRepo {
syncRepo(boot, exit, byHand)
return
}
- WaitForWritingFiles()
- writingDataLock.Lock()
- var err error
- // 将 data 变更同步到 sync
- if _, _, err = workspaceData2SyncDir(); nil != err {
- msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
- Conf.Sync.Stat = msg
- util.PushErrMsg(msg, 7000)
- if boot {
- BootSyncSucc = 1
- }
- if exit {
- ExitSyncSucc = 1
- }
- writingDataLock.Unlock()
- return
- }
-
- // 获取工作空间数据配置（数据版本）
- dataConf, err := getWorkspaceDataConf()
- if nil != err {
- msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
- Conf.Sync.Stat = msg
- util.PushErrMsg(msg, 7000)
- if boot {
- BootSyncSucc = 1
- }
- if exit {
- ExitSyncSucc = 1
- }
- writingDataLock.Unlock()
- return
- }
- writingDataLock.Unlock()
-
- cloudSyncVer, err := getCloudSyncVer(Conf.Sync.CloudName)
- if nil != err {
- msg := fmt.Sprintf(Conf.Language(24), err.Error())
- Conf.Sync.Stat = msg
- util.PushErrMsg(msg, 7000)
- if boot {
- BootSyncSucc = 1
- }
- if exit {
- ExitSyncSucc = 1
- }
- return
- }
-
- //util.LogInfof("sync [cloud=%d, local=%d]", cloudSyncVer, dataConf.SyncVer)
- if cloudSyncVer == dataConf.SyncVer {
- BootSyncSucc = 0
- ExitSyncSucc = 0
- syncSameCount++
- if 10 < syncSameCount {
- syncSameCount = 5
- }
- if !byHand {
- delay := time.Minute * time.Duration(int(math.Pow(2, float64(syncSameCount))))
- if fixSyncInterval.Minutes() > delay.Minutes() {
- delay = time.Minute * 8
- }
- planSyncAfter(delay)
- }
-
- Conf.Sync.Stat = Conf.Language(133)
- return
- }
-
- cloudUsedAssetSize, cloudUsedBackupSize, device, err := getCloudSync(Conf.Sync.CloudName)
- if nil != err {
- msg := fmt.Sprintf(Conf.Language(24), err.Error())
- Conf.Sync.Stat = msg
- util.PushErrMsg(msg, 7000)
- if boot {
- BootSyncSucc = 1
- }
- if exit {
- ExitSyncSucc = 1
- }
- return
- }
-
localSyncDirPath := Conf.Sync.GetSaveDir()
syncSameCount = 0
if cloudSyncVer < dataConf.SyncVer {
@@ -280,159 +187,9 @@ func SyncData(boot, exit, byHand bool) {
}
return
}
-
- // 下载
-
- if !boot && !exit {
- CloseWatchAssets()
- defer WatchAssets()
- }
-
- start := time.Now()
- //util.LogInfof("sync [cloud=%d, local=%d] downloading...", cloudSyncVer, dataConf.SyncVer)
-
- // 使用路径映射文件进行解密验证 https://github.com/siyuan-note/siyuan/issues/3789
- var tmpFetchedFiles int
- var tmpTransferSize uint64
- err = ossDownload0(util.TempDir+"/sync", "sync/"+Conf.Sync.CloudName, "/"+pathJSON, &tmpFetchedFiles, &tmpTransferSize, boot || exit)
- if nil != err {
- util.PushClearProgress()
- msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
- Conf.Sync.Stat = msg
- util.PushErrMsg(msg, 7000)
- if boot {
- BootSyncSucc = 1
- }
- if exit {
- ExitSyncSucc = 1
- }
- syncDownloadErrCount++
- return
- }
-
- tmpPathJSON := filepath.Join(util.TempDir, "/sync/"+pathJSON)
- data, err := os.ReadFile(tmpPathJSON)
- if nil != err {
- return
- }
- data, err = encryption.AESGCMDecryptBinBytes(data, Conf.E2EEPasswd)
- if nil != err {
- util.PushClearProgress()
- msg := Conf.Language(28)
- Conf.Sync.Stat = msg
- util.PushErrMsg(fmt.Sprintf(Conf.Language(80), msg), 7000)
- if boot {
- BootSyncSucc = 1
- }
- if exit {
- ExitSyncSucc = 1
- }
- Conf.Sync.Stat = msg
- syncDownloadErrCount++
- return
- }
-
- fetchedFilesCount, transferSize, downloadedFiles, err := ossDownload(localSyncDirPath, "sync/"+Conf.Sync.CloudName, boot || exit)
-
- // 加上前面的路径映射文件统计
- fetchedFilesCount += tmpFetchedFiles
- transferSize += tmpTransferSize
-
- if nil != err {
- util.PushClearProgress()
- msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
- Conf.Sync.Stat = msg
- util.PushErrMsg(msg, 7000)
-
- indexPath := filepath.Join(util.TempDir, "sync", "index.json")
- _, err = syncDirUpsertWorkspaceData(tmpPathJSON, indexPath, downloadedFiles)
- if nil != err {
- util.LogErrorf("upsert partially downloaded files to workspace data failed: %s", err)
- }
-
- if boot {
- BootSyncSucc = 1
- }
- if exit {
- ExitSyncSucc = 1
- }
- syncDownloadErrCount++
- return
- }
- util.PushClearProgress()
-
- // 恢复
- var upsertFiles, removeFiles []string
- if upsertFiles, removeFiles, err = syncDir2WorkspaceData(boot); nil != err {
- msg := fmt.Sprintf(Conf.Language(80), formatErrorMsg(err))
- Conf.Sync.Stat = msg
- util.PushErrMsg(msg, 7000)
- if boot {
- BootSyncSucc = 1
- }
- if exit {
- ExitSyncSucc = 1
- }
- syncDownloadErrCount++
- return
- }
-
- syncDownloadErrCount = 0
-
- // 清理空文件夹
- clearEmptyDirs(util.DataDir)
-
- elapsed := time.Now().Sub(start).Seconds()
- stat := fmt.Sprintf(Conf.Language(129), fetchedFilesCount, humanize.Bytes(transferSize)) + fmt.Sprintf(Conf.Language(131), elapsed)
- util.LogInfof("sync [cloud=%d, local=%d, fetchedFiles=%d, transferSize=%s] downloaded in [%.2fs]", cloudSyncVer, dataConf.SyncVer, fetchedFilesCount, humanize.Bytes(transferSize), elapsed)
-
- Conf.Sync.Downloaded = now
- Conf.Sync.Stat = stat
- BootSyncSucc = 0
- ExitSyncSucc = 0
- if !byHand {
- planSyncAfter(fixSyncInterval)
- }
-
- if boot && gulu.File.IsExist(util.BlockTreePath) {
- // 在 blocktree 存在的情况下不会重建索引，所以这里需要刷新 blocktree 和 database
- treenode.InitBlockTree()
- incReindex(upsertFiles, removeFiles)
- return
- }
-
- if !boot && !exit {
- incReindex(upsertFiles, removeFiles)
- cache.ClearDocsIAL() // 同步后文档树文档图标没有更新 https://github.com/siyuan-note/siyuan/issues/4939
- util.ReloadUI()
- }
return
}
-// TODO: 新版同步上线后移除
-// 清理 dir 下符合 ID 规则的空文件夹。
-// 因为是深度遍历，所以可能会清理不完全空文件夹，每次遍历仅能清理叶子节点。但是多次调用后，可以清理完全。
-func clearEmptyDirs(dir string) {
- var emptyDirs []string
- filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
- if nil != err || !info.IsDir() || dir == path {
- return err
- }
-
- if util.IsIDPattern(info.Name()) {
- if files, readDirErr := os.ReadDir(path); nil == readDirErr && 0 == len(files) {
- emptyDirs = append(emptyDirs, path)
- }
- }
- return nil
- })
- for _, emptyDir := range emptyDirs {
- if err := os.RemoveAll(emptyDir); nil != err {
- util.LogErrorf("clear empty dir [%s] failed [%s]", emptyDir, err.Error())
- }
- }
-}
-
// incReindex 增量重建索引。
func incReindex(upserts, removes []string) {
needPushUpsertProgress := 32 < len(upserts)
@@ -522,97 +279,6 @@ func SetSyncMode(mode int) (err error) {
var syncLock = sync.Mutex{}
-func syncDirUpsertWorkspaceData(metaPath, indexPath string, downloadedFiles map[string]bool) (upsertFiles []string, err error) {
- start := time.Now()
-
- modified := map[string]bool{}
- syncDir := Conf.Sync.GetSaveDir()
- for file, _ := range downloadedFiles {
- file = filepath.Join(syncDir, file)
- modified[file] = true
- }
-
- decryptedDataDir, upsertFiles, err := recoverSyncData(metaPath, indexPath, modified)
- if nil != err {
- util.LogErrorf("decrypt data dir failed: %s", err)
- return
- }
-
- dataDir := util.DataDir
- if err = stableCopy(decryptedDataDir, dataDir); nil != err {
- util.LogErrorf("copy decrypted data dir from [%s] to data dir [%s] failed: %s", decryptedDataDir, dataDir, err)
- return
- }
- if elapsed := time.Since(start).Milliseconds(); 5000 < elapsed {
- util.LogInfof("sync data to workspace data elapsed [%dms]", elapsed)
- }
- return
-}
-
-// syncDir2WorkspaceData 将 sync 的数据更新到 data 中。
-// 1. 删除 data 中冗余的文件
-// 2. 将 sync 中新增/修改的文件解密后拷贝到 data 中
-func syncDir2WorkspaceData(boot bool) (upsertFiles, removeFiles []string, err error) {
- start := time.Now()
- unchanged, removeFiles, err := calcUnchangedSyncList()
- if nil != err {
- return
- }
-
- modified := modifiedSyncList(unchanged)
- metaPath := filepath.Join(util.TempDir, "sync", pathJSON) // 使用前面解密验证时下载的临时文件
- indexPath := filepath.Join(util.TempDir, "sync", "index.json")
- decryptedDataDir, upsertFiles, err := recoverSyncData(metaPath, indexPath, modified)
- if nil != err {
- util.LogErrorf("decrypt data dir failed: %s", err)
- return
- }
-
- if boot {
- util.IncBootProgress(0, "Copying decrypted data...")
- }
-
- dataDir := util.DataDir
- if err = stableCopy(decryptedDataDir, dataDir); nil != err {
- util.LogErrorf("copy decrypted data dir from [%s] to data dir [%s] failed: %s", decryptedDataDir, dataDir, err)
- return
- }
- if elapsed := time.Since(start).Milliseconds(); 5000 < elapsed {
- util.LogInfof("sync data to workspace data elapsed [%dms]", elapsed)
- }
- return
-}
-
-// workspaceData2SyncDir 将 data 的数据更新到 sync 中。
-// 1. 删除 sync 中多余的文件
-// 2. 将 data 中新增/修改的文件加密后拷贝到 sync 中
-func workspaceData2SyncDir() (removeList, upsertList map[string]bool, err error) {
- start := time.Now()
- filelock.ReleaseAllFileLocks()
-
- passwd := Conf.E2EEPasswd
- unchangedDataList, removeList, err := calcUnchangedDataList(passwd)
- if nil != err {
- return
- }
-
- encryptedDataDir, upsertList, err := prepareSyncData(passwd, unchangedDataList)
- if nil != err {
- util.LogErrorf("encrypt data dir failed: %s", err)
- return
- }
-
- syncDir := Conf.Sync.GetSaveDir()
- if err = stableCopy(encryptedDataDir, syncDir); nil != err {
- util.LogErrorf("copy encrypted data dir from [%s] to sync dir [%s] failed: %s", encryptedDataDir, syncDir, err)
- return
- }
- if elapsed := time.Since(start).Milliseconds(); 5000 < elapsed {
- util.LogInfof("workspace data to sync data elapsed [%dms]", elapsed)
- }
- return
-}
-
type CloudIndex struct {
Hash string `json:"hash"`
Size int64 `json:"size"`
@@ -664,340 +330,6 @@ func genCloudIndex(localDirPath string, excludes map[string]bool, calcHash bool)
return
}
-func recoverSyncData(metaPath, indexPath string, modified map[string]bool) (decryptedDataDir string, upsertFiles []string, err error) {
- passwd := Conf.E2EEPasswd
- decryptedDataDir = filepath.Join(util.TempDir, "incremental", "sync-decrypt")
- if err = os.RemoveAll(decryptedDataDir); nil != err {
- return
- }
- if err = os.MkdirAll(decryptedDataDir, 0755); nil != err {
- return
- }
-
- syncDir := Conf.Sync.GetSaveDir()
- data, err := os.ReadFile(metaPath)
- if nil != err {
- return
- }
- data, err = encryption.AESGCMDecryptBinBytes(data, passwd)
- if nil != err {
- err = errors.New(Conf.Language(40))
- return
- }
-
- metaJSON := map[string]string{}
- if err = gulu.JSON.UnmarshalJSON(data, &metaJSON); nil != err {
- return
- }
-
- index := map[string]*CloudIndex{}
- data, err = os.ReadFile(indexPath)
- if nil != err {
- return
- }
- if err = gulu.JSON.UnmarshalJSON(data, &index); nil != err {
- return
- }
-
- now := time.Now().Format("2006-01-02-150405")
- filepath.Walk(syncDir, func(path string, info fs.FileInfo, _ error) error {
- if syncDir == path || pathJSON == info.Name() {
- return nil
- }
-
- // 如果不是新增或者修改则跳过
- if !modified[path] {
- return nil
- }
-
- encryptedP := strings.TrimPrefix(path, syncDir+string(os.PathSeparator))
- encryptedP = filepath.ToSlash(encryptedP)
- if "" == metaJSON[encryptedP] {
- return nil
- }
-
- plainP := filepath.Join(decryptedDataDir, metaJSON[encryptedP])
- plainP = filepath.FromSlash(plainP)
-
- p := strings.TrimPrefix(plainP, decryptedDataDir+string(os.PathSeparator))
- upsertFiles = append(upsertFiles, p)
- dataPath := filepath.Join(util.DataDir, p)
- if gulu.File.IsExist(dataPath) && !gulu.File.IsDir(dataPath) { // 不是目录的话说明必定是已经存在的文件，这些文件被覆盖需要生成历史
- genSyncHistory(now, dataPath)
- }
-
- if info.IsDir() {
- if err = os.MkdirAll(plainP, 0755); nil != err {
- return io.EOF
- }
- } else {
- if err = os.MkdirAll(filepath.Dir(plainP), 0755); nil != err {
- return io.EOF
- }
-
- var err0 error
- data, err0 = os.ReadFile(path)
- if nil != err0 {
- util.LogErrorf("read file [%s] failed: %s", path, err0)
- err = err0
- return io.EOF
- }
- if !strings.HasPrefix(encryptedP, ".siyuan") {
- data, err0 = encryption.AESGCMDecryptBinBytes(data, passwd)
- if nil != err0 {
- util.LogErrorf("decrypt file [%s] failed: %s", path, err0)
- err = errors.New(Conf.Language(40))
- return io.EOF
- }
- }
-
- if err0 = gulu.File.WriteFileSafer(plainP, data, 0644); nil != err0 {
- util.LogErrorf("write file [%s] failed: %s", plainP, err0)
- err = err0
- return io.EOF
- }
-
- var modTime int64
- idx := index["/"+encryptedP]
- if nil == idx {
- util.LogErrorf("index file [%s] not found", encryptedP)
- modTime = info.ModTime().Unix()
- } else {
- modTime = idx.Updated
- }
- if err0 = os.Chtimes(plainP, time.Unix(modTime, 0), time.Unix(modTime, 0)); nil != err0 {
- util.LogErrorf("change file [%s] time failed: %s", plainP, err0)
- }
- }
- return nil
- })
- return
-}
-
-func prepareSyncData(passwd string, unchangedDataList map[string]bool) (encryptedDataDir string, upsertList map[string]bool, err error) {
- encryptedDataDir = filepath.Join(util.TempDir, "incremental", "sync-encrypt")
- if err = os.RemoveAll(encryptedDataDir); nil != err {
- return
- }
- if err = os.MkdirAll(encryptedDataDir, 0755); nil != err {
- return
- }
-
- ctime := map[string]time.Time{}
- meta := map[string]string{}
- filepath.Walk(util.DataDir, func(path string, info fs.FileInfo, _ error) error {
- if util.DataDir == path || nil == info {
- return nil
- }
-
- isDir := info.IsDir()
- if isCloudSkipFile(path, info) {
- if isDir {
- return filepath.SkipDir
- }
- return nil
- }
-
- plainP := strings.TrimPrefix(path, util.DataDir+string(os.PathSeparator))
- p := plainP
-
- if !strings.HasPrefix(plainP, ".siyuan") { // 配置目录下都用明文，其他文件需要映射文件名
- p = pathSha256Short(p, string(os.PathSeparator))
- }
- if !isDir {
- meta[filepath.ToSlash(p)] = filepath.ToSlash(plainP)
- }
-
- // 如果不是新增或者修改则跳过
- if unchangedDataList[path] {
- return nil
- }
-
- p = encryptedDataDir + string(os.PathSeparator) + p
- //util.LogInfof("update sync [%s] for data [%s]", p, path)
- if isDir {
- if err = os.MkdirAll(p, 0755); nil != err {
- return io.EOF
- }
- } else {
- if err = os.MkdirAll(filepath.Dir(p), 0755); nil != err {
- return io.EOF
- }
-
- data, err0 := filelock.NoLockFileRead(path)
- if nil != err0 {
- util.LogErrorf("read file [%s] failed: %s", path, err0)
- err = err0
- return io.EOF
- }
- if !strings.HasPrefix(plainP, ".siyuan") {
- data, err0 = encryption.AESGCMEncryptBinBytes(data, passwd)
- if nil != err0 {
- util.LogErrorf("encrypt file [%s] failed: %s", path, err0)
- err = errors.New("encrypt file failed")
- return io.EOF
- }
- }
-
- err0 = gulu.File.WriteFileSafer(p, data, 0644)
- if nil != err0 {
- util.LogErrorf("write file [%s] failed: %s", p, err0)
- err = err0
- return io.EOF
- }
-
- fi, err0 := os.Stat(path)
- if nil != err0 {
- util.LogErrorf("stat file [%s] failed: %s", path, err0)
- err = err0
- return io.EOF
- }
- ctime[p] = fi.ModTime()
- }
- return nil
- })
- if nil != err {
- return
- }
-
- for p, t := range ctime {
- if err = os.Chtimes(p, t, t); nil != err {
- return
- }
- }
-
- upsertList = map[string]bool{}
- // 检查文件是否全部已经编入索引
- err = filepath.Walk(encryptedDataDir, func(path string, info fs.FileInfo, _ error) error {
- if encryptedDataDir == path {
- return nil
- }
-
- if !info.IsDir() {
- path = strings.TrimPrefix(path, encryptedDataDir+string(os.PathSeparator))
- path = filepath.ToSlash(path)
- if _, ok := meta[path]; !ok {
- util.LogErrorf("not found sync path in meta [%s]", path)
- return errors.New(Conf.Language(27))
- }
-
- upsertList["/"+path] = true
- }
- return nil
- })
- if nil != err {
- return
- }
-
- data, err := gulu.JSON.MarshalJSON(meta)
- if nil != err {
- return
- }
- data, err = encryption.AESGCMEncryptBinBytes(data, passwd)
- if nil != err {
- util.LogErrorf("encrypt file failed: %s", err)
- return
- }
- if err = gulu.File.WriteFileSafer(filepath.Join(encryptedDataDir, pathJSON), data, 0644); nil != err {
- return
- }
- return
-}
-
-// modifiedSyncList 获取 sync 中新增和修改的文件列表。
-func modifiedSyncList(unchangedList map[string]bool) (ret map[string]bool) {
- ret = map[string]bool{}
- syncDir := Conf.Sync.GetSaveDir()
- filepath.Walk(syncDir, func(path string, info fs.FileInfo, _ error) error {
- if syncDir == path || pathJSON == info.Name() {
- return nil
- }
-
- if !unchangedList[path] {
- ret[path] = true
- }
- return nil
- })
- return
-}
-
-// calcUnchangedSyncList 获取 data 和 sync 一致（没有修改过）的文件列表，并删除 data 中不存在于 sync 中的多余文件。
-func calcUnchangedSyncList() (ret map[string]bool, removes []string, err error) {
- syncDir := Conf.Sync.GetSaveDir()
- meta := filepath.Join(syncDir, pathJSON)
- if !gulu.File.IsExist(meta) {
- return
- }
- data, err := os.ReadFile(meta)
- if nil != err {
- return
- }
- passwd := Conf.E2EEPasswd
- data, err = encryption.AESGCMDecryptBinBytes(data, passwd)
- if nil != err {
- err = errors.New(Conf.Language(40))
- return
- }
-
- metaJSON := map[string]string{}
- if err = gulu.JSON.UnmarshalJSON(data, &metaJSON); nil != err {
- return
- }
-
- excludes := getSyncExcludedList(syncDir)
- ret = map[string]bool{}
- sep := string(os.PathSeparator)
- filepath.Walk(util.DataDir, func(path string, info fs.FileInfo, _ error) error {
- if util.DataDir == path {
- return nil
- }
-
- if isCloudSkipFile(path, info) {
- if info.IsDir() {
- return filepath.SkipDir
- }
- return nil
- }
-
- plainP := strings.TrimPrefix(path, util.DataDir+sep)
- dataP := plainP
- dataP = pathSha256Short(dataP, sep)
- syncP := filepath.Join(syncDir, dataP)
-
- if excludes[syncP] {
- return nil
- }
-
- if !gulu.File.IsExist(syncP) { // sync 已经删除的文件
- removes = append(removes, path)
- if gulu.File.IsDir(path) {
- return filepath.SkipDir
- }
- return nil
- }
-
- stat, _ := os.Stat(syncP)
- syncModTime := stat.ModTime()
- if info.ModTime() == syncModTime {
- ret[syncP] = true
- return nil
- }
- return nil
- })
-
- // 在 data 中删除 sync 已经删除的文件
- now := time.Now().Format("2006-01-02-150405")
- for _, remove := range removes {
- genSyncHistory(now, remove)
- if ".siyuan" != filepath.Base(remove) {
- if err = os.RemoveAll(remove); nil != err {
- util.LogErrorf("remove [%s] failed: %s", remove, err)
- }
- }
- }
- return
-}
-
func getWorkspaceDataConf() (conf *filesys.DataConf, err error) {
conf = &filesys.DataConf{Updated: util.CurrentTimeMillis(), Device: Conf.System.ID}
confPath := filepath.Join(Conf.Sync.GetSaveDir(), ".siyuan", "conf.json")
@@ -1039,26 +371,6 @@ func incLocalSyncVer() {
return
}
-func isCloudSkipFile(path string, info os.FileInfo) bool {
- baseName := info.Name()
- if strings.HasPrefix(baseName, ".") {
- if ".siyuan" == baseName {
- return false
- }
- return true
- }
- if "history" == baseName {
- if strings.HasSuffix(path, ".siyuan"+string(os.PathSeparator)+"history") {
- return true
- }
- }
-
- if (os.ModeSymlink == info.Mode()&os.ModeType) || (!strings.Contains(path, ".siyuan") && gulu.File.IsHidden(path)) {
- return true
- }
- return false
-}
-
func CreateCloudSyncDir(name string) (err error) {
syncLock.Lock()
defer syncLock.Unlock()
@@ -1087,18 +399,13 @@ func RemoveCloudSyncDir(name string) (err error) {
return
}
- if Conf.Sync.UseDataRepo {
- var cloudInfo *dejavu.CloudInfo
- cloudInfo, err = buildCloudInfo()
- if nil != err {
- return
- }
-
- err = dejavu.RemoveCloudRepo(name, cloudInfo)
- } else {
- err = removeCloudDirPath("sync/" + name)
+ var cloudInfo *dejavu.CloudInfo
+ cloudInfo, err = buildCloudInfo()
+ if nil != err {
+ return
}
+ err = dejavu.RemoveCloudRepo(name, cloudInfo)
if nil != err {
return
}
@@ -1115,17 +422,14 @@ func ListCloudSyncDir() (syncDirs []*Sync, hSize string, err error) {
syncDirs = []*Sync{}
var dirs []map[string]interface{}
var size int64
- if Conf.Sync.UseDataRepo {
- var cloudInfo *dejavu.CloudInfo
- cloudInfo, err = buildCloudInfo()
- if nil != err {
- return
- }
- dirs, size, err = dejavu.GetCloudRepos(cloudInfo)
- } else {
- dirs, size, err = listCloudSyncDirOSS()
+ var cloudInfo *dejavu.CloudInfo
+ cloudInfo, err = buildCloudInfo()
+ if nil != err {
+ return
}
+
+ dirs, size, err = dejavu.GetCloudRepos(cloudInfo)
if nil != err {
return
}
@@ -1143,31 +447,6 @@ func ListCloudSyncDir() (syncDirs []*Sync, hSize string, err error) {
return
}
-func genSyncHistory(now, p string) {
- dir := strings.TrimPrefix(p, util.DataDir+string(os.PathSeparator))
- if strings.Contains(dir, string(os.PathSeparator)) {
- dir = dir[:strings.Index(dir, string(os.PathSeparator))]
- }
-
- if ".siyuan" == dir || ".siyuan" == filepath.Base(p) {
- return
- }
-
- historyDir, err := util.GetHistoryDirNow(now, "sync")
- if nil != err {
- util.LogErrorf("get history dir failed: %s", err)
- return
- }
-
- relativePath := strings.TrimPrefix(p, util.DataDir)
- historyPath := filepath.Join(historyDir, relativePath)
- filelock.ReleaseFileLocks(p)
- if err = gulu.File.Copy(p, historyPath); nil != err {
- util.LogErrorf("gen sync history failed: %s", err)
- return
- }
-}
-
func formatErrorMsg(err error) string {
msg := err.Error()
if strings.Contains(msg, "Permission denied") || strings.Contains(msg, "Access is denied") {
@@ -1199,36 +478,6 @@ func IsValidCloudDirName(cloudDirName string) bool {
return true
}
-func getSyncExcludedList(localDirPath string) (ret map[string]bool) {
- syncIgnoreList := getSyncIgnoreList()
- ret = map[string]bool{}
- for _, p := range syncIgnoreList {
- relPath := p
- relPath = pathSha256Short(relPath, "/")
- relPath = filepath.Join(localDirPath, relPath)
- ret[relPath] = true
- }
- return
-}
-
-func getSyncIgnoreList() (ret []string) {
- lines := getIgnoreLines()
- if 1 > len(lines) {
- return
- }
-
- gi := gitignore.CompileIgnoreLines(lines...)
- filepath.Walk(util.DataDir, func(p string, info os.FileInfo, err error) error {
- p = strings.TrimPrefix(p, util.DataDir+string(os.PathSeparator))
- p = filepath.ToSlash(p)
- if gi.MatchesPath(p) {
- ret = append(ret, p)
- }
- return nil
- })
- return
-}
-
func getIgnoreLines() (ret []string) {
ignore := filepath.Join(util.DataDir, ".siyuan", "syncignore")
err := os.MkdirAll(filepath.Dir(ignore), 0755)
@@ -1259,18 +508,6 @@ func getIgnoreLines() (ret []string) {
return
}
-func pathSha256Short(p, sep string) string {
- buf := bytes.Buffer{}
- parts := strings.Split(p, sep)
- for i, part := range parts {
- buf.WriteString(fmt.Sprintf("%x", sha256.Sum256([]byte(part)))[:7])
- if i < len(parts)-1 {
- buf.WriteString(sep)
- }
- }
- return buf.String()
-}
-
func GetSyncDirection(cloudDirName string) (code int, msg string) { // 0：失败，10：上传，20：下载，30：一致，40：使用数据仓库同步
if !IsSubscriber() {
return
diff --git a/kernel/util/working.go b/kernel/util/working.go
index 81021f14a..293f4d051 100644
--- a/kernel/util/working.go
+++ b/kernel/util/working.go
@@ -139,15 +139,6 @@ func SetBooted() {
LogInfof("kernel booted")
}
-func GetHistoryDirNow(now, suffix string) (ret string, err error) {
- ret = filepath.Join(HistoryDir, now+"-"+suffix)
- if err = os.MkdirAll(ret, 0755); nil != err {
- LogErrorf("make history dir failed: %s", err)
- return
- }
- return
-}
-
func GetHistoryDir(suffix string) (ret string, err error) {
ret = filepath.Join(HistoryDir, time.Now().Format("2006-01-02-150405")+"-"+suffix)
if err = os.MkdirAll(ret, 0755); nil != err {