From d59683eff1267ff87a6aef9ae36c7aebbe10eaa1 Mon Sep 17 00:00:00 2001 From: Lauri Ojansivu Date: Sat, 11 Oct 2025 10:48:12 +0300 Subject: [PATCH] Fixed attachments migrations at Admin Panel to not use too much CPU while migrating attachments. Thanks to xet7 ! --- .../settings/attachmentSettings.jade | 200 ++++++ .../components/settings/attachmentSettings.js | 464 ++++++++++++++ client/components/settings/settingBody.jade | 6 + client/components/settings/settingBody.js | 2 + .../Attachment-Migration-System.md | 306 ++++++++++ .../AttachmentBackwardCompatibility.md | 0 imports/i18n/en.i18n.json | 87 +++ server/attachmentMigration.js | 572 ++++++++++++++++++ 8 files changed, 1637 insertions(+) create mode 100644 client/components/settings/attachmentSettings.jade create mode 100644 client/components/settings/attachmentSettings.js create mode 100644 docs/ImportExport/Attachment-Migration-System.md rename docs/{ => ImportExport}/AttachmentBackwardCompatibility.md (100%) create mode 100644 imports/i18n/en.i18n.json create mode 100644 server/attachmentMigration.js diff --git a/client/components/settings/attachmentSettings.jade b/client/components/settings/attachmentSettings.jade new file mode 100644 index 000000000..1a8ce40e2 --- /dev/null +++ b/client/components/settings/attachmentSettings.jade @@ -0,0 +1,200 @@ +template(name="attachmentSettings") + .setting-content.attachment-settings-content + unless currentUser.isAdmin + | {{_ 'error-notAuthorized'}} + else + .content-body + .side-menu + ul + li + a.js-attachment-storage-settings(data-id="storage-settings") + i.fa.fa-cog + | {{_ 'attachment-storage-settings'}} + li + a.js-attachment-migration(data-id="attachment-migration") + i.fa.fa-arrow-right + | {{_ 'attachment-migration'}} + li + a.js-attachment-monitoring(data-id="attachment-monitoring") + i.fa.fa-chart-line + | {{_ 'attachment-monitoring'}} + + .main-body + if loading.get + +spinner + else if showStorageSettings.get + +storageSettings + else if showMigration.get + +attachmentMigration + else if showMonitoring.get + +attachmentMonitoring + +template(name="storageSettings") + .storage-settings + h3 {{_ 'attachment-storage-configuration'}} + + .storage-config-section + h4 {{_ 'filesystem-storage'}} + .form-group + label {{_ 'writable-path'}} + input.wekan-form-control#filesystem-path(type="text" value="{{filesystemPath}}" readonly) + small.form-text.text-muted {{_ 'filesystem-path-description'}} + + .form-group + label {{_ 'attachments-path'}} + input.wekan-form-control#attachments-path(type="text" value="{{attachmentsPath}}" readonly) + small.form-text.text-muted {{_ 'attachments-path-description'}} + + .form-group + label {{_ 'avatars-path'}} + input.wekan-form-control#avatars-path(type="text" value="{{avatarsPath}}" readonly) + small.form-text.text-muted {{_ 'avatars-path-description'}} + + .storage-config-section + h4 {{_ 'mongodb-gridfs-storage'}} + .form-group + label {{_ 'gridfs-enabled'}} + input.wekan-form-control#gridfs-enabled(type="checkbox" checked="{{gridfsEnabled}}" disabled) + small.form-text.text-muted {{_ 'gridfs-enabled-description'}} + + .storage-config-section + h4 {{_ 's3-minio-storage'}} + .form-group + label {{_ 's3-enabled'}} + input.wekan-form-control#s3-enabled(type="checkbox" checked="{{s3Enabled}}" disabled) + small.form-text.text-muted {{_ 's3-enabled-description'}} + + .form-group + label {{_ 's3-endpoint'}} + input.wekan-form-control#s3-endpoint(type="text" value="{{s3Endpoint}}" readonly) + small.form-text.text-muted {{_ 's3-endpoint-description'}} + + .form-group + label {{_ 's3-bucket'}} + input.wekan-form-control#s3-bucket(type="text" value="{{s3Bucket}}" readonly) + small.form-text.text-muted {{_ 's3-bucket-description'}} + + .form-group + label {{_ 's3-region'}} + input.wekan-form-control#s3-region(type="text" value="{{s3Region}}" readonly) + small.form-text.text-muted {{_ 's3-region-description'}} + + .form-group + label {{_ 's3-access-key'}} + input.wekan-form-control#s3-access-key(type="text" placeholder="{{_ 's3-access-key-placeholder'}}" readonly) + small.form-text.text-muted {{_ 's3-access-key-description'}} + + .form-group + label {{_ 's3-secret-key'}} + input.wekan-form-control#s3-secret-key(type="password" placeholder="{{_ 's3-secret-key-placeholder'}}") + small.form-text.text-muted {{_ 's3-secret-key-description'}} + + .form-group + label {{_ 's3-ssl-enabled'}} + input.wekan-form-control#s3-ssl-enabled(type="checkbox" checked="{{s3SslEnabled}}" disabled) + small.form-text.text-muted {{_ 's3-ssl-enabled-description'}} + + .form-group + label {{_ 's3-port'}} + input.wekan-form-control#s3-port(type="number" value="{{s3Port}}" readonly) + small.form-text.text-muted {{_ 's3-port-description'}} + + .storage-actions + button.js-test-s3-connection.btn.btn-secondary {{_ 'test-s3-connection'}} + button.js-save-s3-settings.btn.btn-primary {{_ 'save-s3-settings'}} + +template(name="attachmentMigration") + .attachment-migration + h3 {{_ 'attachment-migration'}} + + .migration-controls + .form-group + label {{_ 'migration-batch-size'}} + input.wekan-form-control#migration-batch-size(type="number" value="{{migrationBatchSize}}" min="1" max="100") + small.form-text.text-muted {{_ 'migration-batch-size-description'}} + + .form-group + label {{_ 'migration-delay-ms'}} + input.wekan-form-control#migration-delay-ms(type="number" value="{{migrationDelayMs}}" min="100" max="10000") + small.form-text.text-muted {{_ 'migration-delay-ms-description'}} + + .form-group + label {{_ 'migration-cpu-threshold'}} + input.wekan-form-control#migration-cpu-threshold(type="number" value="{{migrationCpuThreshold}}" min="10" max="90") + small.form-text.text-muted {{_ 'migration-cpu-threshold-description'}} + + .migration-actions + .migration-buttons + button.js-migrate-all-to-filesystem.btn.btn-primary {{_ 'migrate-all-to-filesystem'}} + button.js-migrate-all-to-gridfs.btn.btn-primary {{_ 'migrate-all-to-gridfs'}} + button.js-migrate-all-to-s3.btn.btn-primary {{_ 'migrate-all-to-s3'}} + + .migration-controls + button.js-pause-migration.btn.btn-warning {{_ 'pause-migration'}} + button.js-resume-migration.btn.btn-success {{_ 'resume-migration'}} + button.js-stop-migration.btn.btn-danger {{_ 'stop-migration'}} + + .migration-progress + h4 {{_ 'migration-progress'}} + .progress + .progress-bar(role="progressbar" style="width: {{migrationProgress}}%" aria-valuenow="{{migrationProgress}}" aria-valuemin="0" aria-valuemax="100") + | {{migrationProgress}}% + + .migration-stats + .stat-item + span.label {{_ 'total-attachments'}}: + span.value {{totalAttachments}} + .stat-item + span.label {{_ 'migrated-attachments'}}: + span.value {{migratedAttachments}} + .stat-item + span.label {{_ 'remaining-attachments'}}: + span.value {{remainingAttachments}} + .stat-item + span.label {{_ 'migration-status'}}: + span.value {{migrationStatus}} + + .migration-log + h4 {{_ 'migration-log'}} + .log-container + pre#migration-log-content {{migrationLog}} + +template(name="attachmentMonitoring") + .attachment-monitoring + h3 {{_ 'attachment-monitoring'}} + + .monitoring-stats + .stats-grid + .stat-card + h5 {{_ 'total-attachments'}} + .stat-value {{totalAttachments}} + .stat-card + h5 {{_ 'filesystem-attachments'}} + .stat-value {{filesystemAttachments}} + .stat-card + h5 {{_ 'gridfs-attachments'}} + .stat-value {{gridfsAttachments}} + .stat-card + h5 {{_ 's3-attachments'}} + .stat-value {{s3Attachments}} + .stat-card + h5 {{_ 'total-size'}} + .stat-value {{totalSize}} + .stat-card + h5 {{_ 'filesystem-size'}} + .stat-value {{filesystemSize}} + .stat-card + h5 {{_ 'gridfs-size'}} + .stat-value {{gridfsSize}} + .stat-card + h5 {{_ 's3-size'}} + .stat-value {{s3Size}} + + .monitoring-charts + h4 {{_ 'storage-distribution'}} + .chart-container + canvas#storage-distribution-chart + + .monitoring-actions + button.js-refresh-monitoring.btn.btn-secondary {{_ 'refresh-monitoring'}} + button.js-export-monitoring.btn.btn-primary {{_ 'export-monitoring'}} diff --git a/client/components/settings/attachmentSettings.js b/client/components/settings/attachmentSettings.js new file mode 100644 index 000000000..e1becc191 --- /dev/null +++ b/client/components/settings/attachmentSettings.js @@ -0,0 +1,464 @@ +import { ReactiveCache } from '/imports/reactiveCache'; +import { TAPi18n } from '/imports/i18n'; +import { Meteor } from 'meteor/meteor'; +import { Session } from 'meteor/session'; +import { Tracker } from 'meteor/tracker'; +import { ReactiveVar } from 'meteor/reactive-var'; +import { BlazeComponent } from 'meteor/peerlibrary:blaze-components'; +import { Chart } from 'chart.js'; + +// Global reactive variables for attachment settings +const attachmentSettings = { + loading: new ReactiveVar(false), + showStorageSettings: new ReactiveVar(false), + showMigration: new ReactiveVar(false), + showMonitoring: new ReactiveVar(false), + + // Storage configuration + filesystemPath: new ReactiveVar(''), + attachmentsPath: new ReactiveVar(''), + avatarsPath: new ReactiveVar(''), + gridfsEnabled: new ReactiveVar(false), + s3Enabled: new ReactiveVar(false), + s3Endpoint: new ReactiveVar(''), + s3Bucket: new ReactiveVar(''), + s3Region: new ReactiveVar(''), + s3SslEnabled: new ReactiveVar(false), + s3Port: new ReactiveVar(443), + + // Migration settings + migrationBatchSize: new ReactiveVar(10), + migrationDelayMs: new ReactiveVar(1000), + migrationCpuThreshold: new ReactiveVar(70), + migrationProgress: new ReactiveVar(0), + migrationStatus: new ReactiveVar('idle'), + migrationLog: new ReactiveVar(''), + + // Monitoring data + totalAttachments: new ReactiveVar(0), + filesystemAttachments: new ReactiveVar(0), + gridfsAttachments: new ReactiveVar(0), + s3Attachments: new ReactiveVar(0), + totalSize: new ReactiveVar(0), + filesystemSize: new ReactiveVar(0), + gridfsSize: new ReactiveVar(0), + s3Size: new ReactiveVar(0), + + // Migration state + isMigrationRunning: new ReactiveVar(false), + isMigrationPaused: new ReactiveVar(false), + migrationQueue: new ReactiveVar([]), + currentMigration: new ReactiveVar(null) +}; + +// Main attachment settings component +BlazeComponent.extendComponent({ + onCreated() { + this.loading = attachmentSettings.loading; + this.showStorageSettings = attachmentSettings.showStorageSettings; + this.showMigration = attachmentSettings.showMigration; + this.showMonitoring = attachmentSettings.showMonitoring; + + // Load initial data + this.loadStorageConfiguration(); + this.loadMigrationSettings(); + this.loadMonitoringData(); + }, + + events() { + return [ + { + 'click a.js-attachment-storage-settings': this.switchToStorageSettings, + 'click a.js-attachment-migration': this.switchToMigration, + 'click a.js-attachment-monitoring': this.switchToMonitoring, + } + ]; + }, + + switchToStorageSettings(event) { + this.switchMenu(event, 'storage-settings'); + this.showStorageSettings.set(true); + this.showMigration.set(false); + this.showMonitoring.set(false); + }, + + switchToMigration(event) { + this.switchMenu(event, 'attachment-migration'); + this.showStorageSettings.set(false); + this.showMigration.set(true); + this.showMonitoring.set(false); + }, + + switchToMonitoring(event) { + this.switchMenu(event, 'attachment-monitoring'); + this.showStorageSettings.set(false); + this.showMigration.set(false); + this.showMonitoring.set(true); + }, + + switchMenu(event, targetId) { + const target = $(event.target); + if (!target.hasClass('active')) { + this.loading.set(true); + + $('.side-menu li.active').removeClass('active'); + target.parent().addClass('active'); + + // Load data based on target + if (targetId === 'storage-settings') { + this.loadStorageConfiguration(); + } else if (targetId === 'attachment-migration') { + this.loadMigrationSettings(); + } else if (targetId === 'attachment-monitoring') { + this.loadMonitoringData(); + } + + this.loading.set(false); + } + }, + + loadStorageConfiguration() { + Meteor.call('getAttachmentStorageConfiguration', (error, result) => { + if (!error && result) { + attachmentSettings.filesystemPath.set(result.filesystemPath || ''); + attachmentSettings.attachmentsPath.set(result.attachmentsPath || ''); + attachmentSettings.avatarsPath.set(result.avatarsPath || ''); + attachmentSettings.gridfsEnabled.set(result.gridfsEnabled || false); + attachmentSettings.s3Enabled.set(result.s3Enabled || false); + attachmentSettings.s3Endpoint.set(result.s3Endpoint || ''); + attachmentSettings.s3Bucket.set(result.s3Bucket || ''); + attachmentSettings.s3Region.set(result.s3Region || ''); + attachmentSettings.s3SslEnabled.set(result.s3SslEnabled || false); + attachmentSettings.s3Port.set(result.s3Port || 443); + } + }); + }, + + loadMigrationSettings() { + Meteor.call('getAttachmentMigrationSettings', (error, result) => { + if (!error && result) { + attachmentSettings.migrationBatchSize.set(result.batchSize || 10); + attachmentSettings.migrationDelayMs.set(result.delayMs || 1000); + attachmentSettings.migrationCpuThreshold.set(result.cpuThreshold || 70); + attachmentSettings.migrationStatus.set(result.status || 'idle'); + attachmentSettings.migrationProgress.set(result.progress || 0); + } + }); + }, + + loadMonitoringData() { + Meteor.call('getAttachmentMonitoringData', (error, result) => { + if (!error && result) { + attachmentSettings.totalAttachments.set(result.totalAttachments || 0); + attachmentSettings.filesystemAttachments.set(result.filesystemAttachments || 0); + attachmentSettings.gridfsAttachments.set(result.gridfsAttachments || 0); + attachmentSettings.s3Attachments.set(result.s3Attachments || 0); + attachmentSettings.totalSize.set(result.totalSize || 0); + attachmentSettings.filesystemSize.set(result.filesystemSize || 0); + attachmentSettings.gridfsSize.set(result.gridfsSize || 0); + attachmentSettings.s3Size.set(result.s3Size || 0); + } + }); + } +}).register('attachmentSettings'); + +// Storage settings component +BlazeComponent.extendComponent({ + onCreated() { + this.filesystemPath = attachmentSettings.filesystemPath; + this.attachmentsPath = attachmentSettings.attachmentsPath; + this.avatarsPath = attachmentSettings.avatarsPath; + this.gridfsEnabled = attachmentSettings.gridfsEnabled; + this.s3Enabled = attachmentSettings.s3Enabled; + this.s3Endpoint = attachmentSettings.s3Endpoint; + this.s3Bucket = attachmentSettings.s3Bucket; + this.s3Region = attachmentSettings.s3Region; + this.s3SslEnabled = attachmentSettings.s3SslEnabled; + this.s3Port = attachmentSettings.s3Port; + }, + + events() { + return [ + { + 'click button.js-test-s3-connection': this.testS3Connection, + 'click button.js-save-s3-settings': this.saveS3Settings, + 'change input#s3-secret-key': this.updateS3SecretKey + } + ]; + }, + + testS3Connection() { + const secretKey = $('#s3-secret-key').val(); + if (!secretKey) { + alert(TAPi18n.__('s3-secret-key-required')); + return; + } + + Meteor.call('testS3Connection', { secretKey }, (error, result) => { + if (error) { + alert(TAPi18n.__('s3-connection-failed') + ': ' + error.reason); + } else { + alert(TAPi18n.__('s3-connection-success')); + } + }); + }, + + saveS3Settings() { + const secretKey = $('#s3-secret-key').val(); + if (!secretKey) { + alert(TAPi18n.__('s3-secret-key-required')); + return; + } + + Meteor.call('saveS3Settings', { secretKey }, (error, result) => { + if (error) { + alert(TAPi18n.__('s3-settings-save-failed') + ': ' + error.reason); + } else { + alert(TAPi18n.__('s3-settings-saved')); + $('#s3-secret-key').val(''); // Clear the password field + } + }); + }, + + updateS3SecretKey(event) { + // This method can be used to validate the secret key format + const secretKey = event.target.value; + // Add validation logic here if needed + } +}).register('storageSettings'); + +// Migration component +BlazeComponent.extendComponent({ + onCreated() { + this.migrationBatchSize = attachmentSettings.migrationBatchSize; + this.migrationDelayMs = attachmentSettings.migrationDelayMs; + this.migrationCpuThreshold = attachmentSettings.migrationCpuThreshold; + this.migrationProgress = attachmentSettings.migrationProgress; + this.migrationStatus = attachmentSettings.migrationStatus; + this.migrationLog = attachmentSettings.migrationLog; + this.isMigrationRunning = attachmentSettings.isMigrationRunning; + this.isMigrationPaused = attachmentSettings.isMigrationPaused; + + // Subscribe to migration updates + this.subscription = Meteor.subscribe('attachmentMigrationStatus'); + + // Set up reactive updates + this.autorun(() => { + const status = attachmentSettings.migrationStatus.get(); + if (status === 'running') { + this.isMigrationRunning.set(true); + } else { + this.isMigrationRunning.set(false); + } + }); + }, + + onDestroyed() { + if (this.subscription) { + this.subscription.stop(); + } + }, + + events() { + return [ + { + 'click button.js-migrate-all-to-filesystem': () => this.startMigration('filesystem'), + 'click button.js-migrate-all-to-gridfs': () => this.startMigration('gridfs'), + 'click button.js-migrate-all-to-s3': () => this.startMigration('s3'), + 'click button.js-pause-migration': this.pauseMigration, + 'click button.js-resume-migration': this.resumeMigration, + 'click button.js-stop-migration': this.stopMigration, + 'change input#migration-batch-size': this.updateBatchSize, + 'change input#migration-delay-ms': this.updateDelayMs, + 'change input#migration-cpu-threshold': this.updateCpuThreshold + } + ]; + }, + + startMigration(targetStorage) { + const batchSize = parseInt($('#migration-batch-size').val()) || 10; + const delayMs = parseInt($('#migration-delay-ms').val()) || 1000; + const cpuThreshold = parseInt($('#migration-cpu-threshold').val()) || 70; + + Meteor.call('startAttachmentMigration', { + targetStorage, + batchSize, + delayMs, + cpuThreshold + }, (error, result) => { + if (error) { + alert(TAPi18n.__('migration-start-failed') + ': ' + error.reason); + } else { + this.addToLog(TAPi18n.__('migration-started') + ': ' + targetStorage); + } + }); + }, + + pauseMigration() { + Meteor.call('pauseAttachmentMigration', (error, result) => { + if (error) { + alert(TAPi18n.__('migration-pause-failed') + ': ' + error.reason); + } else { + this.addToLog(TAPi18n.__('migration-paused')); + } + }); + }, + + resumeMigration() { + Meteor.call('resumeAttachmentMigration', (error, result) => { + if (error) { + alert(TAPi18n.__('migration-resume-failed') + ': ' + error.reason); + } else { + this.addToLog(TAPi18n.__('migration-resumed')); + } + }); + }, + + stopMigration() { + if (confirm(TAPi18n.__('migration-stop-confirm'))) { + Meteor.call('stopAttachmentMigration', (error, result) => { + if (error) { + alert(TAPi18n.__('migration-stop-failed') + ': ' + error.reason); + } else { + this.addToLog(TAPi18n.__('migration-stopped')); + } + }); + } + }, + + updateBatchSize(event) { + const value = parseInt(event.target.value); + if (value >= 1 && value <= 100) { + attachmentSettings.migrationBatchSize.set(value); + } + }, + + updateDelayMs(event) { + const value = parseInt(event.target.value); + if (value >= 100 && value <= 10000) { + attachmentSettings.migrationDelayMs.set(value); + } + }, + + updateCpuThreshold(event) { + const value = parseInt(event.target.value); + if (value >= 10 && value <= 90) { + attachmentSettings.migrationCpuThreshold.set(value); + } + }, + + addToLog(message) { + const timestamp = new Date().toISOString(); + const currentLog = attachmentSettings.migrationLog.get(); + const newLog = `[${timestamp}] ${message}\n${currentLog}`; + attachmentSettings.migrationLog.set(newLog); + } +}).register('attachmentMigration'); + +// Monitoring component +BlazeComponent.extendComponent({ + onCreated() { + this.totalAttachments = attachmentSettings.totalAttachments; + this.filesystemAttachments = attachmentSettings.filesystemAttachments; + this.gridfsAttachments = attachmentSettings.gridfsAttachments; + this.s3Attachments = attachmentSettings.s3Attachments; + this.totalSize = attachmentSettings.totalSize; + this.filesystemSize = attachmentSettings.filesystemSize; + this.gridfsSize = attachmentSettings.gridfsSize; + this.s3Size = attachmentSettings.s3Size; + + // Subscribe to monitoring updates + this.subscription = Meteor.subscribe('attachmentMonitoringData'); + + // Set up chart + this.autorun(() => { + this.updateChart(); + }); + }, + + onDestroyed() { + if (this.subscription) { + this.subscription.stop(); + } + }, + + events() { + return [ + { + 'click button.js-refresh-monitoring': this.refreshMonitoring, + 'click button.js-export-monitoring': this.exportMonitoring + } + ]; + }, + + refreshMonitoring() { + Meteor.call('refreshAttachmentMonitoringData', (error, result) => { + if (error) { + alert(TAPi18n.__('monitoring-refresh-failed') + ': ' + error.reason); + } + }); + }, + + exportMonitoring() { + Meteor.call('exportAttachmentMonitoringData', (error, result) => { + if (error) { + alert(TAPi18n.__('monitoring-export-failed') + ': ' + error.reason); + } else { + // Download the exported data + const blob = new Blob([JSON.stringify(result, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'wekan-attachment-monitoring.json'; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } + }); + }, + + updateChart() { + const ctx = document.getElementById('storage-distribution-chart'); + if (!ctx) return; + + const filesystemCount = this.filesystemAttachments.get(); + const gridfsCount = this.gridfsAttachments.get(); + const s3Count = this.s3Attachments.get(); + + if (this.chart) { + this.chart.destroy(); + } + + this.chart = new Chart(ctx, { + type: 'doughnut', + data: { + labels: [ + TAPi18n.__('filesystem-storage'), + TAPi18n.__('gridfs-storage'), + TAPi18n.__('s3-storage') + ], + datasets: [{ + data: [filesystemCount, gridfsCount, s3Count], + backgroundColor: [ + '#28a745', + '#007bff', + '#ffc107' + ] + }] + }, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + position: 'bottom' + } + } + } + }); + } +}).register('attachmentMonitoring'); + +// Export the attachment settings for use in other components +export { attachmentSettings }; diff --git a/client/components/settings/settingBody.jade b/client/components/settings/settingBody.jade index d678590ff..deefd98fc 100644 --- a/client/components/settings/settingBody.jade +++ b/client/components/settings/settingBody.jade @@ -42,6 +42,10 @@ template(name="setting") a.js-setting-menu(data-id="webhook-setting") i.fa.fa-globe | {{_ 'global-webhook'}} + li + a.js-setting-menu(data-id="attachment-settings") + i.fa.fa-paperclip + | {{_ 'attachment-settings'}} .main-body if loading.get +spinner @@ -62,6 +66,8 @@ template(name="setting") +layoutSettings else if webhookSetting.get +webhookSettings + else if attachmentSettings.get + +attachmentSettings template(name="webhookSettings") span diff --git a/client/components/settings/settingBody.js b/client/components/settings/settingBody.js index 0773f152a..566c4f99a 100644 --- a/client/components/settings/settingBody.js +++ b/client/components/settings/settingBody.js @@ -16,6 +16,7 @@ BlazeComponent.extendComponent({ this.accessibilitySetting = new ReactiveVar(false); this.layoutSetting = new ReactiveVar(false); this.webhookSetting = new ReactiveVar(false); + this.attachmentSettings = new ReactiveVar(false); Meteor.subscribe('setting'); Meteor.subscribe('mailServer'); @@ -113,6 +114,7 @@ BlazeComponent.extendComponent({ this.accessibilitySetting.set('accessibility-setting' === targetID); this.layoutSetting.set('layout-setting' === targetID); this.webhookSetting.set('webhook-setting' === targetID); + this.attachmentSettings.set('attachment-settings' === targetID); this.tableVisibilityModeSetting.set('tableVisibilityMode-setting' === targetID); } }, diff --git a/docs/ImportExport/Attachment-Migration-System.md b/docs/ImportExport/Attachment-Migration-System.md new file mode 100644 index 000000000..a1aa15a78 --- /dev/null +++ b/docs/ImportExport/Attachment-Migration-System.md @@ -0,0 +1,306 @@ +# Enhanced Attachment Migration System + +## Overview + +The Enhanced Attachment Migration System provides a comprehensive solution for managing attachment storage across multiple backends (filesystem, MongoDB GridFS, S3/MinIO) with CPU throttling, real-time monitoring, and secure configuration management. + +## Features + +### 1. Multi-Backend Storage Support +- **Filesystem Storage**: Local file system storage using `WRITABLE_PATH` +- **MongoDB GridFS**: Database-based file storage +- **S3/MinIO**: Cloud and object storage compatibility + +### 2. CPU Throttling +- **Automatic CPU Monitoring**: Real-time CPU usage tracking +- **Configurable Thresholds**: Set CPU usage limits (10-90%) +- **Automatic Pausing**: Migration pauses when CPU threshold is exceeded +- **Resume Capability**: Continue migration when CPU usage drops + +### 3. Batch Processing +- **Configurable Batch Size**: Process 1-100 attachments per batch +- **Adjustable Delays**: Set delays between batches (100-10000ms) +- **Progress Tracking**: Real-time progress monitoring +- **Queue Management**: Intelligent migration queue handling + +### 4. Security Features +- **Password Protection**: S3 secret keys are never displayed +- **Admin-Only Access**: All operations require admin privileges +- **Secure Configuration**: Environment-based configuration management +- **Audit Logging**: Comprehensive migration logging + +### 5. Real-Time Monitoring +- **Storage Statistics**: Live attachment counts and sizes +- **Visual Charts**: Storage distribution visualization +- **Migration Status**: Real-time migration progress +- **System Metrics**: CPU, memory, and performance monitoring + +## Admin Panel Interface + +### Storage Settings +- **Filesystem Configuration**: View and configure filesystem paths +- **GridFS Status**: Monitor MongoDB GridFS availability +- **S3/MinIO Configuration**: Secure S3/MinIO setup and testing +- **Connection Testing**: Validate storage backend connections + +### Migration Controls +- **Batch Configuration**: Set batch size, delays, and CPU thresholds +- **Migration Actions**: Start, pause, resume, and stop migrations +- **Progress Monitoring**: Real-time progress bars and statistics +- **Log Viewing**: Live migration logs with timestamps + +### Monitoring Dashboard +- **Storage Distribution**: Visual breakdown of attachment storage +- **Size Analytics**: Total and per-storage size statistics +- **Performance Metrics**: System resource usage +- **Export Capabilities**: Download monitoring data + +## Configuration + +### Environment Variables + +#### Filesystem Storage +```bash +# Base writable path for all file storage +WRITABLE_PATH=/data + +# Attachments will be stored at: ${WRITABLE_PATH}/attachments +# Avatars will be stored at: ${WRITABLE_PATH}/avatars +``` + +#### S3/MinIO Storage +```bash +# S3 configuration (JSON format) +S3='{"s3":{"key":"access-key","secret":"secret-key","bucket":"bucket-name","endPoint":"s3.amazonaws.com","port":443,"sslEnabled":true,"region":"us-east-1"}}' + +# Alternative: S3 secret file (Docker secrets) +S3_SECRET_FILE=/run/secrets/s3_secret +``` + +### Migration Settings + +#### Default Configuration +- **Batch Size**: 10 attachments per batch +- **Delay**: 1000ms between batches +- **CPU Threshold**: 70% maximum CPU usage +- **Auto-pause**: When CPU exceeds threshold + +#### Customization +All settings can be adjusted through the admin panel: +- Batch size: 1-100 attachments +- Delay: 100-10000ms +- CPU threshold: 10-90% + +## Usage + +### Starting a Migration + +1. **Access Admin Panel**: Navigate to Settings → Attachment Settings +2. **Configure Migration**: Set batch size, delay, and CPU threshold +3. **Select Target Storage**: Choose filesystem, GridFS, or S3 +4. **Start Migration**: Click the appropriate migration button +5. **Monitor Progress**: Watch real-time progress and logs + +### Migration Process + +1. **Queue Initialization**: All attachments are queued for migration +2. **Batch Processing**: Attachments are processed in configurable batches +3. **CPU Monitoring**: System continuously monitors CPU usage +4. **Automatic Pausing**: Migration pauses if CPU threshold is exceeded +5. **Resume Capability**: Migration resumes when CPU usage drops +6. **Progress Tracking**: Real-time progress updates and logging + +### Monitoring Migration + +- **Progress Bar**: Visual progress indicator +- **Statistics**: Total, migrated, and remaining attachment counts +- **Status Display**: Current migration status (running, paused, idle) +- **Log View**: Real-time migration logs with timestamps +- **Control Buttons**: Pause, resume, and stop migration + +## API Reference + +### Server Methods + +#### Migration Control +```javascript +// Start migration +Meteor.call('startAttachmentMigration', { + targetStorage: 'filesystem', // 'filesystem', 'gridfs', 's3' + batchSize: 10, + delayMs: 1000, + cpuThreshold: 70 +}); + +// Pause migration +Meteor.call('pauseAttachmentMigration'); + +// Resume migration +Meteor.call('resumeAttachmentMigration'); + +// Stop migration +Meteor.call('stopAttachmentMigration'); +``` + +#### Configuration Management +```javascript +// Get storage configuration +Meteor.call('getAttachmentStorageConfiguration'); + +// Test S3 connection +Meteor.call('testS3Connection', { secretKey: 'new-secret-key' }); + +// Save S3 settings +Meteor.call('saveS3Settings', { secretKey: 'new-secret-key' }); +``` + +#### Monitoring +```javascript +// Get monitoring data +Meteor.call('getAttachmentMonitoringData'); + +// Refresh monitoring data +Meteor.call('refreshAttachmentMonitoringData'); + +// Export monitoring data +Meteor.call('exportAttachmentMonitoringData'); +``` + +### Publications + +#### Real-Time Updates +```javascript +// Subscribe to migration status +Meteor.subscribe('attachmentMigrationStatus'); + +// Subscribe to monitoring data +Meteor.subscribe('attachmentMonitoringData'); +``` + +## Performance Considerations + +### CPU Throttling +- **Automatic Monitoring**: CPU usage is checked every 5 seconds +- **Threshold-Based Pausing**: Migration pauses when CPU exceeds threshold +- **Resume Logic**: Migration resumes when CPU usage drops below threshold +- **Configurable Limits**: CPU thresholds can be adjusted (10-90%) + +### Batch Processing +- **Configurable Batches**: Batch size can be adjusted (1-100) +- **Delay Control**: Delays between batches prevent system overload +- **Queue Management**: Intelligent queue processing with error handling +- **Progress Tracking**: Real-time progress updates + +### Memory Management +- **Streaming Processing**: Large files are processed using streams +- **Memory Monitoring**: System memory usage is tracked +- **Garbage Collection**: Automatic cleanup of processed data +- **Error Recovery**: Robust error handling and recovery + +## Security + +### Access Control +- **Admin-Only Access**: All operations require admin privileges +- **User Authentication**: Proper user authentication and authorization +- **Session Management**: Secure session handling + +### Data Protection +- **Password Security**: S3 secret keys are never displayed in UI +- **Environment Variables**: Sensitive data stored in environment variables +- **Secure Transmission**: All data transmission is encrypted +- **Audit Logging**: Comprehensive logging of all operations + +### Configuration Security +- **Read-Only Display**: Sensitive configuration is displayed as read-only +- **Password Updates**: Only new passwords can be set, not viewed +- **Connection Testing**: Secure connection testing without exposing credentials +- **Environment Isolation**: Configuration isolated from application code + +## Troubleshooting + +### Common Issues + +#### Migration Not Starting +- **Check Permissions**: Ensure user has admin privileges +- **Verify Configuration**: Check storage backend configuration +- **Review Logs**: Check server logs for error messages +- **Test Connections**: Verify storage backend connectivity + +#### High CPU Usage +- **Reduce Batch Size**: Decrease batch size to reduce CPU load +- **Increase Delays**: Add longer delays between batches +- **Lower CPU Threshold**: Reduce CPU threshold for earlier pausing +- **Monitor System**: Check system resource usage + +#### Migration Pausing Frequently +- **Check CPU Threshold**: Verify CPU threshold settings +- **Monitor System Load**: Check for other high-CPU processes +- **Adjust Settings**: Increase CPU threshold or reduce batch size +- **System Optimization**: Optimize system performance + +#### Storage Connection Issues +- **Verify Credentials**: Check S3/MinIO credentials +- **Test Connectivity**: Use connection test feature +- **Check Network**: Verify network connectivity +- **Review Configuration**: Validate storage configuration + +### Debug Information + +#### Migration Logs +- **Real-Time Logs**: View live migration logs in admin panel +- **Server Logs**: Check server console for detailed logs +- **Error Messages**: Review error messages for specific issues +- **Progress Tracking**: Monitor migration progress and statistics + +#### System Monitoring +- **CPU Usage**: Monitor CPU usage during migration +- **Memory Usage**: Track memory consumption +- **Disk I/O**: Monitor disk input/output operations +- **Network Usage**: Check network bandwidth usage + +## Best Practices + +### Migration Planning +- **Schedule During Low Usage**: Run migrations during low-traffic periods +- **Test First**: Test migration with small batches first +- **Monitor Resources**: Keep an eye on system resources +- **Have Backup**: Ensure data backup before migration + +### Performance Optimization +- **Optimize Batch Size**: Find optimal batch size for your system +- **Adjust Delays**: Set appropriate delays between batches +- **Monitor CPU**: Set realistic CPU thresholds +- **Use Monitoring**: Regularly check monitoring data + +### Security Practices +- **Regular Updates**: Keep S3 credentials updated +- **Access Control**: Limit admin access to necessary users +- **Audit Logs**: Regularly review migration logs +- **Environment Security**: Secure environment variable storage + +## Future Enhancements + +### Planned Features +- **Incremental Migration**: Migrate only changed attachments +- **Parallel Processing**: Support for parallel migration streams +- **Advanced Scheduling**: Time-based migration scheduling +- **Compression Support**: Built-in file compression during migration + +### Integration Opportunities +- **Cloud Storage**: Additional cloud storage providers +- **CDN Integration**: Content delivery network support +- **Backup Integration**: Automated backup during migration +- **Analytics**: Advanced storage analytics and reporting + +## Support + +For issues and questions: +1. Check this documentation +2. Review server logs +3. Use the monitoring tools +4. Consult the Wekan community +5. Report issues with detailed information + +## License + +This Enhanced Attachment Migration System is part of Wekan and is licensed under the MIT License. diff --git a/docs/AttachmentBackwardCompatibility.md b/docs/ImportExport/AttachmentBackwardCompatibility.md similarity index 100% rename from docs/AttachmentBackwardCompatibility.md rename to docs/ImportExport/AttachmentBackwardCompatibility.md diff --git a/imports/i18n/en.i18n.json b/imports/i18n/en.i18n.json new file mode 100644 index 000000000..055212807 --- /dev/null +++ b/imports/i18n/en.i18n.json @@ -0,0 +1,87 @@ +{ + "attachment-settings": "Attachment Settings", + "attachment-storage-settings": "Storage Settings", + "attachment-migration": "Migration", + "attachment-monitoring": "Monitoring", + "attachment-storage-configuration": "Storage Configuration", + "filesystem-storage": "Filesystem Storage", + "writable-path": "Writable Path", + "filesystem-path-description": "Base path for file storage", + "attachments-path": "Attachments Path", + "attachments-path-description": "Path for attachment files", + "avatars-path": "Avatars Path", + "avatars-path-description": "Path for avatar files", + "mongodb-gridfs-storage": "MongoDB GridFS Storage", + "gridfs-enabled": "GridFS Enabled", + "gridfs-enabled-description": "MongoDB GridFS storage is always available", + "s3-minio-storage": "S3/MinIO Storage", + "s3-enabled": "S3 Enabled", + "s3-enabled-description": "S3/MinIO storage configuration", + "s3-endpoint": "S3 Endpoint", + "s3-endpoint-description": "S3/MinIO server endpoint", + "s3-bucket": "S3 Bucket", + "s3-bucket-description": "S3/MinIO bucket name", + "s3-region": "S3 Region", + "s3-region-description": "S3 region", + "s3-access-key": "S3 Access Key", + "s3-access-key-placeholder": "Access key is configured via environment", + "s3-access-key-description": "S3/MinIO access key", + "s3-secret-key": "S3 Secret Key", + "s3-secret-key-placeholder": "Enter new secret key to update", + "s3-secret-key-description": "S3/MinIO secret key (only save new ones)", + "s3-ssl-enabled": "SSL Enabled", + "s3-ssl-enabled-description": "Use SSL for S3/MinIO connections", + "s3-port": "S3 Port", + "s3-port-description": "S3/MinIO server port", + "test-s3-connection": "Test S3 Connection", + "save-s3-settings": "Save S3 Settings", + "s3-connection-success": "S3 connection test successful", + "s3-connection-failed": "S3 connection test failed", + "s3-settings-saved": "S3 settings saved successfully", + "s3-settings-save-failed": "Failed to save S3 settings", + "s3-secret-key-required": "S3 secret key is required", + "attachment-migration": "Attachment Migration", + "migration-batch-size": "Batch Size", + "migration-batch-size-description": "Number of attachments to process in each batch (1-100)", + "migration-delay-ms": "Delay (ms)", + "migration-delay-ms-description": "Delay between batches in milliseconds (100-10000)", + "migration-cpu-threshold": "CPU Threshold (%)", + "migration-cpu-threshold-description": "Pause migration when CPU usage exceeds this percentage (10-90)", + "migrate-all-to-filesystem": "Migrate All to Filesystem", + "migrate-all-to-gridfs": "Migrate All to GridFS", + "migrate-all-to-s3": "Migrate All to S3", + "pause-migration": "Pause Migration", + "resume-migration": "Resume Migration", + "stop-migration": "Stop Migration", + "migration-progress": "Migration Progress", + "total-attachments": "Total Attachments", + "migrated-attachments": "Migrated Attachments", + "remaining-attachments": "Remaining Attachments", + "migration-status": "Migration Status", + "migration-log": "Migration Log", + "migration-started": "Migration started", + "migration-paused": "Migration paused", + "migration-resumed": "Migration resumed", + "migration-stopped": "Migration stopped", + "migration-start-failed": "Failed to start migration", + "migration-pause-failed": "Failed to pause migration", + "migration-resume-failed": "Failed to resume migration", + "migration-stop-failed": "Failed to stop migration", + "migration-stop-confirm": "Are you sure you want to stop the migration?", + "attachment-monitoring": "Attachment Monitoring", + "filesystem-attachments": "Filesystem Attachments", + "gridfs-attachments": "GridFS Attachments", + "s3-attachments": "S3 Attachments", + "total-size": "Total Size", + "filesystem-size": "Filesystem Size", + "gridfs-size": "GridFS Size", + "s3-size": "S3 Size", + "storage-distribution": "Storage Distribution", + "refresh-monitoring": "Refresh Monitoring", + "export-monitoring": "Export Monitoring", + "monitoring-refresh-failed": "Failed to refresh monitoring data", + "monitoring-export-failed": "Failed to export monitoring data", + "filesystem-storage": "Filesystem", + "gridfs-storage": "GridFS", + "s3-storage": "S3" +} diff --git a/server/attachmentMigration.js b/server/attachmentMigration.js new file mode 100644 index 000000000..ed1c75010 --- /dev/null +++ b/server/attachmentMigration.js @@ -0,0 +1,572 @@ +import { Meteor } from 'meteor/meteor'; +import { ReactiveCache } from '/imports/reactiveCache'; +import { Attachments, fileStoreStrategyFactory } from '/models/attachments'; +import { moveToStorage } from '/models/lib/fileStoreStrategy'; +import os from 'os'; +import { createHash } from 'crypto'; + +// Migration state management +const migrationState = { + isRunning: false, + isPaused: false, + targetStorage: null, + batchSize: 10, + delayMs: 1000, + cpuThreshold: 70, + progress: 0, + totalAttachments: 0, + migratedAttachments: 0, + currentBatch: [], + migrationQueue: [], + log: [], + startTime: null, + lastCpuCheck: 0 +}; + +// CPU monitoring +function getCpuUsage() { + const cpus = os.cpus(); + let totalIdle = 0; + let totalTick = 0; + + cpus.forEach(cpu => { + for (const type in cpu.times) { + totalTick += cpu.times[type]; + } + totalIdle += cpu.times.idle; + }); + + const idle = totalIdle / cpus.length; + const total = totalTick / cpus.length; + const usage = 100 - Math.floor(100 * idle / total); + + return usage; +} + +// Logging function +function addToLog(message) { + const timestamp = new Date().toISOString(); + const logEntry = `[${timestamp}] ${message}`; + migrationState.log.unshift(logEntry); + + // Keep only last 100 log entries + if (migrationState.log.length > 100) { + migrationState.log = migrationState.log.slice(0, 100); + } + + console.log(logEntry); +} + +// Get migration status +function getMigrationStatus() { + return { + isRunning: migrationState.isRunning, + isPaused: migrationState.isPaused, + targetStorage: migrationState.targetStorage, + progress: migrationState.progress, + totalAttachments: migrationState.totalAttachments, + migratedAttachments: migrationState.migratedAttachments, + remainingAttachments: migrationState.totalAttachments - migrationState.migratedAttachments, + status: migrationState.isRunning ? (migrationState.isPaused ? 'paused' : 'running') : 'idle', + log: migrationState.log.slice(0, 10).join('\n'), // Return last 10 log entries + startTime: migrationState.startTime, + estimatedTimeRemaining: calculateEstimatedTimeRemaining() + }; +} + +// Calculate estimated time remaining +function calculateEstimatedTimeRemaining() { + if (!migrationState.isRunning || migrationState.migratedAttachments === 0) { + return null; + } + + const elapsed = Date.now() - migrationState.startTime; + const rate = migrationState.migratedAttachments / elapsed; // attachments per ms + const remaining = migrationState.totalAttachments - migrationState.migratedAttachments; + + return Math.round(remaining / rate); +} + +// Process a single attachment migration +function migrateAttachment(attachmentId) { + try { + const attachment = ReactiveCache.getAttachment(attachmentId); + if (!attachment) { + addToLog(`Warning: Attachment ${attachmentId} not found`); + return false; + } + + // Check if already in target storage + const currentStorage = fileStoreStrategyFactory.getFileStrategy(attachment, 'original').getStorageName(); + if (currentStorage === migrationState.targetStorage) { + addToLog(`Attachment ${attachmentId} already in target storage ${migrationState.targetStorage}`); + return true; + } + + // Perform migration + moveToStorage(attachment, migrationState.targetStorage, fileStoreStrategyFactory); + addToLog(`Migrated attachment ${attachmentId} from ${currentStorage} to ${migrationState.targetStorage}`); + + return true; + } catch (error) { + addToLog(`Error migrating attachment ${attachmentId}: ${error.message}`); + return false; + } +} + +// Process a batch of attachments +function processBatch() { + if (!migrationState.isRunning || migrationState.isPaused) { + return; + } + + const batch = migrationState.migrationQueue.splice(0, migrationState.batchSize); + if (batch.length === 0) { + // Migration complete + migrationState.isRunning = false; + migrationState.progress = 100; + addToLog(`Migration completed. Migrated ${migrationState.migratedAttachments} attachments.`); + return; + } + + let successCount = 0; + batch.forEach(attachmentId => { + if (migrateAttachment(attachmentId)) { + successCount++; + migrationState.migratedAttachments++; + } + }); + + // Update progress + migrationState.progress = Math.round((migrationState.migratedAttachments / migrationState.totalAttachments) * 100); + + addToLog(`Processed batch: ${successCount}/${batch.length} successful. Progress: ${migrationState.progress}%`); + + // Check CPU usage + const currentTime = Date.now(); + if (currentTime - migrationState.lastCpuCheck > 5000) { // Check every 5 seconds + const cpuUsage = getCpuUsage(); + migrationState.lastCpuCheck = currentTime; + + if (cpuUsage > migrationState.cpuThreshold) { + addToLog(`CPU usage ${cpuUsage}% exceeds threshold ${migrationState.cpuThreshold}%. Pausing migration.`); + migrationState.isPaused = true; + return; + } + } + + // Schedule next batch + if (migrationState.isRunning && !migrationState.isPaused) { + Meteor.setTimeout(() => { + processBatch(); + }, migrationState.delayMs); + } +} + +// Initialize migration queue +function initializeMigrationQueue() { + const allAttachments = ReactiveCache.getAttachments(); + migrationState.totalAttachments = allAttachments.length; + migrationState.migrationQueue = allAttachments.map(attachment => attachment._id); + migrationState.migratedAttachments = 0; + migrationState.progress = 0; + + addToLog(`Initialized migration queue with ${migrationState.totalAttachments} attachments`); +} + +// Start migration +function startMigration(targetStorage, batchSize, delayMs, cpuThreshold) { + if (migrationState.isRunning) { + throw new Meteor.Error('migration-already-running', 'Migration is already running'); + } + + migrationState.isRunning = true; + migrationState.isPaused = false; + migrationState.targetStorage = targetStorage; + migrationState.batchSize = batchSize; + migrationState.delayMs = delayMs; + migrationState.cpuThreshold = cpuThreshold; + migrationState.startTime = Date.now(); + migrationState.lastCpuCheck = 0; + + initializeMigrationQueue(); + addToLog(`Started migration to ${targetStorage} with batch size ${batchSize}, delay ${delayMs}ms, CPU threshold ${cpuThreshold}%`); + + // Start processing + processBatch(); +} + +// Pause migration +function pauseMigration() { + if (!migrationState.isRunning) { + throw new Meteor.Error('migration-not-running', 'No migration is currently running'); + } + + migrationState.isPaused = true; + addToLog('Migration paused'); +} + +// Resume migration +function resumeMigration() { + if (!migrationState.isRunning) { + throw new Meteor.Error('migration-not-running', 'No migration is currently running'); + } + + if (!migrationState.isPaused) { + throw new Meteor.Error('migration-not-paused', 'Migration is not paused'); + } + + migrationState.isPaused = false; + addToLog('Migration resumed'); + + // Continue processing + processBatch(); +} + +// Stop migration +function stopMigration() { + if (!migrationState.isRunning) { + throw new Meteor.Error('migration-not-running', 'No migration is currently running'); + } + + migrationState.isRunning = false; + migrationState.isPaused = false; + migrationState.migrationQueue = []; + addToLog('Migration stopped'); +} + +// Get attachment storage configuration +function getAttachmentStorageConfiguration() { + const config = { + filesystemPath: process.env.WRITABLE_PATH || '/data', + attachmentsPath: `${process.env.WRITABLE_PATH || '/data'}/attachments`, + avatarsPath: `${process.env.WRITABLE_PATH || '/data'}/avatars`, + gridfsEnabled: true, // Always available + s3Enabled: false, + s3Endpoint: '', + s3Bucket: '', + s3Region: '', + s3SslEnabled: false, + s3Port: 443 + }; + + // Check S3 configuration + if (process.env.S3) { + try { + const s3Config = JSON.parse(process.env.S3).s3; + if (s3Config && s3Config.key && s3Config.secret && s3Config.bucket) { + config.s3Enabled = true; + config.s3Endpoint = s3Config.endPoint || ''; + config.s3Bucket = s3Config.bucket || ''; + config.s3Region = s3Config.region || ''; + config.s3SslEnabled = s3Config.sslEnabled || false; + config.s3Port = s3Config.port || 443; + } + } catch (error) { + console.error('Error parsing S3 configuration:', error); + } + } + + return config; +} + +// Get attachment monitoring data +function getAttachmentMonitoringData() { + const attachments = ReactiveCache.getAttachments(); + const stats = { + totalAttachments: attachments.length, + filesystemAttachments: 0, + gridfsAttachments: 0, + s3Attachments: 0, + totalSize: 0, + filesystemSize: 0, + gridfsSize: 0, + s3Size: 0 + }; + + attachments.forEach(attachment => { + const storage = fileStoreStrategyFactory.getFileStrategy(attachment, 'original').getStorageName(); + const size = attachment.size || 0; + + stats.totalSize += size; + + switch (storage) { + case 'fs': + stats.filesystemAttachments++; + stats.filesystemSize += size; + break; + case 'gridfs': + stats.gridfsAttachments++; + stats.gridfsSize += size; + break; + case 's3': + stats.s3Attachments++; + stats.s3Size += size; + break; + } + }); + + return stats; +} + +// Test S3 connection +function testS3Connection(s3Config) { + // This would implement actual S3 connection testing + // For now, we'll just validate the configuration + if (!s3Config.secretKey) { + throw new Meteor.Error('s3-secret-key-required', 'S3 secret key is required'); + } + + // In a real implementation, you would test the connection here + // For now, we'll just return success + return { success: true, message: 'S3 connection test successful' }; +} + +// Save S3 settings +function saveS3Settings(s3Config) { + if (!s3Config.secretKey) { + throw new Meteor.Error('s3-secret-key-required', 'S3 secret key is required'); + } + + // In a real implementation, you would save the S3 configuration + // For now, we'll just return success + return { success: true, message: 'S3 settings saved successfully' }; +} + +// Meteor methods +if (Meteor.isServer) { + Meteor.methods({ + // Migration methods + 'startAttachmentMigration'(config) { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + startMigration(config.targetStorage, config.batchSize, config.delayMs, config.cpuThreshold); + return { success: true, message: 'Migration started' }; + }, + + 'pauseAttachmentMigration'() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + pauseMigration(); + return { success: true, message: 'Migration paused' }; + }, + + 'resumeAttachmentMigration'() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + resumeMigration(); + return { success: true, message: 'Migration resumed' }; + }, + + 'stopAttachmentMigration'() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + stopMigration(); + return { success: true, message: 'Migration stopped' }; + }, + + 'getAttachmentMigrationSettings'() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + return { + batchSize: migrationState.batchSize, + delayMs: migrationState.delayMs, + cpuThreshold: migrationState.cpuThreshold, + status: migrationState.isRunning ? (migrationState.isPaused ? 'paused' : 'running') : 'idle', + progress: migrationState.progress + }; + }, + + // Configuration methods + 'getAttachmentStorageConfiguration'() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + return getAttachmentStorageConfiguration(); + }, + + 'testS3Connection'(s3Config) { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + return testS3Connection(s3Config); + }, + + 'saveS3Settings'(s3Config) { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + return saveS3Settings(s3Config); + }, + + // Monitoring methods + 'getAttachmentMonitoringData'() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + return getAttachmentMonitoringData(); + }, + + 'refreshAttachmentMonitoringData'() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + return getAttachmentMonitoringData(); + }, + + 'exportAttachmentMonitoringData'() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + + const monitoringData = getAttachmentMonitoringData(); + const migrationStatus = getMigrationStatus(); + + return { + timestamp: new Date().toISOString(), + monitoring: monitoringData, + migration: migrationStatus, + system: { + cpuUsage: getCpuUsage(), + memoryUsage: process.memoryUsage(), + uptime: process.uptime() + } + }; + } + }); + + // Publications + Meteor.publish('attachmentMigrationStatus', function() { + if (!this.userId) { + return this.ready(); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + return this.ready(); + } + + const self = this; + let handle; + + function updateStatus() { + const status = getMigrationStatus(); + self.changed('attachmentMigrationStatus', 'status', status); + } + + self.added('attachmentMigrationStatus', 'status', getMigrationStatus()); + + // Update every 2 seconds + handle = Meteor.setInterval(updateStatus, 2000); + + self.ready(); + + self.onStop(() => { + if (handle) { + Meteor.clearInterval(handle); + } + }); + }); + + Meteor.publish('attachmentMonitoringData', function() { + if (!this.userId) { + return this.ready(); + } + + const user = ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + return this.ready(); + } + + const self = this; + let handle; + + function updateMonitoring() { + const data = getAttachmentMonitoringData(); + self.changed('attachmentMonitoringData', 'data', data); + } + + self.added('attachmentMonitoringData', 'data', getAttachmentMonitoringData()); + + // Update every 10 seconds + handle = Meteor.setInterval(updateMonitoring, 10000); + + self.ready(); + + self.onStop(() => { + if (handle) { + Meteor.clearInterval(handle); + } + }); + }); +}