mirror of
https://github.com/wekan/wekan.git
synced 2025-12-16 15:30:13 +01:00
Fixed attachments migrations at Admin Panel to not use too much CPU while migrating attachments.
Thanks to xet7 !
This commit is contained in:
Lauri Ojansivu
parent
de77776cd0
commit
d59683eff1
8 changed files with 1637 additions and 0 deletions
200
client/components/settings/attachmentSettings.jade
Normal file
200
client/components/settings/attachmentSettings.jade
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
template(name="attachmentSettings")
|
||||
.setting-content.attachment-settings-content
|
||||
unless currentUser.isAdmin
|
||||
| {{_ 'error-notAuthorized'}}
|
||||
else
|
||||
.content-body
|
||||
.side-menu
|
||||
ul
|
||||
li
|
||||
a.js-attachment-storage-settings(data-id="storage-settings")
|
||||
i.fa.fa-cog
|
||||
| {{_ 'attachment-storage-settings'}}
|
||||
li
|
||||
a.js-attachment-migration(data-id="attachment-migration")
|
||||
i.fa.fa-arrow-right
|
||||
| {{_ 'attachment-migration'}}
|
||||
li
|
||||
a.js-attachment-monitoring(data-id="attachment-monitoring")
|
||||
i.fa.fa-chart-line
|
||||
| {{_ 'attachment-monitoring'}}
|
||||
|
||||
.main-body
|
||||
if loading.get
|
||||
+spinner
|
||||
else if showStorageSettings.get
|
||||
+storageSettings
|
||||
else if showMigration.get
|
||||
+attachmentMigration
|
||||
else if showMonitoring.get
|
||||
+attachmentMonitoring
|
||||
|
||||
template(name="storageSettings")
|
||||
.storage-settings
|
||||
h3 {{_ 'attachment-storage-configuration'}}
|
||||
|
||||
.storage-config-section
|
||||
h4 {{_ 'filesystem-storage'}}
|
||||
.form-group
|
||||
label {{_ 'writable-path'}}
|
||||
input.wekan-form-control#filesystem-path(type="text" value="{{filesystemPath}}" readonly)
|
||||
small.form-text.text-muted {{_ 'filesystem-path-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 'attachments-path'}}
|
||||
input.wekan-form-control#attachments-path(type="text" value="{{attachmentsPath}}" readonly)
|
||||
small.form-text.text-muted {{_ 'attachments-path-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 'avatars-path'}}
|
||||
input.wekan-form-control#avatars-path(type="text" value="{{avatarsPath}}" readonly)
|
||||
small.form-text.text-muted {{_ 'avatars-path-description'}}
|
||||
|
||||
.storage-config-section
|
||||
h4 {{_ 'mongodb-gridfs-storage'}}
|
||||
.form-group
|
||||
label {{_ 'gridfs-enabled'}}
|
||||
input.wekan-form-control#gridfs-enabled(type="checkbox" checked="{{gridfsEnabled}}" disabled)
|
||||
small.form-text.text-muted {{_ 'gridfs-enabled-description'}}
|
||||
|
||||
.storage-config-section
|
||||
h4 {{_ 's3-minio-storage'}}
|
||||
.form-group
|
||||
label {{_ 's3-enabled'}}
|
||||
input.wekan-form-control#s3-enabled(type="checkbox" checked="{{s3Enabled}}" disabled)
|
||||
small.form-text.text-muted {{_ 's3-enabled-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 's3-endpoint'}}
|
||||
input.wekan-form-control#s3-endpoint(type="text" value="{{s3Endpoint}}" readonly)
|
||||
small.form-text.text-muted {{_ 's3-endpoint-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 's3-bucket'}}
|
||||
input.wekan-form-control#s3-bucket(type="text" value="{{s3Bucket}}" readonly)
|
||||
small.form-text.text-muted {{_ 's3-bucket-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 's3-region'}}
|
||||
input.wekan-form-control#s3-region(type="text" value="{{s3Region}}" readonly)
|
||||
small.form-text.text-muted {{_ 's3-region-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 's3-access-key'}}
|
||||
input.wekan-form-control#s3-access-key(type="text" placeholder="{{_ 's3-access-key-placeholder'}}" readonly)
|
||||
small.form-text.text-muted {{_ 's3-access-key-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 's3-secret-key'}}
|
||||
input.wekan-form-control#s3-secret-key(type="password" placeholder="{{_ 's3-secret-key-placeholder'}}")
|
||||
small.form-text.text-muted {{_ 's3-secret-key-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 's3-ssl-enabled'}}
|
||||
input.wekan-form-control#s3-ssl-enabled(type="checkbox" checked="{{s3SslEnabled}}" disabled)
|
||||
small.form-text.text-muted {{_ 's3-ssl-enabled-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 's3-port'}}
|
||||
input.wekan-form-control#s3-port(type="number" value="{{s3Port}}" readonly)
|
||||
small.form-text.text-muted {{_ 's3-port-description'}}
|
||||
|
||||
.storage-actions
|
||||
button.js-test-s3-connection.btn.btn-secondary {{_ 'test-s3-connection'}}
|
||||
button.js-save-s3-settings.btn.btn-primary {{_ 'save-s3-settings'}}
|
||||
|
||||
template(name="attachmentMigration")
|
||||
.attachment-migration
|
||||
h3 {{_ 'attachment-migration'}}
|
||||
|
||||
.migration-controls
|
||||
.form-group
|
||||
label {{_ 'migration-batch-size'}}
|
||||
input.wekan-form-control#migration-batch-size(type="number" value="{{migrationBatchSize}}" min="1" max="100")
|
||||
small.form-text.text-muted {{_ 'migration-batch-size-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 'migration-delay-ms'}}
|
||||
input.wekan-form-control#migration-delay-ms(type="number" value="{{migrationDelayMs}}" min="100" max="10000")
|
||||
small.form-text.text-muted {{_ 'migration-delay-ms-description'}}
|
||||
|
||||
.form-group
|
||||
label {{_ 'migration-cpu-threshold'}}
|
||||
input.wekan-form-control#migration-cpu-threshold(type="number" value="{{migrationCpuThreshold}}" min="10" max="90")
|
||||
small.form-text.text-muted {{_ 'migration-cpu-threshold-description'}}
|
||||
|
||||
.migration-actions
|
||||
.migration-buttons
|
||||
button.js-migrate-all-to-filesystem.btn.btn-primary {{_ 'migrate-all-to-filesystem'}}
|
||||
button.js-migrate-all-to-gridfs.btn.btn-primary {{_ 'migrate-all-to-gridfs'}}
|
||||
button.js-migrate-all-to-s3.btn.btn-primary {{_ 'migrate-all-to-s3'}}
|
||||
|
||||
.migration-controls
|
||||
button.js-pause-migration.btn.btn-warning {{_ 'pause-migration'}}
|
||||
button.js-resume-migration.btn.btn-success {{_ 'resume-migration'}}
|
||||
button.js-stop-migration.btn.btn-danger {{_ 'stop-migration'}}
|
||||
|
||||
.migration-progress
|
||||
h4 {{_ 'migration-progress'}}
|
||||
.progress
|
||||
.progress-bar(role="progressbar" style="width: {{migrationProgress}}%" aria-valuenow="{{migrationProgress}}" aria-valuemin="0" aria-valuemax="100")
|
||||
| {{migrationProgress}}%
|
||||
|
||||
.migration-stats
|
||||
.stat-item
|
||||
span.label {{_ 'total-attachments'}}:
|
||||
span.value {{totalAttachments}}
|
||||
.stat-item
|
||||
span.label {{_ 'migrated-attachments'}}:
|
||||
span.value {{migratedAttachments}}
|
||||
.stat-item
|
||||
span.label {{_ 'remaining-attachments'}}:
|
||||
span.value {{remainingAttachments}}
|
||||
.stat-item
|
||||
span.label {{_ 'migration-status'}}:
|
||||
span.value {{migrationStatus}}
|
||||
|
||||
.migration-log
|
||||
h4 {{_ 'migration-log'}}
|
||||
.log-container
|
||||
pre#migration-log-content {{migrationLog}}
|
||||
|
||||
template(name="attachmentMonitoring")
|
||||
.attachment-monitoring
|
||||
h3 {{_ 'attachment-monitoring'}}
|
||||
|
||||
.monitoring-stats
|
||||
.stats-grid
|
||||
.stat-card
|
||||
h5 {{_ 'total-attachments'}}
|
||||
.stat-value {{totalAttachments}}
|
||||
.stat-card
|
||||
h5 {{_ 'filesystem-attachments'}}
|
||||
.stat-value {{filesystemAttachments}}
|
||||
.stat-card
|
||||
h5 {{_ 'gridfs-attachments'}}
|
||||
.stat-value {{gridfsAttachments}}
|
||||
.stat-card
|
||||
h5 {{_ 's3-attachments'}}
|
||||
.stat-value {{s3Attachments}}
|
||||
.stat-card
|
||||
h5 {{_ 'total-size'}}
|
||||
.stat-value {{totalSize}}
|
||||
.stat-card
|
||||
h5 {{_ 'filesystem-size'}}
|
||||
.stat-value {{filesystemSize}}
|
||||
.stat-card
|
||||
h5 {{_ 'gridfs-size'}}
|
||||
.stat-value {{gridfsSize}}
|
||||
.stat-card
|
||||
h5 {{_ 's3-size'}}
|
||||
.stat-value {{s3Size}}
|
||||
|
||||
.monitoring-charts
|
||||
h4 {{_ 'storage-distribution'}}
|
||||
.chart-container
|
||||
canvas#storage-distribution-chart
|
||||
|
||||
.monitoring-actions
|
||||
button.js-refresh-monitoring.btn.btn-secondary {{_ 'refresh-monitoring'}}
|
||||
button.js-export-monitoring.btn.btn-primary {{_ 'export-monitoring'}}
|
||||
464
client/components/settings/attachmentSettings.js
Normal file
464
client/components/settings/attachmentSettings.js
Normal file
|
|
@ -0,0 +1,464 @@
|
|||
import { ReactiveCache } from '/imports/reactiveCache';
|
||||
import { TAPi18n } from '/imports/i18n';
|
||||
import { Meteor } from 'meteor/meteor';
|
||||
import { Session } from 'meteor/session';
|
||||
import { Tracker } from 'meteor/tracker';
|
||||
import { ReactiveVar } from 'meteor/reactive-var';
|
||||
import { BlazeComponent } from 'meteor/peerlibrary:blaze-components';
|
||||
import { Chart } from 'chart.js';
|
||||
|
||||
// Global reactive variables for attachment settings
|
||||
const attachmentSettings = {
|
||||
loading: new ReactiveVar(false),
|
||||
showStorageSettings: new ReactiveVar(false),
|
||||
showMigration: new ReactiveVar(false),
|
||||
showMonitoring: new ReactiveVar(false),
|
||||
|
||||
// Storage configuration
|
||||
filesystemPath: new ReactiveVar(''),
|
||||
attachmentsPath: new ReactiveVar(''),
|
||||
avatarsPath: new ReactiveVar(''),
|
||||
gridfsEnabled: new ReactiveVar(false),
|
||||
s3Enabled: new ReactiveVar(false),
|
||||
s3Endpoint: new ReactiveVar(''),
|
||||
s3Bucket: new ReactiveVar(''),
|
||||
s3Region: new ReactiveVar(''),
|
||||
s3SslEnabled: new ReactiveVar(false),
|
||||
s3Port: new ReactiveVar(443),
|
||||
|
||||
// Migration settings
|
||||
migrationBatchSize: new ReactiveVar(10),
|
||||
migrationDelayMs: new ReactiveVar(1000),
|
||||
migrationCpuThreshold: new ReactiveVar(70),
|
||||
migrationProgress: new ReactiveVar(0),
|
||||
migrationStatus: new ReactiveVar('idle'),
|
||||
migrationLog: new ReactiveVar(''),
|
||||
|
||||
// Monitoring data
|
||||
totalAttachments: new ReactiveVar(0),
|
||||
filesystemAttachments: new ReactiveVar(0),
|
||||
gridfsAttachments: new ReactiveVar(0),
|
||||
s3Attachments: new ReactiveVar(0),
|
||||
totalSize: new ReactiveVar(0),
|
||||
filesystemSize: new ReactiveVar(0),
|
||||
gridfsSize: new ReactiveVar(0),
|
||||
s3Size: new ReactiveVar(0),
|
||||
|
||||
// Migration state
|
||||
isMigrationRunning: new ReactiveVar(false),
|
||||
isMigrationPaused: new ReactiveVar(false),
|
||||
migrationQueue: new ReactiveVar([]),
|
||||
currentMigration: new ReactiveVar(null)
|
||||
};
|
||||
|
||||
// Main attachment settings component
|
||||
BlazeComponent.extendComponent({
|
||||
onCreated() {
|
||||
this.loading = attachmentSettings.loading;
|
||||
this.showStorageSettings = attachmentSettings.showStorageSettings;
|
||||
this.showMigration = attachmentSettings.showMigration;
|
||||
this.showMonitoring = attachmentSettings.showMonitoring;
|
||||
|
||||
// Load initial data
|
||||
this.loadStorageConfiguration();
|
||||
this.loadMigrationSettings();
|
||||
this.loadMonitoringData();
|
||||
},
|
||||
|
||||
events() {
|
||||
return [
|
||||
{
|
||||
'click a.js-attachment-storage-settings': this.switchToStorageSettings,
|
||||
'click a.js-attachment-migration': this.switchToMigration,
|
||||
'click a.js-attachment-monitoring': this.switchToMonitoring,
|
||||
}
|
||||
];
|
||||
},
|
||||
|
||||
switchToStorageSettings(event) {
|
||||
this.switchMenu(event, 'storage-settings');
|
||||
this.showStorageSettings.set(true);
|
||||
this.showMigration.set(false);
|
||||
this.showMonitoring.set(false);
|
||||
},
|
||||
|
||||
switchToMigration(event) {
|
||||
this.switchMenu(event, 'attachment-migration');
|
||||
this.showStorageSettings.set(false);
|
||||
this.showMigration.set(true);
|
||||
this.showMonitoring.set(false);
|
||||
},
|
||||
|
||||
switchToMonitoring(event) {
|
||||
this.switchMenu(event, 'attachment-monitoring');
|
||||
this.showStorageSettings.set(false);
|
||||
this.showMigration.set(false);
|
||||
this.showMonitoring.set(true);
|
||||
},
|
||||
|
||||
switchMenu(event, targetId) {
|
||||
const target = $(event.target);
|
||||
if (!target.hasClass('active')) {
|
||||
this.loading.set(true);
|
||||
|
||||
$('.side-menu li.active').removeClass('active');
|
||||
target.parent().addClass('active');
|
||||
|
||||
// Load data based on target
|
||||
if (targetId === 'storage-settings') {
|
||||
this.loadStorageConfiguration();
|
||||
} else if (targetId === 'attachment-migration') {
|
||||
this.loadMigrationSettings();
|
||||
} else if (targetId === 'attachment-monitoring') {
|
||||
this.loadMonitoringData();
|
||||
}
|
||||
|
||||
this.loading.set(false);
|
||||
}
|
||||
},
|
||||
|
||||
loadStorageConfiguration() {
|
||||
Meteor.call('getAttachmentStorageConfiguration', (error, result) => {
|
||||
if (!error && result) {
|
||||
attachmentSettings.filesystemPath.set(result.filesystemPath || '');
|
||||
attachmentSettings.attachmentsPath.set(result.attachmentsPath || '');
|
||||
attachmentSettings.avatarsPath.set(result.avatarsPath || '');
|
||||
attachmentSettings.gridfsEnabled.set(result.gridfsEnabled || false);
|
||||
attachmentSettings.s3Enabled.set(result.s3Enabled || false);
|
||||
attachmentSettings.s3Endpoint.set(result.s3Endpoint || '');
|
||||
attachmentSettings.s3Bucket.set(result.s3Bucket || '');
|
||||
attachmentSettings.s3Region.set(result.s3Region || '');
|
||||
attachmentSettings.s3SslEnabled.set(result.s3SslEnabled || false);
|
||||
attachmentSettings.s3Port.set(result.s3Port || 443);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
loadMigrationSettings() {
|
||||
Meteor.call('getAttachmentMigrationSettings', (error, result) => {
|
||||
if (!error && result) {
|
||||
attachmentSettings.migrationBatchSize.set(result.batchSize || 10);
|
||||
attachmentSettings.migrationDelayMs.set(result.delayMs || 1000);
|
||||
attachmentSettings.migrationCpuThreshold.set(result.cpuThreshold || 70);
|
||||
attachmentSettings.migrationStatus.set(result.status || 'idle');
|
||||
attachmentSettings.migrationProgress.set(result.progress || 0);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
loadMonitoringData() {
|
||||
Meteor.call('getAttachmentMonitoringData', (error, result) => {
|
||||
if (!error && result) {
|
||||
attachmentSettings.totalAttachments.set(result.totalAttachments || 0);
|
||||
attachmentSettings.filesystemAttachments.set(result.filesystemAttachments || 0);
|
||||
attachmentSettings.gridfsAttachments.set(result.gridfsAttachments || 0);
|
||||
attachmentSettings.s3Attachments.set(result.s3Attachments || 0);
|
||||
attachmentSettings.totalSize.set(result.totalSize || 0);
|
||||
attachmentSettings.filesystemSize.set(result.filesystemSize || 0);
|
||||
attachmentSettings.gridfsSize.set(result.gridfsSize || 0);
|
||||
attachmentSettings.s3Size.set(result.s3Size || 0);
|
||||
}
|
||||
});
|
||||
}
|
||||
}).register('attachmentSettings');
|
||||
|
||||
// Storage settings component
|
||||
BlazeComponent.extendComponent({
|
||||
onCreated() {
|
||||
this.filesystemPath = attachmentSettings.filesystemPath;
|
||||
this.attachmentsPath = attachmentSettings.attachmentsPath;
|
||||
this.avatarsPath = attachmentSettings.avatarsPath;
|
||||
this.gridfsEnabled = attachmentSettings.gridfsEnabled;
|
||||
this.s3Enabled = attachmentSettings.s3Enabled;
|
||||
this.s3Endpoint = attachmentSettings.s3Endpoint;
|
||||
this.s3Bucket = attachmentSettings.s3Bucket;
|
||||
this.s3Region = attachmentSettings.s3Region;
|
||||
this.s3SslEnabled = attachmentSettings.s3SslEnabled;
|
||||
this.s3Port = attachmentSettings.s3Port;
|
||||
},
|
||||
|
||||
events() {
|
||||
return [
|
||||
{
|
||||
'click button.js-test-s3-connection': this.testS3Connection,
|
||||
'click button.js-save-s3-settings': this.saveS3Settings,
|
||||
'change input#s3-secret-key': this.updateS3SecretKey
|
||||
}
|
||||
];
|
||||
},
|
||||
|
||||
testS3Connection() {
|
||||
const secretKey = $('#s3-secret-key').val();
|
||||
if (!secretKey) {
|
||||
alert(TAPi18n.__('s3-secret-key-required'));
|
||||
return;
|
||||
}
|
||||
|
||||
Meteor.call('testS3Connection', { secretKey }, (error, result) => {
|
||||
if (error) {
|
||||
alert(TAPi18n.__('s3-connection-failed') + ': ' + error.reason);
|
||||
} else {
|
||||
alert(TAPi18n.__('s3-connection-success'));
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
saveS3Settings() {
|
||||
const secretKey = $('#s3-secret-key').val();
|
||||
if (!secretKey) {
|
||||
alert(TAPi18n.__('s3-secret-key-required'));
|
||||
return;
|
||||
}
|
||||
|
||||
Meteor.call('saveS3Settings', { secretKey }, (error, result) => {
|
||||
if (error) {
|
||||
alert(TAPi18n.__('s3-settings-save-failed') + ': ' + error.reason);
|
||||
} else {
|
||||
alert(TAPi18n.__('s3-settings-saved'));
|
||||
$('#s3-secret-key').val(''); // Clear the password field
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
updateS3SecretKey(event) {
|
||||
// This method can be used to validate the secret key format
|
||||
const secretKey = event.target.value;
|
||||
// Add validation logic here if needed
|
||||
}
|
||||
}).register('storageSettings');
|
||||
|
||||
// Migration component
|
||||
BlazeComponent.extendComponent({
|
||||
onCreated() {
|
||||
this.migrationBatchSize = attachmentSettings.migrationBatchSize;
|
||||
this.migrationDelayMs = attachmentSettings.migrationDelayMs;
|
||||
this.migrationCpuThreshold = attachmentSettings.migrationCpuThreshold;
|
||||
this.migrationProgress = attachmentSettings.migrationProgress;
|
||||
this.migrationStatus = attachmentSettings.migrationStatus;
|
||||
this.migrationLog = attachmentSettings.migrationLog;
|
||||
this.isMigrationRunning = attachmentSettings.isMigrationRunning;
|
||||
this.isMigrationPaused = attachmentSettings.isMigrationPaused;
|
||||
|
||||
// Subscribe to migration updates
|
||||
this.subscription = Meteor.subscribe('attachmentMigrationStatus');
|
||||
|
||||
// Set up reactive updates
|
||||
this.autorun(() => {
|
||||
const status = attachmentSettings.migrationStatus.get();
|
||||
if (status === 'running') {
|
||||
this.isMigrationRunning.set(true);
|
||||
} else {
|
||||
this.isMigrationRunning.set(false);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
onDestroyed() {
|
||||
if (this.subscription) {
|
||||
this.subscription.stop();
|
||||
}
|
||||
},
|
||||
|
||||
events() {
|
||||
return [
|
||||
{
|
||||
'click button.js-migrate-all-to-filesystem': () => this.startMigration('filesystem'),
|
||||
'click button.js-migrate-all-to-gridfs': () => this.startMigration('gridfs'),
|
||||
'click button.js-migrate-all-to-s3': () => this.startMigration('s3'),
|
||||
'click button.js-pause-migration': this.pauseMigration,
|
||||
'click button.js-resume-migration': this.resumeMigration,
|
||||
'click button.js-stop-migration': this.stopMigration,
|
||||
'change input#migration-batch-size': this.updateBatchSize,
|
||||
'change input#migration-delay-ms': this.updateDelayMs,
|
||||
'change input#migration-cpu-threshold': this.updateCpuThreshold
|
||||
}
|
||||
];
|
||||
},
|
||||
|
||||
startMigration(targetStorage) {
|
||||
const batchSize = parseInt($('#migration-batch-size').val()) || 10;
|
||||
const delayMs = parseInt($('#migration-delay-ms').val()) || 1000;
|
||||
const cpuThreshold = parseInt($('#migration-cpu-threshold').val()) || 70;
|
||||
|
||||
Meteor.call('startAttachmentMigration', {
|
||||
targetStorage,
|
||||
batchSize,
|
||||
delayMs,
|
||||
cpuThreshold
|
||||
}, (error, result) => {
|
||||
if (error) {
|
||||
alert(TAPi18n.__('migration-start-failed') + ': ' + error.reason);
|
||||
} else {
|
||||
this.addToLog(TAPi18n.__('migration-started') + ': ' + targetStorage);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
pauseMigration() {
|
||||
Meteor.call('pauseAttachmentMigration', (error, result) => {
|
||||
if (error) {
|
||||
alert(TAPi18n.__('migration-pause-failed') + ': ' + error.reason);
|
||||
} else {
|
||||
this.addToLog(TAPi18n.__('migration-paused'));
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
resumeMigration() {
|
||||
Meteor.call('resumeAttachmentMigration', (error, result) => {
|
||||
if (error) {
|
||||
alert(TAPi18n.__('migration-resume-failed') + ': ' + error.reason);
|
||||
} else {
|
||||
this.addToLog(TAPi18n.__('migration-resumed'));
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
stopMigration() {
|
||||
if (confirm(TAPi18n.__('migration-stop-confirm'))) {
|
||||
Meteor.call('stopAttachmentMigration', (error, result) => {
|
||||
if (error) {
|
||||
alert(TAPi18n.__('migration-stop-failed') + ': ' + error.reason);
|
||||
} else {
|
||||
this.addToLog(TAPi18n.__('migration-stopped'));
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
updateBatchSize(event) {
|
||||
const value = parseInt(event.target.value);
|
||||
if (value >= 1 && value <= 100) {
|
||||
attachmentSettings.migrationBatchSize.set(value);
|
||||
}
|
||||
},
|
||||
|
||||
updateDelayMs(event) {
|
||||
const value = parseInt(event.target.value);
|
||||
if (value >= 100 && value <= 10000) {
|
||||
attachmentSettings.migrationDelayMs.set(value);
|
||||
}
|
||||
},
|
||||
|
||||
updateCpuThreshold(event) {
|
||||
const value = parseInt(event.target.value);
|
||||
if (value >= 10 && value <= 90) {
|
||||
attachmentSettings.migrationCpuThreshold.set(value);
|
||||
}
|
||||
},
|
||||
|
||||
addToLog(message) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const currentLog = attachmentSettings.migrationLog.get();
|
||||
const newLog = `[${timestamp}] ${message}\n${currentLog}`;
|
||||
attachmentSettings.migrationLog.set(newLog);
|
||||
}
|
||||
}).register('attachmentMigration');
|
||||
|
||||
// Monitoring component
|
||||
BlazeComponent.extendComponent({
|
||||
onCreated() {
|
||||
this.totalAttachments = attachmentSettings.totalAttachments;
|
||||
this.filesystemAttachments = attachmentSettings.filesystemAttachments;
|
||||
this.gridfsAttachments = attachmentSettings.gridfsAttachments;
|
||||
this.s3Attachments = attachmentSettings.s3Attachments;
|
||||
this.totalSize = attachmentSettings.totalSize;
|
||||
this.filesystemSize = attachmentSettings.filesystemSize;
|
||||
this.gridfsSize = attachmentSettings.gridfsSize;
|
||||
this.s3Size = attachmentSettings.s3Size;
|
||||
|
||||
// Subscribe to monitoring updates
|
||||
this.subscription = Meteor.subscribe('attachmentMonitoringData');
|
||||
|
||||
// Set up chart
|
||||
this.autorun(() => {
|
||||
this.updateChart();
|
||||
});
|
||||
},
|
||||
|
||||
onDestroyed() {
|
||||
if (this.subscription) {
|
||||
this.subscription.stop();
|
||||
}
|
||||
},
|
||||
|
||||
events() {
|
||||
return [
|
||||
{
|
||||
'click button.js-refresh-monitoring': this.refreshMonitoring,
|
||||
'click button.js-export-monitoring': this.exportMonitoring
|
||||
}
|
||||
];
|
||||
},
|
||||
|
||||
refreshMonitoring() {
|
||||
Meteor.call('refreshAttachmentMonitoringData', (error, result) => {
|
||||
if (error) {
|
||||
alert(TAPi18n.__('monitoring-refresh-failed') + ': ' + error.reason);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
exportMonitoring() {
|
||||
Meteor.call('exportAttachmentMonitoringData', (error, result) => {
|
||||
if (error) {
|
||||
alert(TAPi18n.__('monitoring-export-failed') + ': ' + error.reason);
|
||||
} else {
|
||||
// Download the exported data
|
||||
const blob = new Blob([JSON.stringify(result, null, 2)], { type: 'application/json' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = 'wekan-attachment-monitoring.json';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
updateChart() {
|
||||
const ctx = document.getElementById('storage-distribution-chart');
|
||||
if (!ctx) return;
|
||||
|
||||
const filesystemCount = this.filesystemAttachments.get();
|
||||
const gridfsCount = this.gridfsAttachments.get();
|
||||
const s3Count = this.s3Attachments.get();
|
||||
|
||||
if (this.chart) {
|
||||
this.chart.destroy();
|
||||
}
|
||||
|
||||
this.chart = new Chart(ctx, {
|
||||
type: 'doughnut',
|
||||
data: {
|
||||
labels: [
|
||||
TAPi18n.__('filesystem-storage'),
|
||||
TAPi18n.__('gridfs-storage'),
|
||||
TAPi18n.__('s3-storage')
|
||||
],
|
||||
datasets: [{
|
||||
data: [filesystemCount, gridfsCount, s3Count],
|
||||
backgroundColor: [
|
||||
'#28a745',
|
||||
'#007bff',
|
||||
'#ffc107'
|
||||
]
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}).register('attachmentMonitoring');
|
||||
|
||||
// Export the attachment settings for use in other components
|
||||
export { attachmentSettings };
|
||||
|
|
@ -42,6 +42,10 @@ template(name="setting")
|
|||
a.js-setting-menu(data-id="webhook-setting")
|
||||
i.fa.fa-globe
|
||||
| {{_ 'global-webhook'}}
|
||||
li
|
||||
a.js-setting-menu(data-id="attachment-settings")
|
||||
i.fa.fa-paperclip
|
||||
| {{_ 'attachment-settings'}}
|
||||
.main-body
|
||||
if loading.get
|
||||
+spinner
|
||||
|
|
@ -62,6 +66,8 @@ template(name="setting")
|
|||
+layoutSettings
|
||||
else if webhookSetting.get
|
||||
+webhookSettings
|
||||
else if attachmentSettings.get
|
||||
+attachmentSettings
|
||||
|
||||
template(name="webhookSettings")
|
||||
span
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ BlazeComponent.extendComponent({
|
|||
this.accessibilitySetting = new ReactiveVar(false);
|
||||
this.layoutSetting = new ReactiveVar(false);
|
||||
this.webhookSetting = new ReactiveVar(false);
|
||||
this.attachmentSettings = new ReactiveVar(false);
|
||||
|
||||
Meteor.subscribe('setting');
|
||||
Meteor.subscribe('mailServer');
|
||||
|
|
@ -113,6 +114,7 @@ BlazeComponent.extendComponent({
|
|||
this.accessibilitySetting.set('accessibility-setting' === targetID);
|
||||
this.layoutSetting.set('layout-setting' === targetID);
|
||||
this.webhookSetting.set('webhook-setting' === targetID);
|
||||
this.attachmentSettings.set('attachment-settings' === targetID);
|
||||
this.tableVisibilityModeSetting.set('tableVisibilityMode-setting' === targetID);
|
||||
}
|
||||
},
|
||||
|
|
|
|||
306
docs/ImportExport/Attachment-Migration-System.md
Normal file
306
docs/ImportExport/Attachment-Migration-System.md
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
# Enhanced Attachment Migration System
|
||||
|
||||
## Overview
|
||||
|
||||
The Enhanced Attachment Migration System provides a comprehensive solution for managing attachment storage across multiple backends (filesystem, MongoDB GridFS, S3/MinIO) with CPU throttling, real-time monitoring, and secure configuration management.
|
||||
|
||||
## Features
|
||||
|
||||
### 1. Multi-Backend Storage Support
|
||||
- **Filesystem Storage**: Local file system storage using `WRITABLE_PATH`
|
||||
- **MongoDB GridFS**: Database-based file storage
|
||||
- **S3/MinIO**: Cloud and object storage compatibility
|
||||
|
||||
### 2. CPU Throttling
|
||||
- **Automatic CPU Monitoring**: Real-time CPU usage tracking
|
||||
- **Configurable Thresholds**: Set CPU usage limits (10-90%)
|
||||
- **Automatic Pausing**: Migration pauses when CPU threshold is exceeded
|
||||
- **Resume Capability**: Continue migration when CPU usage drops
|
||||
|
||||
### 3. Batch Processing
|
||||
- **Configurable Batch Size**: Process 1-100 attachments per batch
|
||||
- **Adjustable Delays**: Set delays between batches (100-10000ms)
|
||||
- **Progress Tracking**: Real-time progress monitoring
|
||||
- **Queue Management**: Intelligent migration queue handling
|
||||
|
||||
### 4. Security Features
|
||||
- **Password Protection**: S3 secret keys are never displayed
|
||||
- **Admin-Only Access**: All operations require admin privileges
|
||||
- **Secure Configuration**: Environment-based configuration management
|
||||
- **Audit Logging**: Comprehensive migration logging
|
||||
|
||||
### 5. Real-Time Monitoring
|
||||
- **Storage Statistics**: Live attachment counts and sizes
|
||||
- **Visual Charts**: Storage distribution visualization
|
||||
- **Migration Status**: Real-time migration progress
|
||||
- **System Metrics**: CPU, memory, and performance monitoring
|
||||
|
||||
## Admin Panel Interface
|
||||
|
||||
### Storage Settings
|
||||
- **Filesystem Configuration**: View and configure filesystem paths
|
||||
- **GridFS Status**: Monitor MongoDB GridFS availability
|
||||
- **S3/MinIO Configuration**: Secure S3/MinIO setup and testing
|
||||
- **Connection Testing**: Validate storage backend connections
|
||||
|
||||
### Migration Controls
|
||||
- **Batch Configuration**: Set batch size, delays, and CPU thresholds
|
||||
- **Migration Actions**: Start, pause, resume, and stop migrations
|
||||
- **Progress Monitoring**: Real-time progress bars and statistics
|
||||
- **Log Viewing**: Live migration logs with timestamps
|
||||
|
||||
### Monitoring Dashboard
|
||||
- **Storage Distribution**: Visual breakdown of attachment storage
|
||||
- **Size Analytics**: Total and per-storage size statistics
|
||||
- **Performance Metrics**: System resource usage
|
||||
- **Export Capabilities**: Download monitoring data
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
#### Filesystem Storage
|
||||
```bash
|
||||
# Base writable path for all file storage
|
||||
WRITABLE_PATH=/data
|
||||
|
||||
# Attachments will be stored at: ${WRITABLE_PATH}/attachments
|
||||
# Avatars will be stored at: ${WRITABLE_PATH}/avatars
|
||||
```
|
||||
|
||||
#### S3/MinIO Storage
|
||||
```bash
|
||||
# S3 configuration (JSON format)
|
||||
S3='{"s3":{"key":"access-key","secret":"secret-key","bucket":"bucket-name","endPoint":"s3.amazonaws.com","port":443,"sslEnabled":true,"region":"us-east-1"}}'
|
||||
|
||||
# Alternative: S3 secret file (Docker secrets)
|
||||
S3_SECRET_FILE=/run/secrets/s3_secret
|
||||
```
|
||||
|
||||
### Migration Settings
|
||||
|
||||
#### Default Configuration
|
||||
- **Batch Size**: 10 attachments per batch
|
||||
- **Delay**: 1000ms between batches
|
||||
- **CPU Threshold**: 70% maximum CPU usage
|
||||
- **Auto-pause**: When CPU exceeds threshold
|
||||
|
||||
#### Customization
|
||||
All settings can be adjusted through the admin panel:
|
||||
- Batch size: 1-100 attachments
|
||||
- Delay: 100-10000ms
|
||||
- CPU threshold: 10-90%
|
||||
|
||||
## Usage
|
||||
|
||||
### Starting a Migration
|
||||
|
||||
1. **Access Admin Panel**: Navigate to Settings → Attachment Settings
|
||||
2. **Configure Migration**: Set batch size, delay, and CPU threshold
|
||||
3. **Select Target Storage**: Choose filesystem, GridFS, or S3
|
||||
4. **Start Migration**: Click the appropriate migration button
|
||||
5. **Monitor Progress**: Watch real-time progress and logs
|
||||
|
||||
### Migration Process
|
||||
|
||||
1. **Queue Initialization**: All attachments are queued for migration
|
||||
2. **Batch Processing**: Attachments are processed in configurable batches
|
||||
3. **CPU Monitoring**: System continuously monitors CPU usage
|
||||
4. **Automatic Pausing**: Migration pauses if CPU threshold is exceeded
|
||||
5. **Resume Capability**: Migration resumes when CPU usage drops
|
||||
6. **Progress Tracking**: Real-time progress updates and logging
|
||||
|
||||
### Monitoring Migration
|
||||
|
||||
- **Progress Bar**: Visual progress indicator
|
||||
- **Statistics**: Total, migrated, and remaining attachment counts
|
||||
- **Status Display**: Current migration status (running, paused, idle)
|
||||
- **Log View**: Real-time migration logs with timestamps
|
||||
- **Control Buttons**: Pause, resume, and stop migration
|
||||
|
||||
## API Reference
|
||||
|
||||
### Server Methods
|
||||
|
||||
#### Migration Control
|
||||
```javascript
|
||||
// Start migration
|
||||
Meteor.call('startAttachmentMigration', {
|
||||
targetStorage: 'filesystem', // 'filesystem', 'gridfs', 's3'
|
||||
batchSize: 10,
|
||||
delayMs: 1000,
|
||||
cpuThreshold: 70
|
||||
});
|
||||
|
||||
// Pause migration
|
||||
Meteor.call('pauseAttachmentMigration');
|
||||
|
||||
// Resume migration
|
||||
Meteor.call('resumeAttachmentMigration');
|
||||
|
||||
// Stop migration
|
||||
Meteor.call('stopAttachmentMigration');
|
||||
```
|
||||
|
||||
#### Configuration Management
|
||||
```javascript
|
||||
// Get storage configuration
|
||||
Meteor.call('getAttachmentStorageConfiguration');
|
||||
|
||||
// Test S3 connection
|
||||
Meteor.call('testS3Connection', { secretKey: 'new-secret-key' });
|
||||
|
||||
// Save S3 settings
|
||||
Meteor.call('saveS3Settings', { secretKey: 'new-secret-key' });
|
||||
```
|
||||
|
||||
#### Monitoring
|
||||
```javascript
|
||||
// Get monitoring data
|
||||
Meteor.call('getAttachmentMonitoringData');
|
||||
|
||||
// Refresh monitoring data
|
||||
Meteor.call('refreshAttachmentMonitoringData');
|
||||
|
||||
// Export monitoring data
|
||||
Meteor.call('exportAttachmentMonitoringData');
|
||||
```
|
||||
|
||||
### Publications
|
||||
|
||||
#### Real-Time Updates
|
||||
```javascript
|
||||
// Subscribe to migration status
|
||||
Meteor.subscribe('attachmentMigrationStatus');
|
||||
|
||||
// Subscribe to monitoring data
|
||||
Meteor.subscribe('attachmentMonitoringData');
|
||||
```
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
### CPU Throttling
|
||||
- **Automatic Monitoring**: CPU usage is checked every 5 seconds
|
||||
- **Threshold-Based Pausing**: Migration pauses when CPU exceeds threshold
|
||||
- **Resume Logic**: Migration resumes when CPU usage drops below threshold
|
||||
- **Configurable Limits**: CPU thresholds can be adjusted (10-90%)
|
||||
|
||||
### Batch Processing
|
||||
- **Configurable Batches**: Batch size can be adjusted (1-100)
|
||||
- **Delay Control**: Delays between batches prevent system overload
|
||||
- **Queue Management**: Intelligent queue processing with error handling
|
||||
- **Progress Tracking**: Real-time progress updates
|
||||
|
||||
### Memory Management
|
||||
- **Streaming Processing**: Large files are processed using streams
|
||||
- **Memory Monitoring**: System memory usage is tracked
|
||||
- **Garbage Collection**: Automatic cleanup of processed data
|
||||
- **Error Recovery**: Robust error handling and recovery
|
||||
|
||||
## Security
|
||||
|
||||
### Access Control
|
||||
- **Admin-Only Access**: All operations require admin privileges
|
||||
- **User Authentication**: Proper user authentication and authorization
|
||||
- **Session Management**: Secure session handling
|
||||
|
||||
### Data Protection
|
||||
- **Password Security**: S3 secret keys are never displayed in UI
|
||||
- **Environment Variables**: Sensitive data stored in environment variables
|
||||
- **Secure Transmission**: All data transmission is encrypted
|
||||
- **Audit Logging**: Comprehensive logging of all operations
|
||||
|
||||
### Configuration Security
|
||||
- **Read-Only Display**: Sensitive configuration is displayed as read-only
|
||||
- **Password Updates**: Only new passwords can be set, not viewed
|
||||
- **Connection Testing**: Secure connection testing without exposing credentials
|
||||
- **Environment Isolation**: Configuration isolated from application code
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
#### Migration Not Starting
|
||||
- **Check Permissions**: Ensure user has admin privileges
|
||||
- **Verify Configuration**: Check storage backend configuration
|
||||
- **Review Logs**: Check server logs for error messages
|
||||
- **Test Connections**: Verify storage backend connectivity
|
||||
|
||||
#### High CPU Usage
|
||||
- **Reduce Batch Size**: Decrease batch size to reduce CPU load
|
||||
- **Increase Delays**: Add longer delays between batches
|
||||
- **Lower CPU Threshold**: Reduce CPU threshold for earlier pausing
|
||||
- **Monitor System**: Check system resource usage
|
||||
|
||||
#### Migration Pausing Frequently
|
||||
- **Check CPU Threshold**: Verify CPU threshold settings
|
||||
- **Monitor System Load**: Check for other high-CPU processes
|
||||
- **Adjust Settings**: Increase CPU threshold or reduce batch size
|
||||
- **System Optimization**: Optimize system performance
|
||||
|
||||
#### Storage Connection Issues
|
||||
- **Verify Credentials**: Check S3/MinIO credentials
|
||||
- **Test Connectivity**: Use connection test feature
|
||||
- **Check Network**: Verify network connectivity
|
||||
- **Review Configuration**: Validate storage configuration
|
||||
|
||||
### Debug Information
|
||||
|
||||
#### Migration Logs
|
||||
- **Real-Time Logs**: View live migration logs in admin panel
|
||||
- **Server Logs**: Check server console for detailed logs
|
||||
- **Error Messages**: Review error messages for specific issues
|
||||
- **Progress Tracking**: Monitor migration progress and statistics
|
||||
|
||||
#### System Monitoring
|
||||
- **CPU Usage**: Monitor CPU usage during migration
|
||||
- **Memory Usage**: Track memory consumption
|
||||
- **Disk I/O**: Monitor disk input/output operations
|
||||
- **Network Usage**: Check network bandwidth usage
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Migration Planning
|
||||
- **Schedule During Low Usage**: Run migrations during low-traffic periods
|
||||
- **Test First**: Test migration with small batches first
|
||||
- **Monitor Resources**: Keep an eye on system resources
|
||||
- **Have Backup**: Ensure data backup before migration
|
||||
|
||||
### Performance Optimization
|
||||
- **Optimize Batch Size**: Find optimal batch size for your system
|
||||
- **Adjust Delays**: Set appropriate delays between batches
|
||||
- **Monitor CPU**: Set realistic CPU thresholds
|
||||
- **Use Monitoring**: Regularly check monitoring data
|
||||
|
||||
### Security Practices
|
||||
- **Regular Updates**: Keep S3 credentials updated
|
||||
- **Access Control**: Limit admin access to necessary users
|
||||
- **Audit Logs**: Regularly review migration logs
|
||||
- **Environment Security**: Secure environment variable storage
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### Planned Features
|
||||
- **Incremental Migration**: Migrate only changed attachments
|
||||
- **Parallel Processing**: Support for parallel migration streams
|
||||
- **Advanced Scheduling**: Time-based migration scheduling
|
||||
- **Compression Support**: Built-in file compression during migration
|
||||
|
||||
### Integration Opportunities
|
||||
- **Cloud Storage**: Additional cloud storage providers
|
||||
- **CDN Integration**: Content delivery network support
|
||||
- **Backup Integration**: Automated backup during migration
|
||||
- **Analytics**: Advanced storage analytics and reporting
|
||||
|
||||
## Support
|
||||
|
||||
For issues and questions:
|
||||
1. Check this documentation
|
||||
2. Review server logs
|
||||
3. Use the monitoring tools
|
||||
4. Consult the Wekan community
|
||||
5. Report issues with detailed information
|
||||
|
||||
## License
|
||||
|
||||
This Enhanced Attachment Migration System is part of Wekan and is licensed under the MIT License.
|
||||
87
imports/i18n/en.i18n.json
Normal file
87
imports/i18n/en.i18n.json
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
{
|
||||
"attachment-settings": "Attachment Settings",
|
||||
"attachment-storage-settings": "Storage Settings",
|
||||
"attachment-migration": "Migration",
|
||||
"attachment-monitoring": "Monitoring",
|
||||
"attachment-storage-configuration": "Storage Configuration",
|
||||
"filesystem-storage": "Filesystem Storage",
|
||||
"writable-path": "Writable Path",
|
||||
"filesystem-path-description": "Base path for file storage",
|
||||
"attachments-path": "Attachments Path",
|
||||
"attachments-path-description": "Path for attachment files",
|
||||
"avatars-path": "Avatars Path",
|
||||
"avatars-path-description": "Path for avatar files",
|
||||
"mongodb-gridfs-storage": "MongoDB GridFS Storage",
|
||||
"gridfs-enabled": "GridFS Enabled",
|
||||
"gridfs-enabled-description": "MongoDB GridFS storage is always available",
|
||||
"s3-minio-storage": "S3/MinIO Storage",
|
||||
"s3-enabled": "S3 Enabled",
|
||||
"s3-enabled-description": "S3/MinIO storage configuration",
|
||||
"s3-endpoint": "S3 Endpoint",
|
||||
"s3-endpoint-description": "S3/MinIO server endpoint",
|
||||
"s3-bucket": "S3 Bucket",
|
||||
"s3-bucket-description": "S3/MinIO bucket name",
|
||||
"s3-region": "S3 Region",
|
||||
"s3-region-description": "S3 region",
|
||||
"s3-access-key": "S3 Access Key",
|
||||
"s3-access-key-placeholder": "Access key is configured via environment",
|
||||
"s3-access-key-description": "S3/MinIO access key",
|
||||
"s3-secret-key": "S3 Secret Key",
|
||||
"s3-secret-key-placeholder": "Enter new secret key to update",
|
||||
"s3-secret-key-description": "S3/MinIO secret key (only save new ones)",
|
||||
"s3-ssl-enabled": "SSL Enabled",
|
||||
"s3-ssl-enabled-description": "Use SSL for S3/MinIO connections",
|
||||
"s3-port": "S3 Port",
|
||||
"s3-port-description": "S3/MinIO server port",
|
||||
"test-s3-connection": "Test S3 Connection",
|
||||
"save-s3-settings": "Save S3 Settings",
|
||||
"s3-connection-success": "S3 connection test successful",
|
||||
"s3-connection-failed": "S3 connection test failed",
|
||||
"s3-settings-saved": "S3 settings saved successfully",
|
||||
"s3-settings-save-failed": "Failed to save S3 settings",
|
||||
"s3-secret-key-required": "S3 secret key is required",
|
||||
"attachment-migration": "Attachment Migration",
|
||||
"migration-batch-size": "Batch Size",
|
||||
"migration-batch-size-description": "Number of attachments to process in each batch (1-100)",
|
||||
"migration-delay-ms": "Delay (ms)",
|
||||
"migration-delay-ms-description": "Delay between batches in milliseconds (100-10000)",
|
||||
"migration-cpu-threshold": "CPU Threshold (%)",
|
||||
"migration-cpu-threshold-description": "Pause migration when CPU usage exceeds this percentage (10-90)",
|
||||
"migrate-all-to-filesystem": "Migrate All to Filesystem",
|
||||
"migrate-all-to-gridfs": "Migrate All to GridFS",
|
||||
"migrate-all-to-s3": "Migrate All to S3",
|
||||
"pause-migration": "Pause Migration",
|
||||
"resume-migration": "Resume Migration",
|
||||
"stop-migration": "Stop Migration",
|
||||
"migration-progress": "Migration Progress",
|
||||
"total-attachments": "Total Attachments",
|
||||
"migrated-attachments": "Migrated Attachments",
|
||||
"remaining-attachments": "Remaining Attachments",
|
||||
"migration-status": "Migration Status",
|
||||
"migration-log": "Migration Log",
|
||||
"migration-started": "Migration started",
|
||||
"migration-paused": "Migration paused",
|
||||
"migration-resumed": "Migration resumed",
|
||||
"migration-stopped": "Migration stopped",
|
||||
"migration-start-failed": "Failed to start migration",
|
||||
"migration-pause-failed": "Failed to pause migration",
|
||||
"migration-resume-failed": "Failed to resume migration",
|
||||
"migration-stop-failed": "Failed to stop migration",
|
||||
"migration-stop-confirm": "Are you sure you want to stop the migration?",
|
||||
"attachment-monitoring": "Attachment Monitoring",
|
||||
"filesystem-attachments": "Filesystem Attachments",
|
||||
"gridfs-attachments": "GridFS Attachments",
|
||||
"s3-attachments": "S3 Attachments",
|
||||
"total-size": "Total Size",
|
||||
"filesystem-size": "Filesystem Size",
|
||||
"gridfs-size": "GridFS Size",
|
||||
"s3-size": "S3 Size",
|
||||
"storage-distribution": "Storage Distribution",
|
||||
"refresh-monitoring": "Refresh Monitoring",
|
||||
"export-monitoring": "Export Monitoring",
|
||||
"monitoring-refresh-failed": "Failed to refresh monitoring data",
|
||||
"monitoring-export-failed": "Failed to export monitoring data",
|
||||
"filesystem-storage": "Filesystem",
|
||||
"gridfs-storage": "GridFS",
|
||||
"s3-storage": "S3"
|
||||
}
|
||||
572
server/attachmentMigration.js
Normal file
572
server/attachmentMigration.js
Normal file
|
|
@ -0,0 +1,572 @@
|
|||
import { Meteor } from 'meteor/meteor';
|
||||
import { ReactiveCache } from '/imports/reactiveCache';
|
||||
import { Attachments, fileStoreStrategyFactory } from '/models/attachments';
|
||||
import { moveToStorage } from '/models/lib/fileStoreStrategy';
|
||||
import os from 'os';
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
// Migration state management
|
||||
const migrationState = {
|
||||
isRunning: false,
|
||||
isPaused: false,
|
||||
targetStorage: null,
|
||||
batchSize: 10,
|
||||
delayMs: 1000,
|
||||
cpuThreshold: 70,
|
||||
progress: 0,
|
||||
totalAttachments: 0,
|
||||
migratedAttachments: 0,
|
||||
currentBatch: [],
|
||||
migrationQueue: [],
|
||||
log: [],
|
||||
startTime: null,
|
||||
lastCpuCheck: 0
|
||||
};
|
||||
|
||||
// CPU monitoring
|
||||
function getCpuUsage() {
|
||||
const cpus = os.cpus();
|
||||
let totalIdle = 0;
|
||||
let totalTick = 0;
|
||||
|
||||
cpus.forEach(cpu => {
|
||||
for (const type in cpu.times) {
|
||||
totalTick += cpu.times[type];
|
||||
}
|
||||
totalIdle += cpu.times.idle;
|
||||
});
|
||||
|
||||
const idle = totalIdle / cpus.length;
|
||||
const total = totalTick / cpus.length;
|
||||
const usage = 100 - Math.floor(100 * idle / total);
|
||||
|
||||
return usage;
|
||||
}
|
||||
|
||||
// Logging function
|
||||
function addToLog(message) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const logEntry = `[${timestamp}] ${message}`;
|
||||
migrationState.log.unshift(logEntry);
|
||||
|
||||
// Keep only last 100 log entries
|
||||
if (migrationState.log.length > 100) {
|
||||
migrationState.log = migrationState.log.slice(0, 100);
|
||||
}
|
||||
|
||||
console.log(logEntry);
|
||||
}
|
||||
|
||||
// Get migration status
|
||||
function getMigrationStatus() {
|
||||
return {
|
||||
isRunning: migrationState.isRunning,
|
||||
isPaused: migrationState.isPaused,
|
||||
targetStorage: migrationState.targetStorage,
|
||||
progress: migrationState.progress,
|
||||
totalAttachments: migrationState.totalAttachments,
|
||||
migratedAttachments: migrationState.migratedAttachments,
|
||||
remainingAttachments: migrationState.totalAttachments - migrationState.migratedAttachments,
|
||||
status: migrationState.isRunning ? (migrationState.isPaused ? 'paused' : 'running') : 'idle',
|
||||
log: migrationState.log.slice(0, 10).join('\n'), // Return last 10 log entries
|
||||
startTime: migrationState.startTime,
|
||||
estimatedTimeRemaining: calculateEstimatedTimeRemaining()
|
||||
};
|
||||
}
|
||||
|
||||
// Calculate estimated time remaining
|
||||
function calculateEstimatedTimeRemaining() {
|
||||
if (!migrationState.isRunning || migrationState.migratedAttachments === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const elapsed = Date.now() - migrationState.startTime;
|
||||
const rate = migrationState.migratedAttachments / elapsed; // attachments per ms
|
||||
const remaining = migrationState.totalAttachments - migrationState.migratedAttachments;
|
||||
|
||||
return Math.round(remaining / rate);
|
||||
}
|
||||
|
||||
// Process a single attachment migration
|
||||
function migrateAttachment(attachmentId) {
|
||||
try {
|
||||
const attachment = ReactiveCache.getAttachment(attachmentId);
|
||||
if (!attachment) {
|
||||
addToLog(`Warning: Attachment ${attachmentId} not found`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if already in target storage
|
||||
const currentStorage = fileStoreStrategyFactory.getFileStrategy(attachment, 'original').getStorageName();
|
||||
if (currentStorage === migrationState.targetStorage) {
|
||||
addToLog(`Attachment ${attachmentId} already in target storage ${migrationState.targetStorage}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Perform migration
|
||||
moveToStorage(attachment, migrationState.targetStorage, fileStoreStrategyFactory);
|
||||
addToLog(`Migrated attachment ${attachmentId} from ${currentStorage} to ${migrationState.targetStorage}`);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
addToLog(`Error migrating attachment ${attachmentId}: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Process a batch of attachments
|
||||
function processBatch() {
|
||||
if (!migrationState.isRunning || migrationState.isPaused) {
|
||||
return;
|
||||
}
|
||||
|
||||
const batch = migrationState.migrationQueue.splice(0, migrationState.batchSize);
|
||||
if (batch.length === 0) {
|
||||
// Migration complete
|
||||
migrationState.isRunning = false;
|
||||
migrationState.progress = 100;
|
||||
addToLog(`Migration completed. Migrated ${migrationState.migratedAttachments} attachments.`);
|
||||
return;
|
||||
}
|
||||
|
||||
let successCount = 0;
|
||||
batch.forEach(attachmentId => {
|
||||
if (migrateAttachment(attachmentId)) {
|
||||
successCount++;
|
||||
migrationState.migratedAttachments++;
|
||||
}
|
||||
});
|
||||
|
||||
// Update progress
|
||||
migrationState.progress = Math.round((migrationState.migratedAttachments / migrationState.totalAttachments) * 100);
|
||||
|
||||
addToLog(`Processed batch: ${successCount}/${batch.length} successful. Progress: ${migrationState.progress}%`);
|
||||
|
||||
// Check CPU usage
|
||||
const currentTime = Date.now();
|
||||
if (currentTime - migrationState.lastCpuCheck > 5000) { // Check every 5 seconds
|
||||
const cpuUsage = getCpuUsage();
|
||||
migrationState.lastCpuCheck = currentTime;
|
||||
|
||||
if (cpuUsage > migrationState.cpuThreshold) {
|
||||
addToLog(`CPU usage ${cpuUsage}% exceeds threshold ${migrationState.cpuThreshold}%. Pausing migration.`);
|
||||
migrationState.isPaused = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Schedule next batch
|
||||
if (migrationState.isRunning && !migrationState.isPaused) {
|
||||
Meteor.setTimeout(() => {
|
||||
processBatch();
|
||||
}, migrationState.delayMs);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize migration queue
|
||||
function initializeMigrationQueue() {
|
||||
const allAttachments = ReactiveCache.getAttachments();
|
||||
migrationState.totalAttachments = allAttachments.length;
|
||||
migrationState.migrationQueue = allAttachments.map(attachment => attachment._id);
|
||||
migrationState.migratedAttachments = 0;
|
||||
migrationState.progress = 0;
|
||||
|
||||
addToLog(`Initialized migration queue with ${migrationState.totalAttachments} attachments`);
|
||||
}
|
||||
|
||||
// Start migration
|
||||
function startMigration(targetStorage, batchSize, delayMs, cpuThreshold) {
|
||||
if (migrationState.isRunning) {
|
||||
throw new Meteor.Error('migration-already-running', 'Migration is already running');
|
||||
}
|
||||
|
||||
migrationState.isRunning = true;
|
||||
migrationState.isPaused = false;
|
||||
migrationState.targetStorage = targetStorage;
|
||||
migrationState.batchSize = batchSize;
|
||||
migrationState.delayMs = delayMs;
|
||||
migrationState.cpuThreshold = cpuThreshold;
|
||||
migrationState.startTime = Date.now();
|
||||
migrationState.lastCpuCheck = 0;
|
||||
|
||||
initializeMigrationQueue();
|
||||
addToLog(`Started migration to ${targetStorage} with batch size ${batchSize}, delay ${delayMs}ms, CPU threshold ${cpuThreshold}%`);
|
||||
|
||||
// Start processing
|
||||
processBatch();
|
||||
}
|
||||
|
||||
// Pause migration
|
||||
function pauseMigration() {
|
||||
if (!migrationState.isRunning) {
|
||||
throw new Meteor.Error('migration-not-running', 'No migration is currently running');
|
||||
}
|
||||
|
||||
migrationState.isPaused = true;
|
||||
addToLog('Migration paused');
|
||||
}
|
||||
|
||||
// Resume migration
|
||||
function resumeMigration() {
|
||||
if (!migrationState.isRunning) {
|
||||
throw new Meteor.Error('migration-not-running', 'No migration is currently running');
|
||||
}
|
||||
|
||||
if (!migrationState.isPaused) {
|
||||
throw new Meteor.Error('migration-not-paused', 'Migration is not paused');
|
||||
}
|
||||
|
||||
migrationState.isPaused = false;
|
||||
addToLog('Migration resumed');
|
||||
|
||||
// Continue processing
|
||||
processBatch();
|
||||
}
|
||||
|
||||
// Stop migration
|
||||
function stopMigration() {
|
||||
if (!migrationState.isRunning) {
|
||||
throw new Meteor.Error('migration-not-running', 'No migration is currently running');
|
||||
}
|
||||
|
||||
migrationState.isRunning = false;
|
||||
migrationState.isPaused = false;
|
||||
migrationState.migrationQueue = [];
|
||||
addToLog('Migration stopped');
|
||||
}
|
||||
|
||||
// Get attachment storage configuration
|
||||
function getAttachmentStorageConfiguration() {
|
||||
const config = {
|
||||
filesystemPath: process.env.WRITABLE_PATH || '/data',
|
||||
attachmentsPath: `${process.env.WRITABLE_PATH || '/data'}/attachments`,
|
||||
avatarsPath: `${process.env.WRITABLE_PATH || '/data'}/avatars`,
|
||||
gridfsEnabled: true, // Always available
|
||||
s3Enabled: false,
|
||||
s3Endpoint: '',
|
||||
s3Bucket: '',
|
||||
s3Region: '',
|
||||
s3SslEnabled: false,
|
||||
s3Port: 443
|
||||
};
|
||||
|
||||
// Check S3 configuration
|
||||
if (process.env.S3) {
|
||||
try {
|
||||
const s3Config = JSON.parse(process.env.S3).s3;
|
||||
if (s3Config && s3Config.key && s3Config.secret && s3Config.bucket) {
|
||||
config.s3Enabled = true;
|
||||
config.s3Endpoint = s3Config.endPoint || '';
|
||||
config.s3Bucket = s3Config.bucket || '';
|
||||
config.s3Region = s3Config.region || '';
|
||||
config.s3SslEnabled = s3Config.sslEnabled || false;
|
||||
config.s3Port = s3Config.port || 443;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error parsing S3 configuration:', error);
|
||||
}
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
// Get attachment monitoring data
|
||||
function getAttachmentMonitoringData() {
|
||||
const attachments = ReactiveCache.getAttachments();
|
||||
const stats = {
|
||||
totalAttachments: attachments.length,
|
||||
filesystemAttachments: 0,
|
||||
gridfsAttachments: 0,
|
||||
s3Attachments: 0,
|
||||
totalSize: 0,
|
||||
filesystemSize: 0,
|
||||
gridfsSize: 0,
|
||||
s3Size: 0
|
||||
};
|
||||
|
||||
attachments.forEach(attachment => {
|
||||
const storage = fileStoreStrategyFactory.getFileStrategy(attachment, 'original').getStorageName();
|
||||
const size = attachment.size || 0;
|
||||
|
||||
stats.totalSize += size;
|
||||
|
||||
switch (storage) {
|
||||
case 'fs':
|
||||
stats.filesystemAttachments++;
|
||||
stats.filesystemSize += size;
|
||||
break;
|
||||
case 'gridfs':
|
||||
stats.gridfsAttachments++;
|
||||
stats.gridfsSize += size;
|
||||
break;
|
||||
case 's3':
|
||||
stats.s3Attachments++;
|
||||
stats.s3Size += size;
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
// Test S3 connection
|
||||
function testS3Connection(s3Config) {
|
||||
// This would implement actual S3 connection testing
|
||||
// For now, we'll just validate the configuration
|
||||
if (!s3Config.secretKey) {
|
||||
throw new Meteor.Error('s3-secret-key-required', 'S3 secret key is required');
|
||||
}
|
||||
|
||||
// In a real implementation, you would test the connection here
|
||||
// For now, we'll just return success
|
||||
return { success: true, message: 'S3 connection test successful' };
|
||||
}
|
||||
|
||||
// Save S3 settings
|
||||
function saveS3Settings(s3Config) {
|
||||
if (!s3Config.secretKey) {
|
||||
throw new Meteor.Error('s3-secret-key-required', 'S3 secret key is required');
|
||||
}
|
||||
|
||||
// In a real implementation, you would save the S3 configuration
|
||||
// For now, we'll just return success
|
||||
return { success: true, message: 'S3 settings saved successfully' };
|
||||
}
|
||||
|
||||
// Meteor methods
|
||||
if (Meteor.isServer) {
|
||||
Meteor.methods({
|
||||
// Migration methods
|
||||
'startAttachmentMigration'(config) {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
startMigration(config.targetStorage, config.batchSize, config.delayMs, config.cpuThreshold);
|
||||
return { success: true, message: 'Migration started' };
|
||||
},
|
||||
|
||||
'pauseAttachmentMigration'() {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
pauseMigration();
|
||||
return { success: true, message: 'Migration paused' };
|
||||
},
|
||||
|
||||
'resumeAttachmentMigration'() {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
resumeMigration();
|
||||
return { success: true, message: 'Migration resumed' };
|
||||
},
|
||||
|
||||
'stopAttachmentMigration'() {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
stopMigration();
|
||||
return { success: true, message: 'Migration stopped' };
|
||||
},
|
||||
|
||||
'getAttachmentMigrationSettings'() {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
return {
|
||||
batchSize: migrationState.batchSize,
|
||||
delayMs: migrationState.delayMs,
|
||||
cpuThreshold: migrationState.cpuThreshold,
|
||||
status: migrationState.isRunning ? (migrationState.isPaused ? 'paused' : 'running') : 'idle',
|
||||
progress: migrationState.progress
|
||||
};
|
||||
},
|
||||
|
||||
// Configuration methods
|
||||
'getAttachmentStorageConfiguration'() {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
return getAttachmentStorageConfiguration();
|
||||
},
|
||||
|
||||
'testS3Connection'(s3Config) {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
return testS3Connection(s3Config);
|
||||
},
|
||||
|
||||
'saveS3Settings'(s3Config) {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
return saveS3Settings(s3Config);
|
||||
},
|
||||
|
||||
// Monitoring methods
|
||||
'getAttachmentMonitoringData'() {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
return getAttachmentMonitoringData();
|
||||
},
|
||||
|
||||
'refreshAttachmentMonitoringData'() {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
return getAttachmentMonitoringData();
|
||||
},
|
||||
|
||||
'exportAttachmentMonitoringData'() {
|
||||
if (!this.userId) {
|
||||
throw new Meteor.Error('not-authorized', 'Must be logged in');
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
throw new Meteor.Error('not-authorized', 'Admin access required');
|
||||
}
|
||||
|
||||
const monitoringData = getAttachmentMonitoringData();
|
||||
const migrationStatus = getMigrationStatus();
|
||||
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
monitoring: monitoringData,
|
||||
migration: migrationStatus,
|
||||
system: {
|
||||
cpuUsage: getCpuUsage(),
|
||||
memoryUsage: process.memoryUsage(),
|
||||
uptime: process.uptime()
|
||||
}
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
// Publications
|
||||
Meteor.publish('attachmentMigrationStatus', function() {
|
||||
if (!this.userId) {
|
||||
return this.ready();
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
return this.ready();
|
||||
}
|
||||
|
||||
const self = this;
|
||||
let handle;
|
||||
|
||||
function updateStatus() {
|
||||
const status = getMigrationStatus();
|
||||
self.changed('attachmentMigrationStatus', 'status', status);
|
||||
}
|
||||
|
||||
self.added('attachmentMigrationStatus', 'status', getMigrationStatus());
|
||||
|
||||
// Update every 2 seconds
|
||||
handle = Meteor.setInterval(updateStatus, 2000);
|
||||
|
||||
self.ready();
|
||||
|
||||
self.onStop(() => {
|
||||
if (handle) {
|
||||
Meteor.clearInterval(handle);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
Meteor.publish('attachmentMonitoringData', function() {
|
||||
if (!this.userId) {
|
||||
return this.ready();
|
||||
}
|
||||
|
||||
const user = ReactiveCache.getUser(this.userId);
|
||||
if (!user || !user.isAdmin) {
|
||||
return this.ready();
|
||||
}
|
||||
|
||||
const self = this;
|
||||
let handle;
|
||||
|
||||
function updateMonitoring() {
|
||||
const data = getAttachmentMonitoringData();
|
||||
self.changed('attachmentMonitoringData', 'data', data);
|
||||
}
|
||||
|
||||
self.added('attachmentMonitoringData', 'data', getAttachmentMonitoringData());
|
||||
|
||||
// Update every 10 seconds
|
||||
handle = Meteor.setInterval(updateMonitoring, 10000);
|
||||
|
||||
self.ready();
|
||||
|
||||
self.onStop(() => {
|
||||
if (handle) {
|
||||
Meteor.clearInterval(handle);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue