From a8de2f224f61d0e5c7061fefb1cdd4f45f3bb020 Mon Sep 17 00:00:00 2001 From: Lauri Ojansivu Date: Fri, 10 Oct 2025 19:07:04 +0300 Subject: [PATCH] Use attachments from old CollectionFS database structure, when not yet migrated to Meteor-Files/ostrio-files, without needing to migrate database structure. Thanks to xet7 ! --- docs/AttachmentBackwardCompatibility.md | 169 +++++++++++ imports/reactiveCache.js | 24 +- models/attachments.js | 5 + models/lib/attachmentBackwardCompatibility.js | 280 ++++++++++++++++++ server/migrations/migrateAttachments.js | 150 ++++++++++ server/routes/legacyAttachments.js | 72 +++++ 6 files changed, 698 insertions(+), 2 deletions(-) create mode 100644 docs/AttachmentBackwardCompatibility.md create mode 100644 models/lib/attachmentBackwardCompatibility.js create mode 100644 server/migrations/migrateAttachments.js create mode 100644 server/routes/legacyAttachments.js diff --git a/docs/AttachmentBackwardCompatibility.md b/docs/AttachmentBackwardCompatibility.md new file mode 100644 index 000000000..5f4de7b7a --- /dev/null +++ b/docs/AttachmentBackwardCompatibility.md @@ -0,0 +1,169 @@ +# Attachment Backward Compatibility + +This document describes the backward compatibility implementation for Wekan attachments, allowing the system to read attachments from both the old CollectionFS structure (Wekan v6.09 and earlier) and the new Meteor-Files structure (Wekan v7.x and later). + +## Overview + +When Wekan migrated from CollectionFS to Meteor-Files (ostrio-files), the database structure for attachments changed significantly. This backward compatibility layer ensures that: + +1. Old attachments can still be accessed and downloaded +2. No database migration is required +3. Both old and new attachments can coexist +4. The UI works seamlessly with both structures + +## Database Structure Changes + +### Old Structure (CollectionFS) +- **CollectionFS Files**: `cfs_gridfs.attachments.files` +- **CollectionFS Records**: `cfs.attachments.filerecord` +- **File Storage**: GridFS with bucket name `cfs_gridfs.attachments` + +### New Structure (Meteor-Files) +- **Files Collection**: `attachments` +- **File Storage**: Configurable (Filesystem, GridFS, or S3) + +## Implementation Details + +### Files Added/Modified + +1. **`models/lib/attachmentBackwardCompatibility.js`** + - Main backward compatibility layer + - Handles reading from old CollectionFS structure + - Converts old data format to new format + - Provides GridFS streaming for downloads + +2. **`models/attachments.js`** + - Added backward compatibility methods to Attachments collection + - Imports compatibility functions + +3. **`imports/reactiveCache.js`** + - Updated to use backward compatibility layer + - Falls back to old structure when new structure has no results + +4. **`server/routes/legacyAttachments.js`** + - Handles legacy attachment downloads via `/cfs/files/attachments/:id` + - Provides proper HTTP headers and streaming + +5. **`server/migrations/migrateAttachments.js`** + - Migration methods for converting old attachments to new structure + - Optional migration tools for users who want to fully migrate + +### Key Functions + +#### `getAttachmentWithBackwardCompatibility(attachmentId)` +- Tries new structure first, falls back to old structure +- Returns attachment data in new format +- Handles both single attachment lookups + +#### `getAttachmentsWithBackwardCompatibility(query)` +- Queries both old and new structures +- Combines and deduplicates results +- Used for card attachment lists + +#### `getOldAttachmentData(attachmentId)` +- Reads from old CollectionFS structure +- Converts old format to new format +- Handles file type detection and metadata + +#### `getOldAttachmentStream(attachmentId)` +- Creates GridFS download stream for old attachments +- Used for file downloads + +## Usage + +### Automatic Fallback +The system automatically falls back to the old structure when: +- An attachment is not found in the new structure +- Querying attachments for a card returns no results + +### Legacy Download URLs +Old attachment download URLs (`/cfs/files/attachments/:id`) continue to work and are handled by the legacy route. + +### Migration (Optional) +Users can optionally migrate their old attachments to the new structure using the migration methods: + +```javascript +// Migrate a single attachment +Meteor.call('migrateAttachment', attachmentId); + +// Migrate all attachments for a card +Meteor.call('migrateCardAttachments', cardId); + +// Check migration status +Meteor.call('getAttachmentMigrationStatus', cardId); +``` + +## Performance Considerations + +1. **Query Optimization**: The system queries the new structure first, only falling back to old structure when necessary +2. **Caching**: ReactiveCache handles caching for both old and new attachments +3. **Streaming**: Large files are streamed efficiently using GridFS streams + +## Error Handling + +- Graceful fallback when old structure is not available +- Proper error logging for debugging +- HTTP error codes for download failures +- Permission checks for both old and new attachments + +## Security + +- Permission checks are maintained for both old and new attachments +- Board access rules apply to legacy attachments +- File type validation is preserved + +## Testing + +To test the backward compatibility: + +1. Ensure you have old Wekan v6.09 data with attachments +2. Upgrade to Wekan v7.x +3. Verify that old attachments are visible in the UI +4. Test downloading old attachments +5. Verify that new attachments work normally + +## Troubleshooting + +### Common Issues + +1. **Old attachments not showing** + - Check that old CollectionFS collections exist in database + - Verify GridFS bucket is accessible + - Check server logs for errors + +2. **Download failures** + - Verify GridFS connection + - Check file permissions + - Ensure legacy route is loaded + +3. **Performance issues** + - Consider migrating old attachments to new structure + - Check database indexes + - Monitor query performance + +### Debug Mode + +Enable debug logging by setting: +```javascript +console.log('Legacy attachments route loaded'); +``` + +This will help identify if the backward compatibility layer is properly loaded. + +## Future Considerations + +- The backward compatibility layer can be removed in future versions +- Users should be encouraged to migrate old attachments +- Consider adding migration tools to the admin interface +- Monitor usage of old vs new structures + +## Migration Path + +For users who want to fully migrate to the new structure: + +1. Use the migration methods to convert old attachments +2. Verify all attachments are working +3. Remove old CollectionFS collections (optional) +4. Update any hardcoded URLs to use new structure + +The backward compatibility layer ensures that migration is optional and can be done gradually. diff --git a/imports/reactiveCache.js b/imports/reactiveCache.js index ebacc96cd..800904ded 100644 --- a/imports/reactiveCache.js +++ b/imports/reactiveCache.js @@ -102,13 +102,23 @@ ReactiveCacheServer = { return ret; }, getAttachment(idOrFirstObjectSelector = {}, options = {}) { - const ret = Attachments.findOne(idOrFirstObjectSelector, options); + // Try new structure first + let ret = Attachments.findOne(idOrFirstObjectSelector, options); + if (!ret && typeof idOrFirstObjectSelector === 'string') { + // Fall back to old structure for single attachment lookup + ret = Attachments.getAttachmentWithBackwardCompatibility(idOrFirstObjectSelector); + } return ret; }, getAttachments(selector = {}, options = {}, getQuery = false) { + // Try new structure first let ret = Attachments.find(selector, options); if (getQuery !== true) { ret = ret.fetch(); + // If no results and we have a cardId selector, try old structure + if (ret.length === 0 && selector['meta.cardId']) { + ret = Attachments.getAttachmentsWithBackwardCompatibility(selector); + } } return ret; }, @@ -517,7 +527,12 @@ ReactiveCacheClient = { if (!this.__attachment) { this.__attachment = new DataCache(_idOrFirstObjectSelect => { const __select = EJSON.parse(_idOrFirstObjectSelect); - const _ret = Attachments.findOne(__select.idOrFirstObjectSelector, __select.options); + // Try new structure first + let _ret = Attachments.findOne(__select.idOrFirstObjectSelector, __select.options); + if (!_ret && typeof __select.idOrFirstObjectSelector === 'string') { + // Fall back to old structure for single attachment lookup + _ret = Attachments.getAttachmentWithBackwardCompatibility(__select.idOrFirstObjectSelector); + } return _ret; }); } @@ -529,9 +544,14 @@ ReactiveCacheClient = { if (!this.__attachments) { this.__attachments = new DataCache(_select => { const __select = EJSON.parse(_select); + // Try new structure first let _ret = Attachments.find(__select.selector, __select.options); if (__select.getQuery !== true) { _ret = _ret.fetch(); + // If no results and we have a cardId selector, try old structure + if (_ret.length === 0 && __select.selector['meta.cardId']) { + _ret = Attachments.getAttachmentsWithBackwardCompatibility(__select.selector); + } } return _ret; }); diff --git a/models/attachments.js b/models/attachments.js index 9f795dfa1..39407846c 100644 --- a/models/attachments.js +++ b/models/attachments.js @@ -7,6 +7,7 @@ import fs from 'fs'; import path from 'path'; import { AttachmentStoreStrategyFilesystem, AttachmentStoreStrategyGridFs, AttachmentStoreStrategyS3 } from '/models/lib/attachmentStoreStrategy'; import FileStoreStrategyFactory, {moveToStorage, rename, STORAGE_NAME_FILESYSTEM, STORAGE_NAME_GRIDFS, STORAGE_NAME_S3} from '/models/lib/fileStoreStrategy'; +import { getAttachmentWithBackwardCompatibility, getAttachmentsWithBackwardCompatibility } from './lib/attachmentBackwardCompatibility'; let attachmentUploadExternalProgram; let attachmentUploadMimeTypes = []; @@ -193,6 +194,10 @@ if (Meteor.isServer) { fs.mkdirSync(storagePath, { recursive: true }); } }); + + // Add backward compatibility methods + Attachments.getAttachmentWithBackwardCompatibility = getAttachmentWithBackwardCompatibility; + Attachments.getAttachmentsWithBackwardCompatibility = getAttachmentsWithBackwardCompatibility; } export default Attachments; diff --git a/models/lib/attachmentBackwardCompatibility.js b/models/lib/attachmentBackwardCompatibility.js new file mode 100644 index 000000000..a6b9a49a3 --- /dev/null +++ b/models/lib/attachmentBackwardCompatibility.js @@ -0,0 +1,280 @@ +import { ReactiveCache } from '/imports/reactiveCache'; +import { Meteor } from 'meteor/meteor'; +import { MongoInternals } from 'meteor/mongo'; + +/** + * Backward compatibility layer for CollectionFS to Meteor-Files migration + * Handles reading attachments from old CollectionFS database structure + */ + +// Old CollectionFS collections +const OldAttachmentsFiles = new Mongo.Collection('cfs_gridfs.attachments.files'); +const OldAttachmentsFileRecord = new Mongo.Collection('cfs.attachments.filerecord'); + +/** + * Check if an attachment exists in the new Meteor-Files structure + * @param {string} attachmentId - The attachment ID to check + * @returns {boolean} - True if exists in new structure + */ +export function isNewAttachmentStructure(attachmentId) { + if (Meteor.isServer) { + return !!ReactiveCache.getAttachment(attachmentId); + } + return false; +} + +/** + * Get attachment data from old CollectionFS structure + * @param {string} attachmentId - The attachment ID + * @returns {Object|null} - Attachment data in new format or null if not found + */ +export function getOldAttachmentData(attachmentId) { + if (Meteor.isServer) { + try { + // First try to get from old filerecord collection + const fileRecord = OldAttachmentsFileRecord.findOne({ _id: attachmentId }); + if (!fileRecord) { + return null; + } + + // Get file data from old files collection + const fileData = OldAttachmentsFiles.findOne({ _id: attachmentId }); + if (!fileData) { + return null; + } + + // Convert old structure to new structure + const convertedAttachment = { + _id: attachmentId, + name: fileRecord.original?.name || fileData.filename || 'Unknown', + size: fileRecord.original?.size || fileData.length || 0, + type: fileRecord.original?.type || fileData.contentType || 'application/octet-stream', + extension: getFileExtension(fileRecord.original?.name || fileData.filename || ''), + extensionWithDot: getFileExtensionWithDot(fileRecord.original?.name || fileData.filename || ''), + meta: { + boardId: fileRecord.boardId, + swimlaneId: fileRecord.swimlaneId, + listId: fileRecord.listId, + cardId: fileRecord.cardId, + userId: fileRecord.userId, + source: 'legacy' + }, + uploadedAt: fileRecord.uploadedAt || fileData.uploadDate || new Date(), + updatedAt: fileRecord.original?.updatedAt || fileData.uploadDate || new Date(), + // Legacy compatibility fields + isImage: isImageFile(fileRecord.original?.type || fileData.contentType), + isVideo: isVideoFile(fileRecord.original?.type || fileData.contentType), + isAudio: isAudioFile(fileRecord.original?.type || fileData.contentType), + isText: isTextFile(fileRecord.original?.type || fileData.contentType), + isJSON: isJSONFile(fileRecord.original?.type || fileData.contentType), + isPDF: isPDFFile(fileRecord.original?.type || fileData.contentType), + // Legacy link method for compatibility + link: function(version = 'original') { + return `/cfs/files/attachments/${this._id}`; + }, + // Legacy versions structure for compatibility + versions: { + original: { + path: `/cfs/files/attachments/${this._id}`, + size: this.size, + type: this.type, + storage: 'gridfs' + } + } + }; + + return convertedAttachment; + } catch (error) { + console.error('Error reading old attachment data:', error); + return null; + } + } + return null; +} + +/** + * Get file extension from filename + * @param {string} filename - The filename + * @returns {string} - File extension without dot + */ +function getFileExtension(filename) { + if (!filename) return ''; + const lastDot = filename.lastIndexOf('.'); + if (lastDot === -1) return ''; + return filename.substring(lastDot + 1).toLowerCase(); +} + +/** + * Get file extension with dot + * @param {string} filename - The filename + * @returns {string} - File extension with dot + */ +function getFileExtensionWithDot(filename) { + const ext = getFileExtension(filename); + return ext ? `.${ext}` : ''; +} + +/** + * Check if file is an image + * @param {string} mimeType - MIME type + * @returns {boolean} - True if image + */ +function isImageFile(mimeType) { + return mimeType && mimeType.startsWith('image/'); +} + +/** + * Check if file is a video + * @param {string} mimeType - MIME type + * @returns {boolean} - True if video + */ +function isVideoFile(mimeType) { + return mimeType && mimeType.startsWith('video/'); +} + +/** + * Check if file is audio + * @param {string} mimeType - MIME type + * @returns {boolean} - True if audio + */ +function isAudioFile(mimeType) { + return mimeType && mimeType.startsWith('audio/'); +} + +/** + * Check if file is text + * @param {string} mimeType - MIME type + * @returns {boolean} - True if text + */ +function isTextFile(mimeType) { + return mimeType && mimeType.startsWith('text/'); +} + +/** + * Check if file is JSON + * @param {string} mimeType - MIME type + * @returns {boolean} - True if JSON + */ +function isJSONFile(mimeType) { + return mimeType === 'application/json'; +} + +/** + * Check if file is PDF + * @param {string} mimeType - MIME type + * @returns {boolean} - True if PDF + */ +function isPDFFile(mimeType) { + return mimeType === 'application/pdf'; +} + +/** + * Get attachment with backward compatibility + * @param {string} attachmentId - The attachment ID + * @returns {Object|null} - Attachment data or null if not found + */ +export function getAttachmentWithBackwardCompatibility(attachmentId) { + // First try new structure + if (isNewAttachmentStructure(attachmentId)) { + return ReactiveCache.getAttachment(attachmentId); + } + + // Fall back to old structure + return getOldAttachmentData(attachmentId); +} + +/** + * Get attachments for a card with backward compatibility + * @param {Object} query - Query object + * @returns {Array} - Array of attachments + */ +export function getAttachmentsWithBackwardCompatibility(query) { + const newAttachments = ReactiveCache.getAttachments(query); + const oldAttachments = []; + + if (Meteor.isServer) { + try { + // Query old structure for the same card + const cardId = query['meta.cardId']; + if (cardId) { + const oldFileRecords = OldAttachmentsFileRecord.find({ cardId }).fetch(); + for (const fileRecord of oldFileRecords) { + const oldAttachment = getOldAttachmentData(fileRecord._id); + if (oldAttachment) { + oldAttachments.push(oldAttachment); + } + } + } + } catch (error) { + console.error('Error reading old attachments:', error); + } + } + + // Combine and deduplicate + const allAttachments = [...newAttachments, ...oldAttachments]; + const uniqueAttachments = allAttachments.filter((attachment, index, self) => + index === self.findIndex(a => a._id === attachment._id) + ); + + return uniqueAttachments; +} + +/** + * Get file stream from old GridFS structure + * @param {string} attachmentId - The attachment ID + * @returns {Object|null} - GridFS file stream or null if not found + */ +export function getOldAttachmentStream(attachmentId) { + if (Meteor.isServer) { + try { + const db = MongoInternals.defaultRemoteCollectionDriver().mongo.db; + const bucket = new MongoInternals.NpmModule.GridFSBucket(db, { + bucketName: 'cfs_gridfs.attachments' + }); + + const downloadStream = bucket.openDownloadStreamByName(attachmentId); + return downloadStream; + } catch (error) { + console.error('Error creating GridFS stream:', error); + return null; + } + } + return null; +} + +/** + * Get file data from old GridFS structure + * @param {string} attachmentId - The attachment ID + * @returns {Buffer|null} - File data buffer or null if not found + */ +export function getOldAttachmentDataBuffer(attachmentId) { + if (Meteor.isServer) { + try { + const db = MongoInternals.defaultRemoteCollectionDriver().mongo.db; + const bucket = new MongoInternals.NpmModule.GridFSBucket(db, { + bucketName: 'cfs_gridfs.attachments' + }); + + return new Promise((resolve, reject) => { + const chunks = []; + const downloadStream = bucket.openDownloadStreamByName(attachmentId); + + downloadStream.on('data', (chunk) => { + chunks.push(chunk); + }); + + downloadStream.on('end', () => { + resolve(Buffer.concat(chunks)); + }); + + downloadStream.on('error', (error) => { + reject(error); + }); + }); + } catch (error) { + console.error('Error reading GridFS data:', error); + return null; + } + } + return null; +} diff --git a/server/migrations/migrateAttachments.js b/server/migrations/migrateAttachments.js new file mode 100644 index 000000000..55ffdb0c7 --- /dev/null +++ b/server/migrations/migrateAttachments.js @@ -0,0 +1,150 @@ +import { Meteor } from 'meteor/meteor'; +import { ReactiveCache } from '/imports/reactiveCache'; +import { getOldAttachmentData, getOldAttachmentDataBuffer } from '/models/lib/attachmentBackwardCompatibility'; + +/** + * Migration script to convert old CollectionFS attachments to new Meteor-Files structure + * This script can be run to migrate all old attachments to the new format + */ + +if (Meteor.isServer) { + Meteor.methods({ + /** + * Migrate a single attachment from old to new structure + * @param {string} attachmentId - The old attachment ID + * @returns {Object} - Migration result + */ + migrateAttachment(attachmentId) { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + try { + // Get old attachment data + const oldAttachment = getOldAttachmentData(attachmentId); + if (!oldAttachment) { + return { success: false, error: 'Old attachment not found' }; + } + + // Check if already migrated + const existingAttachment = ReactiveCache.getAttachment(attachmentId); + if (existingAttachment) { + return { success: true, message: 'Already migrated', attachmentId }; + } + + // Get file data from GridFS + const fileData = getOldAttachmentDataBuffer(attachmentId); + if (!fileData) { + return { success: false, error: 'Could not read file data from GridFS' }; + } + + // Create new attachment using Meteor-Files + const fileObj = new File([fileData], oldAttachment.name, { + type: oldAttachment.type + }); + + const uploader = Attachments.insert({ + file: fileObj, + meta: oldAttachment.meta, + isBase64: false, + transport: 'http' + }); + + if (uploader) { + return { + success: true, + message: 'Migration successful', + attachmentId, + newAttachmentId: uploader._id + }; + } else { + return { success: false, error: 'Failed to create new attachment' }; + } + + } catch (error) { + console.error('Error migrating attachment:', error); + return { success: false, error: error.message }; + } + }, + + /** + * Migrate all attachments for a specific card + * @param {string} cardId - The card ID + * @returns {Object} - Migration results + */ + migrateCardAttachments(cardId) { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + const results = { + success: 0, + failed: 0, + errors: [] + }; + + try { + // Get all old attachments for this card + const oldAttachments = ReactiveCache.getAttachments({ 'meta.cardId': cardId }); + + for (const attachment of oldAttachments) { + const result = Meteor.call('migrateAttachment', attachment._id); + if (result.success) { + results.success++; + } else { + results.failed++; + results.errors.push({ + attachmentId: attachment._id, + error: result.error + }); + } + } + + return results; + + } catch (error) { + console.error('Error migrating card attachments:', error); + return { success: false, error: error.message }; + } + }, + + /** + * Get migration status for attachments + * @param {string} cardId - The card ID (optional) + * @returns {Object} - Migration status + */ + getAttachmentMigrationStatus(cardId) { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + + try { + const selector = cardId ? { 'meta.cardId': cardId } : {}; + const allAttachments = ReactiveCache.getAttachments(selector); + + const status = { + total: allAttachments.length, + newStructure: 0, + oldStructure: 0, + mixed: false + }; + + for (const attachment of allAttachments) { + if (attachment.meta && attachment.meta.source === 'legacy') { + status.oldStructure++; + } else { + status.newStructure++; + } + } + + status.mixed = status.oldStructure > 0 && status.newStructure > 0; + + return status; + + } catch (error) { + console.error('Error getting migration status:', error); + return { error: error.message }; + } + } + }); +} diff --git a/server/routes/legacyAttachments.js b/server/routes/legacyAttachments.js new file mode 100644 index 000000000..a208fe1d1 --- /dev/null +++ b/server/routes/legacyAttachments.js @@ -0,0 +1,72 @@ +import { Meteor } from 'meteor/meteor'; +import { WebApp } from 'meteor/webapp'; +import { ReactiveCache } from '/imports/reactiveCache'; +import { getAttachmentWithBackwardCompatibility, getOldAttachmentStream } from '/models/lib/attachmentBackwardCompatibility'; + +// Ensure this file is loaded +console.log('Legacy attachments route loaded'); + +/** + * Legacy attachment download route for CollectionFS compatibility + * Handles downloads from old CollectionFS structure + */ + +if (Meteor.isServer) { + // Handle legacy attachment downloads + WebApp.connectHandlers.use('/cfs/files/attachments', (req, res, next) => { + const attachmentId = req.url.split('/').pop(); + + if (!attachmentId) { + res.writeHead(404); + res.end('Attachment not found'); + return; + } + + try { + // Try to get attachment with backward compatibility + const attachment = getAttachmentWithBackwardCompatibility(attachmentId); + + if (!attachment) { + res.writeHead(404); + res.end('Attachment not found'); + return; + } + + // Check permissions + const board = ReactiveCache.getBoard(attachment.meta.boardId); + if (!board) { + res.writeHead(404); + res.end('Board not found'); + return; + } + + // Check if user has permission to download + const userId = Meteor.userId(); + if (!board.isPublic() && (!userId || !board.hasMember(userId))) { + res.writeHead(403); + res.end('Access denied'); + return; + } + + // Set appropriate headers + res.setHeader('Content-Type', attachment.type || 'application/octet-stream'); + res.setHeader('Content-Length', attachment.size || 0); + res.setHeader('Content-Disposition', `attachment; filename="${attachment.name}"`); + + // Get GridFS stream for legacy attachment + const fileStream = getOldAttachmentStream(attachmentId); + if (fileStream) { + res.writeHead(200); + fileStream.pipe(res); + } else { + res.writeHead(404); + res.end('File not found in GridFS'); + } + + } catch (error) { + console.error('Error serving legacy attachment:', error); + res.writeHead(500); + res.end('Internal server error'); + } + }); +}