From 6612ce0193a2f87bc60521cfc1e317f62f584614 Mon Sep 17 00:00:00 2001 From: David Arnold Date: Tue, 20 Oct 2020 12:55:28 -0500 Subject: [PATCH] migration: extract files from old storage and load into new storage --- models/attachments_old.js | 116 +++++++++++++++++++++++++++ models/avatars_old.js | 29 +++++++ server/migrations.js | 162 ++++++++++++++++++++++++++++++-------- 3 files changed, 274 insertions(+), 33 deletions(-) create mode 100644 models/attachments_old.js create mode 100644 models/avatars_old.js diff --git a/models/attachments_old.js b/models/attachments_old.js new file mode 100644 index 000000000..3ef8a388e --- /dev/null +++ b/models/attachments_old.js @@ -0,0 +1,116 @@ +const storeName = 'attachments'; +const defaultStoreOptions = { + beforeWrite: fileObj => { + if (!fileObj.isImage()) { + return { + type: 'application/octet-stream', + }; + } + return {}; + }, +}; +let store; +store = new FS.Store.GridFS(storeName, { + // XXX Add a new store for cover thumbnails so we don't load big images in + // the general board view + // If the uploaded document is not an image we need to enforce browser + // download instead of execution. This is particularly important for HTML + // files that the browser will just execute if we don't serve them with the + // appropriate `application/octet-stream` MIME header which can lead to user + // data leaks. I imagine other formats (like PDF) can also be attack vectors. + // See https://github.com/wekan/wekan/issues/99 + // XXX Should we use `beforeWrite` option of CollectionFS instead of + // collection-hooks? + // We should use `beforeWrite`. + ...defaultStoreOptions, +}); +AttachmentsOld = new FS.Collection('attachments', { + stores: [store], +}); + +if (Meteor.isServer) { + Meteor.startup(() => { + AttachmentsOld.files._ensureIndex({ cardId: 1 }); + }); + + AttachmentsOld.allow({ + insert(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + update(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + remove(userId, doc) { + return allowIsBoardMember(userId, Boards.findOne(doc.boardId)); + }, + // We authorize the attachment download either: + // - if the board is public, everyone (even unconnected) can download it + // - if the board is private, only board members can download it + download(userId, doc) { + const board = Boards.findOne(doc.boardId); + if (board.isPublic()) { + return true; + } else { + return board.hasMember(userId); + } + }, + + fetch: ['boardId'], + }); +} + +// XXX Enforce a schema for the AttachmentsOld CollectionFS + +if (Meteor.isServer) { + AttachmentsOld.files.after.insert((userId, doc) => { + // If the attachment doesn't have a source field + // or its source is different than import + if (!doc.source || doc.source !== 'import') { + // Add activity about adding the attachment + Activities.insert({ + userId, + type: 'card', + activityType: 'addAttachment', + attachmentId: doc._id, + // this preserves the name so that notifications can be meaningful after + // this file is removed + attachmentName: doc.original.name, + boardId: doc.boardId, + cardId: doc.cardId, + listId: doc.listId, + swimlaneId: doc.swimlaneId, + }); + } else { + // Don't add activity about adding the attachment as the activity + // be imported and delete source field + AttachmentsOld.update( + { + _id: doc._id, + }, + { + $unset: { + source: '', + }, + }, + ); + } + }); + + AttachmentsOld.files.before.remove((userId, doc) => { + Activities.insert({ + userId, + type: 'card', + activityType: 'deleteAttachment', + attachmentId: doc._id, + // this preserves the name so that notifications can be meaningful after + // this file is removed + attachmentName: doc.original.name, + boardId: doc.boardId, + cardId: doc.cardId, + listId: doc.listId, + swimlaneId: doc.swimlaneId, + }); + }); +} + +export default AttachmentsOld; diff --git a/models/avatars_old.js b/models/avatars_old.js new file mode 100644 index 000000000..deae4bbc6 --- /dev/null +++ b/models/avatars_old.js @@ -0,0 +1,29 @@ +AvatarsOld = new FS.Collection('avatars', { + stores: [new FS.Store.GridFS('avatars')], + filter: { + maxSize: 72000, + allow: { + contentTypes: ['image/*'], + }, + }, +}); + +function isOwner(userId, file) { + return userId && userId === file.userId; +} + +AvatarsOld.allow({ + insert: isOwner, + update: isOwner, + remove: isOwner, + download() { + return true; + }, + fetch: ['userId'], +}); + +AvatarsOld.files.before.insert((userId, doc) => { + doc.userId = userId; +}); + +export default AvatarsOld; diff --git a/server/migrations.js b/server/migrations.js index cac1e017d..55bc644fb 100644 --- a/server/migrations.js +++ b/server/migrations.js @@ -3,6 +3,9 @@ import Actions from '../models/actions'; import Activities from '../models/activities'; import Announcements from '../models/announcements'; import Attachments from '../models/attachments'; +import AttachmentsOld from '../models/attachments_old'; +import Avatars from '../models/avatars'; +import AvatarsOld from '../models/avatars_old'; import Boards from '../models/boards'; import CardComments from '../models/cardComments'; import Cards from '../models/cards'; @@ -1061,38 +1064,131 @@ Migrations.add('add-hide-logo-by-default', () => { noValidateMulti, ); }); -Migrations.add( - 'adapt-attachments-to-ostrio-files-api-using-meta-and-drp-cfs-leacy', - () => { - const newTypeForNonImage = 'application/octet-stream'; - Attachments.find().forEach(file => { - Attachments.update( - file._id, - { - $set: { - 'meta.boardId': file.boardId, - 'meta.cardId': file.cardId, - 'meta.listId': file.listId, - 'meta.swimlaneId': file.swimlaneId, - }, - }, - noValidate, - ); +Migrations.add('migrate-attachments-collectionFS-to-ostrioFiles', () => { + AttachmentsOld.find().forEach(function(fileObj) { + //console.log('File: ', fileObj.userId); + + // This directory must be writable on server, so a test run first + // We are going to copy the files locally, then move them to S3 + const fileName = `./assets/app/uploads/attachments/${fileObj.name()}`; + const newFileName = fileObj.name(); + + // This is "example" variable, change it to the userId that you might be using. + const userId = fileObj.userId; + + const fileType = fileObj.type(); + const fileSize = fileObj.size(); + const fileId = fileObj._id; + + const readStream = fileObj.createReadStream('attachments'); + const writeStream = fs.createWriteStream(fileName); + + writeStream.on('error', function(err) { + console.log('Writing error: ', err, fileName); }); - Attachments.update( - {}, - { - $unset: { - original: '', // cfs:* legacy - copies: '', // cfs:* legacy - failures: '', // cfs:* legacy - boardId: '', - cardId: '', - listId: '', - swimlaneId: '', + + // Once we have a file, then upload it to our new data storage + readStream.on('end', () => { + console.log('Ended: ', fileName); + // UserFiles is the new Meteor-Files/FilesCollection collection instance + + Attachments.addFile( + fileName, + { + fileName: newFileName, + type: fileType, + meta: { + boardId: fileObj.boardId, + cardId: fileObj.cardId, + listId: fileObj.listId, + swimlaneId: fileObj.swimlaneId, + }, + userId, + size: fileSize, + fileId, }, - }, - noValidateMulti, - ); - }, -); + (err, fileRef) => { + if (err) { + console.log(err); + } else { + console.log('File Inserted: ', fileRef._id); + // Set the userId again + Attachments.update({ _id: fileRef._id }, { $set: { userId } }); + fileObj.remove(); + } + }, + true, + ); // proceedAfterUpload + }); + + readStream.on('error', error => { + console.log('Error: ', fileName, error); + }); + + readStream.pipe(writeStream); + }); +}); +Migrations.add('migrate-avatars-collectionFS-to-ostrioFiles', () => { + AvatarsOld.find().forEach(function(fileObj) { + //console.log('File: ', fileObj.userId); + + // This directory must be writable on server, so a test run first + // We are going to copy the files locally, then move them to S3 + const fileName = `./assets/app/uploads/avatars/${fileObj.name()}`; + const newFileName = fileObj.name(); + + // This is "example" variable, change it to the userId that you might be using. + const userId = fileObj.userId; + + const fileType = fileObj.type(); + const fileSize = fileObj.size(); + const fileId = fileObj._id; + + const readStream = fileObj.createReadStream('avatars'); + const writeStream = fs.createWriteStream(fileName); + + writeStream.on('error', function(err) { + console.log('Writing error: ', err, fileName); + }); + + // Once we have a file, then upload it to our new data storage + readStream.on('end', () => { + console.log('Ended: ', fileName); + // UserFiles is the new Meteor-Files/FilesCollection collection instance + + Avatars.addFile( + fileName, + { + fileName: newFileName, + type: fileType, + meta: { + boardId: fileObj.boardId, + cardId: fileObj.cardId, + listId: fileObj.listId, + swimlaneId: fileObj.swimlaneId, + }, + userId, + size: fileSize, + fileId, + }, + (err, fileRef) => { + if (err) { + console.log(err); + } else { + console.log('File Inserted: ', fileRef._id); + // Set the userId again + Avatars.update({ _id: fileRef._id }, { $set: { userId } }); + fileObj.remove(); + } + }, + true, + ); // proceedAfterUpload + }); + + readStream.on('error', error => { + console.log('Error: ', fileName, error); + }); + + readStream.pipe(writeStream); + }); +});