Multi-File Storage.

Thanks to mfilser !

Related https://github.com/wekan/wekan/pull/4484

Merge branch 'master' into upgrade-meteor
This commit is contained in:
Lauri Ojansivu 2022-04-22 00:55:42 +03:00
commit 68e8155805
29 changed files with 921 additions and 276 deletions

View file

@ -247,9 +247,8 @@ if (Meteor.isServer) {
params.commentId = comment._id;
}
if (activity.attachmentId) {
const attachment = activity.attachment();
params.attachment = attachment.name;
params.attachmentId = attachment._id;
params.attachment = activity.attachmentName;
params.attachmentId = activity.attachmentId;
}
if (activity.checklistId) {
const checklist = activity.checklist();

View file

@ -1,30 +1,17 @@
import { Meteor } from 'meteor/meteor';
import { FilesCollection } from 'meteor/ostrio:files';
import path from 'path';
import { createBucket } from './lib/grid/createBucket';
import { createOnAfterUpload } from './lib/fsHooks/createOnAfterUpload';
import { createInterceptDownload } from './lib/fsHooks/createInterceptDownload';
import { createOnAfterRemove } from './lib/fsHooks/createOnAfterRemove';
import { AttachmentStoreStrategyFilesystem, AttachmentStoreStrategyGridFs} from '/models/lib/attachmentStoreStrategy';
import FileStoreStrategyFactory, {moveToStorage, STORAGE_NAME_FILESYSTEM, STORAGE_NAME_GRIDFS} from '/models/lib/fileStoreStrategy';
let attachmentBucket;
let storagePath;
if (Meteor.isServer) {
attachmentBucket = createBucket('attachments');
storagePath = path.join(process.env.WRITABLE_PATH, 'attachments');
}
const insertActivity = (fileObj, activityType) =>
Activities.insert({
userId: fileObj.userId,
type: 'card',
activityType,
attachmentId: fileObj._id,
// this preserves the name so that notifications can be meaningful after
// this file is removed
attachmentName: fileObj.name,
boardId: fileObj.meta.boardId,
cardId: fileObj.meta.cardId,
listId: fileObj.meta.listId,
swimlaneId: fileObj.meta.swimlaneId,
});
export const fileStoreStrategyFactory = new FileStoreStrategyFactory(AttachmentStoreStrategyFilesystem, storagePath, AttachmentStoreStrategyGridFs, attachmentBucket);
// XXX Enforce a schema for the Attachments FilesCollection
// see: https://github.com/VeliovGroup/Meteor-Files/wiki/Schema
@ -33,26 +20,34 @@ Attachments = new FilesCollection({
debug: false, // Change to `true` for debugging
collectionName: 'attachments',
allowClientCode: true,
namingFunction(opts) {
const filenameWithoutExtension = opts.name.replace(/(.+)\..+/, "$1");
const ret = opts.meta.fileId + "-original-" + filenameWithoutExtension;
// remove fileId from meta, it was only stored there to have this information here in the namingFunction function
delete opts.meta.fileId;
return ret;
},
storagePath() {
if (process.env.WRITABLE_PATH) {
return path.join(process.env.WRITABLE_PATH, 'uploads', 'attachments');
}
return path.normalize(`assets/app/uploads/${this.collectionName}`);
const ret = fileStoreStrategyFactory.storagePath;
return ret;
},
onAfterUpload: function onAfterUpload(fileRef) {
createOnAfterUpload(attachmentBucket).call(this, fileRef);
// If the attachment doesn't have a source field
// or its source is different than import
if (!fileRef.meta.source || fileRef.meta.source !== 'import') {
// Add activity about adding the attachment
insertActivity(fileRef, 'addAttachment');
}
onAfterUpload(fileObj) {
// current storage is the filesystem, update object and database
Object.keys(fileObj.versions).forEach(versionName => {
fileObj.versions[versionName].storage = STORAGE_NAME_FILESYSTEM;
});
Attachments.update({ _id: fileObj._id }, { $set: { "versions" : fileObj.versions } });
moveToStorage(fileObj, STORAGE_NAME_GRIDFS, fileStoreStrategyFactory);
},
interceptDownload: createInterceptDownload(attachmentBucket),
onAfterRemove: function onAfterRemove(files) {
createOnAfterRemove(attachmentBucket).call(this, files);
interceptDownload(http, fileObj, versionName) {
const ret = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName).interceptDownload(http, this.cacheControl);
return ret;
},
onAfterRemove(files) {
files.forEach(fileObj => {
insertActivity(fileObj, 'deleteAttachment');
Object.keys(fileObj.versions).forEach(versionName => {
fileStoreStrategyFactory.getFileStrategy(fileObj, versionName).onAfterRemove();
});
});
},
// We authorize the attachment download either:
@ -81,6 +76,16 @@ if (Meteor.isServer) {
fetch: ['meta'],
});
Meteor.methods({
moveAttachmentToStorage(fileObjId, storageDestination) {
check(fileObjId, String);
check(storageDestination, String);
const fileObj = Attachments.findOne({_id: fileObjId});
moveToStorage(fileObj, storageDestination, fileStoreStrategyFactory);
},
});
Meteor.startup(() => {
Attachments.collection.createIndex({ cardId: 1 });
});

View file

@ -1,25 +1,24 @@
import { Meteor } from 'meteor/meteor';
import { FilesCollection } from 'meteor/ostrio:files';
import path from 'path';
import { createBucket } from './lib/grid/createBucket';
import { createOnAfterUpload } from './lib/fsHooks/createOnAfterUpload';
import { createInterceptDownload } from './lib/fsHooks/createInterceptDownload';
import { createOnAfterRemove } from './lib/fsHooks/createOnAfterRemove';
import FileStoreStrategyFactory, { FileStoreStrategyFilesystem, FileStoreStrategyGridFs} from '/models/lib/fileStoreStrategy';
let avatarsBucket;
let storagePath;
if (Meteor.isServer) {
avatarsBucket = createBucket('avatars');
storagePath = path.join(process.env.WRITABLE_PATH, 'avatars');
}
const fileStoreStrategyFactory = new FileStoreStrategyFactory(FileStoreStrategyFilesystem, storagePath, FileStoreStrategyGridFs, avatarsBucket);
Avatars = new FilesCollection({
debug: false, // Change to `true` for debugging
collectionName: 'avatars',
allowClientCode: true,
storagePath() {
if (process.env.WRITABLE_PATH) {
return path.join(process.env.WRITABLE_PATH, 'uploads', 'avatars');
}
return path.normalize(`assets/app/uploads/${this.collectionName}`);;
const ret = fileStoreStrategyFactory.storagePath;
return ret;
},
onBeforeUpload(file) {
if (file.size <= 72000 && file.type.startsWith('image/')) {
@ -27,9 +26,24 @@ Avatars = new FilesCollection({
}
return 'avatar-too-big';
},
onAfterUpload: createOnAfterUpload(avatarsBucket),
interceptDownload: createInterceptDownload(avatarsBucket),
onAfterRemove: createOnAfterRemove(avatarsBucket),
onAfterUpload(fileObj) {
// current storage is the filesystem, update object and database
Object.keys(fileObj.versions).forEach(versionName => {
fileObj.versions[versionName].storage = "fs";
});
Avatars.update({ _id: fileObj._id }, { $set: { "versions" : fileObj.versions } });
},
interceptDownload(http, fileObj, versionName) {
const ret = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName).interceptDownload(http, this.cacheControl);
return ret;
},
onAfterRemove(files) {
files.forEach(fileObj => {
Object.keys(fileObj.versions).forEach(versionName => {
fileStoreStrategyFactory.getFileStrategy(fileObj, versionName).onAfterRemove();
});
});
},
});
function isOwner(userId, doc) {

View file

@ -0,0 +1,72 @@
import fs from 'fs';
import FileStoreStrategy, {FileStoreStrategyFilesystem, FileStoreStrategyGridFs} from './fileStoreStrategy'
const insertActivity = (fileObj, activityType) =>
Activities.insert({
userId: fileObj.userId,
type: 'card',
activityType,
attachmentId: fileObj._id,
attachmentName: fileObj.name,
boardId: fileObj.meta.boardId,
cardId: fileObj.meta.cardId,
listId: fileObj.meta.listId,
swimlaneId: fileObj.meta.swimlaneId,
});
/** Strategy to store attachments at GridFS (MongoDB) */
export class AttachmentStoreStrategyGridFs extends FileStoreStrategyGridFs {
/** constructor
* @param gridFsBucket use this GridFS Bucket
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(gridFsBucket, fileObj, versionName) {
super(gridFsBucket, fileObj, versionName);
}
/** after successfull upload */
onAfterUpload() {
super.onAfterUpload();
// If the attachment doesn't have a source field or its source is different than import
if (!this.fileObj.meta.source || this.fileObj.meta.source !== 'import') {
// Add activity about adding the attachment
insertActivity(this.fileObj, 'addAttachment');
}
}
/** after file remove */
onAfterRemove() {
super.onAfterRemove();
insertActivity(this.fileObj, 'deleteAttachment');
}
}
/** Strategy to store attachments at filesystem */
export class AttachmentStoreStrategyFilesystem extends FileStoreStrategyFilesystem {
/** constructor
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(fileObj, versionName) {
super(fileObj, versionName);
}
/** after successfull upload */
onAfterUpload() {
super.onAfterUpload();
// If the attachment doesn't have a source field or its source is different than import
if (!this.fileObj.meta.source || this.fileObj.meta.source !== 'import') {
// Add activity about adding the attachment
insertActivity(this.fileObj, 'addAttachment');
}
}
/** after file remove */
onAfterRemove() {
super.onAfterRemove();
insertActivity(this.fileObj, 'deleteAttachment');
}
}

View file

@ -0,0 +1,338 @@
import fs from 'fs';
import path from 'path';
import { createObjectId } from './grid/createObjectId';
import { httpStreamOutput } from './httpStream.js'
export const STORAGE_NAME_FILESYSTEM = "fs";
export const STORAGE_NAME_GRIDFS = "gridfs";
/** Factory for FileStoreStrategy */
export default class FileStoreStrategyFactory {
/** constructor
* @param classFileStoreStrategyFilesystem use this strategy for filesystem storage
* @param storagePath file storage path
* @param classFileStoreStrategyGridFs use this strategy for GridFS storage
* @param gridFsBucket use this GridFS Bucket as GridFS Storage
*/
constructor(classFileStoreStrategyFilesystem, storagePath, classFileStoreStrategyGridFs, gridFsBucket) {
this.classFileStoreStrategyFilesystem = classFileStoreStrategyFilesystem;
this.storagePath = storagePath;
this.classFileStoreStrategyGridFs = classFileStoreStrategyGridFs;
this.gridFsBucket = gridFsBucket;
}
/** returns the right FileStoreStrategy
* @param fileObj the current file object
* @param versionName the current version
* @param use this storage, or if not set, get the storage from fileObj
*/
getFileStrategy(fileObj, versionName, storage) {
if (!storage) {
storage = fileObj.versions[versionName].storage;
if (!storage) {
if (fileObj.meta.source == "import") {
// uploaded by import, so it's in GridFS (MongoDB)
storage = STORAGE_NAME_GRIDFS;
} else {
// newly uploaded, so it's at the filesystem
storage = STORAGE_NAME_FILESYSTEM;
}
}
}
let ret;
if ([STORAGE_NAME_FILESYSTEM, STORAGE_NAME_GRIDFS].includes(storage)) {
if (storage == STORAGE_NAME_FILESYSTEM) {
ret = new this.classFileStoreStrategyFilesystem(fileObj, versionName);
} else if (storage == STORAGE_NAME_GRIDFS) {
ret = new this.classFileStoreStrategyGridFs(this.gridFsBucket, fileObj, versionName);
}
}
return ret;
}
}
/** Strategy to store files */
class FileStoreStrategy {
/** constructor
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(fileObj, versionName) {
this.fileObj = fileObj;
this.versionName = versionName;
}
/** after successfull upload */
onAfterUpload() {
}
/** download the file
* @param http the current http request
* @param cacheControl cacheControl of FilesCollection
*/
interceptDownload(http, cacheControl) {
}
/** after file remove */
onAfterRemove() {
}
/** returns a read stream
* @return the read stream
*/
getReadStream() {
}
/** returns a write stream
* @param filePath if set, use this path
* @return the write stream
*/
getWriteStream(filePath) {
}
/** writing finished
* @param finishedData the data of the write stream finish event
*/
writeStreamFinished(finishedData) {
}
/** returns the new file path
* @param storagePath use this storage path
* @return the new file path
*/
getNewPath(storagePath, name) {
if (!_.isString(name)) {
name = this.fileObj.name;
}
const ret = path.join(storagePath, this.fileObj._id + "-" + this.versionName + "-" + name);
return ret;
}
/** remove the file */
unlink() {
}
/** return the storage name
* @return the storage name
*/
getStorageName() {
}
}
/** Strategy to store attachments at GridFS (MongoDB) */
export class FileStoreStrategyGridFs extends FileStoreStrategy {
/** constructor
* @param gridFsBucket use this GridFS Bucket
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(gridFsBucket, fileObj, versionName) {
super(fileObj, versionName);
this.gridFsBucket = gridFsBucket;
}
/** download the file
* @param http the current http request
* @param cacheControl cacheControl of FilesCollection
*/
interceptDownload(http, cacheControl) {
const readStream = this.getReadStream();
const downloadFlag = http?.params?.query?.download;
let ret = false;
if (readStream) {
ret = true;
httpStreamOutput(readStream, this.fileObj.name, http, downloadFlag, cacheControl);
}
return ret;
}
/** after file remove */
onAfterRemove() {
this.unlink();
super.onAfterRemove();
}
/** returns a read stream
* @return the read stream
*/
getReadStream() {
const gfsId = this.getGridFsObjectId();
let ret;
if (gfsId) {
ret = this.gridFsBucket.openDownloadStream(gfsId);
}
return ret;
}
/** returns a write stream
* @param filePath if set, use this path
* @return the write stream
*/
getWriteStream(filePath) {
const fileObj = this.fileObj;
const versionName = this.versionName;
const metadata = { ...fileObj.meta, versionName, fileId: fileObj._id };
const ret = this.gridFsBucket.openUploadStream(this.fileObj.name, {
contentType: fileObj.type || 'binary/octet-stream',
metadata,
});
return ret;
}
/** writing finished
* @param finishedData the data of the write stream finish event
*/
writeStreamFinished(finishedData) {
const gridFsFileIdName = this.getGridFsFileIdName();
Attachments.update({ _id: this.fileObj._id }, { $set: { [gridFsFileIdName]: finishedData._id.toHexString(), } });
}
/** remove the file */
unlink() {
const gfsId = this.getGridFsObjectId();
if (gfsId) {
this.gridFsBucket.delete(gfsId, err => {
if (err) {
console.error("error on gfs bucket.delete: ", err);
}
});
}
const gridFsFileIdName = this.getGridFsFileIdName();
Attachments.update({ _id: this.fileObj._id }, { $unset: { [gridFsFileIdName]: 1 } });
}
/** return the storage name
* @return the storage name
*/
getStorageName() {
return STORAGE_NAME_GRIDFS;
}
/** returns the GridFS Object-Id
* @return the GridFS Object-Id
*/
getGridFsObjectId() {
let ret;
const gridFsFileId = this.getGridFsFileId();
if (gridFsFileId) {
ret = createObjectId({ gridFsFileId });
}
return ret;
}
/** returns the GridFS Object-Id
* @return the GridFS Object-Id
*/
getGridFsFileId() {
const ret = (this.fileObj.versions[this.versionName].meta || {})
.gridFsFileId;
return ret;
}
/** returns the property name of gridFsFileId
* @return the property name of gridFsFileId
*/
getGridFsFileIdName() {
const ret = `versions.${this.versionName}.meta.gridFsFileId`;
return ret;
}
}
/** Strategy to store attachments at filesystem */
export class FileStoreStrategyFilesystem extends FileStoreStrategy {
/** constructor
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(fileObj, versionName) {
super(fileObj, versionName);
}
/** returns a read stream
* @return the read stream
*/
getReadStream() {
const ret = fs.createReadStream(this.fileObj.versions[this.versionName].path)
return ret;
}
/** returns a write stream
* @param filePath if set, use this path
* @return the write stream
*/
getWriteStream(filePath) {
if (!_.isString(filePath)) {
filePath = this.fileObj.versions[this.versionName].path;
}
const ret = fs.createWriteStream(filePath);
return ret;
}
/** writing finished
* @param finishedData the data of the write stream finish event
*/
writeStreamFinished(finishedData) {
}
/** remove the file */
unlink() {
const filePath = this.fileObj.versions[this.versionName].path;
fs.unlink(filePath, () => {});
}
/** return the storage name
* @return the storage name
*/
getStorageName() {
return STORAGE_NAME_FILESYSTEM;
}
}
/** move the fileObj to another storage
* @param fileObj move this fileObj to another storage
* @param storageDestination the storage destination (fs or gridfs)
* @param fileStoreStrategyFactory get FileStoreStrategy from this factory
*/
export const moveToStorage = function(fileObj, storageDestination, fileStoreStrategyFactory) {
Object.keys(fileObj.versions).forEach(versionName => {
const strategyRead = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName);
const strategyWrite = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName, storageDestination);
if (strategyRead.constructor.name != strategyWrite.constructor.name) {
const readStream = strategyRead.getReadStream();
const filePath = strategyWrite.getNewPath(fileStoreStrategyFactory.storagePath);
const writeStream = strategyWrite.getWriteStream(filePath);
writeStream.on('error', error => {
console.error('[writeStream error]: ', error, fileObjId);
});
readStream.on('error', error => {
console.error('[readStream error]: ', error, fileObjId);
});
writeStream.on('finish', Meteor.bindEnvironment((finishedData) => {
strategyWrite.writeStreamFinished(finishedData);
}));
// https://forums.meteor.com/t/meteor-code-must-always-run-within-a-fiber-try-wrapping-callbacks-that-you-pass-to-non-meteor-libraries-with-meteor-bindenvironmen/40099/8
readStream.on('end', Meteor.bindEnvironment(() => {
Attachments.update({ _id: fileObj._id }, { $set: {
[`versions.${versionName}.storage`]: strategyWrite.getStorageName(),
[`versions.${versionName}.path`]: filePath,
} });
strategyRead.unlink();
}));
readStream.pipe(writeStream);
}
});
};

View file

@ -1,47 +0,0 @@
import { createObjectId } from '../grid/createObjectId';
export const createInterceptDownload = bucket =>
function interceptDownload(http, file, versionName) {
const { gridFsFileId } = file.versions[versionName].meta || {};
if (gridFsFileId) {
// opens the download stream using a given gfs id
// see: http://mongodb.github.io/node-mongodb-native/3.2/api/GridFSBucket.html#openDownloadStream
const gfsId = createObjectId({ gridFsFileId });
const readStream = bucket.openDownloadStream(gfsId);
readStream.on('data', data => {
http.response.write(data);
});
readStream.on('end', () => {
http.response.end(); // don't pass parameters to end() or it will be attached to the file's binary stream
});
readStream.on('error', () => {
// not found probably
// eslint-disable-next-line no-param-reassign
http.response.statusCode = 404;
http.response.end('not found');
});
http.response.setHeader('Cache-Control', this.cacheControl);
http.response.setHeader(
'Content-Disposition',
getContentDisposition(file.name, http?.params?.query?.download),
);
}
return Boolean(gridFsFileId); // Serve file from either GridFS or FS if it wasn't uploaded yet
};
/**
* Will initiate download, if links are called with ?download="true" queryparam.
**/
const getContentDisposition = (name, downloadFlag) => {
const dispositionType = downloadFlag === 'true' ? 'attachment;' : 'inline;';
const encodedName = encodeURIComponent(name);
const dispositionName = `filename="${encodedName}"; filename=*UTF-8"${encodedName}";`;
const dispositionEncoding = 'charset=utf-8';
return `${dispositionType} ${dispositionName} ${dispositionEncoding}`;
};

View file

@ -1,17 +0,0 @@
import { createObjectId } from '../grid/createObjectId';
export const createOnAfterRemove = bucket =>
function onAfterRemove(files) {
files.forEach(file => {
Object.keys(file.versions).forEach(versionName => {
const gridFsFileId = (file.versions[versionName].meta || {})
.gridFsFileId;
if (gridFsFileId) {
const gfsId = createObjectId({ gridFsFileId });
bucket.delete(gfsId, err => {
// if (err) console.error(err);
});
}
});
});
};

31
models/lib/httpStream.js Normal file
View file

@ -0,0 +1,31 @@
export const httpStreamOutput = function(readStream, name, http, downloadFlag, cacheControl) {
readStream.on('data', data => {
http.response.write(data);
});
readStream.on('end', () => {
// don't pass parameters to end() or it will be attached to the file's binary stream
http.response.end();
});
readStream.on('error', () => {
http.response.statusCode = 404;
http.response.end('not found');
});
if (cacheControl) {
http.response.setHeader('Cache-Control', cacheControl);
}
http.response.setHeader('Content-Disposition', getContentDisposition(name, http?.params?.query?.download));
};
/** will initiate download, if links are called with ?download="true" queryparam */
const getContentDisposition = (name, downloadFlag) => {
const dispositionType = downloadFlag === 'true' ? 'attachment;' : 'inline;';
const encodedName = encodeURIComponent(name);
const dispositionName = `filename="${encodedName}"; filename=*UTF-8"${encodedName}";`;
const dispositionEncoding = 'charset=utf-8';
return `${dispositionType} ${dispositionName} ${dispositionEncoding}`;
};