Store files to S3. In Progress.

Thanks to xet7 !

Related #142

models/lib/attachmentStoreStrategy.js

@@ -1,5 +1,5 @@
 import fs from 'fs';
-import FileStoreStrategy, {FileStoreStrategyFilesystem, FileStoreStrategyGridFs} from './fileStoreStrategy'
+import FileStoreStrategy, {FileStoreStrategyFilesystem, FileStoreStrategyGridFs, FileStoreStrategyS3} from './fileStoreStrategy'
 
 const insertActivity = (fileObj, activityType) =>
   Activities.insert({
@@ -70,3 +70,32 @@ export class AttachmentStoreStrategyFilesystem extends FileStoreStrategyFilesyst
     insertActivity(this.fileObj, 'deleteAttachment');
   }
 }
+
+/** Strategy to store attachments at filesystem */
+export class AttachmentStoreStrategyS3 extends FileStoreStrategyS3 {
+
+  /** constructor
+   * @param s3Bucket use this S3 Bucket
+   * @param fileObj the current file object
+   * @param versionName the current version
+   */
+  constructor(s3Bucket, fileObj, versionName) {
+    super(s3Bucket, fileObj, versionName);
+  }
+
+  /** after successfull upload */
+  onAfterUpload() {
+    super.onAfterUpload();
+    // If the attachment doesn't have a source field or its source is different than import
+    if (!this.fileObj.meta.source || this.fileObj.meta.source !== 'import') {
+      // Add activity about adding the attachment
+      insertActivity(this.fileObj, 'addAttachment');
+    }
+  }
+
+  /** after file remove */
+  onAfterRemove() {
+    super.onAfterRemove();
+    insertActivity(this.fileObj, 'deleteAttachment');
+  }
+}

models/lib/fileStoreStrategy.js

@@ -6,6 +6,7 @@ import { ObjectID } from 'bson';
 
 export const STORAGE_NAME_FILESYSTEM = "fs";
 export const STORAGE_NAME_GRIDFS     = "gridfs";
+export const STORAGE_NAME_S3         = "s3";
 
 /** Factory for FileStoreStrategy */
 export default class FileStoreStrategyFactory {
@@ -15,12 +16,16 @@ export default class FileStoreStrategyFactory {
    * @param storagePath file storage path
    * @param classFileStoreStrategyGridFs use this strategy for GridFS storage
    * @param gridFsBucket use this GridFS Bucket as GridFS Storage
+   * @param classFileStoreStartegyS3 use this strategy for S3 storage
+   * @param s3Bucket use this S3 Bucket as S3 Storage
    */
   constructor(classFileStoreStrategyFilesystem, storagePath, classFileStoreStrategyGridFs, gridFsBucket) {
     this.classFileStoreStrategyFilesystem = classFileStoreStrategyFilesystem;
     this.storagePath = storagePath;
     this.classFileStoreStrategyGridFs = classFileStoreStrategyGridFs;
     this.gridFsBucket = gridFsBucket;
+    this.classFileStoreStrategyS3 = classFileStoreStrategyS3;
+    this.s3bucket = s3Bucket;
   }
 
   /** returns the right FileStoreStrategy
@@ -35,6 +40,8 @@ export default class FileStoreStrategyFactory {
         if (fileObj.meta.source == "import" || fileObj.versions[versionName].meta.gridFsFileId) {
           // uploaded by import, so it's in GridFS (MongoDB)
           storage = STORAGE_NAME_GRIDFS;
+        } else if (fileRef && fileRef.versions && fileRef.versions[version] && fileRef.versions[version].meta && fileRef.versions[version].meta.pipePath) {
+          storage = STORAGE_NAME_S3;
         } else {
           // newly uploaded, so it's at the filesystem
           storage = STORAGE_NAME_FILESYSTEM;
@@ -42,9 +49,11 @@ export default class FileStoreStrategyFactory {
       }
     }
     let ret;
-    if ([STORAGE_NAME_FILESYSTEM, STORAGE_NAME_GRIDFS].includes(storage)) {
+    if ([STORAGE_NAME_FILESYSTEM, STORAGE_NAME_GRIDFS, STORAGE_NAME_S3].includes(storage)) {
       if (storage == STORAGE_NAME_FILESYSTEM) {
         ret = new this.classFileStoreStrategyFilesystem(fileObj, versionName);
+      } else if (storage == STORAGE_NAME_S3) {
+        ret = new this.classFileStoreStrategyS3(this.s3Bucket, fileObj, versionName);
       } else if (storage == STORAGE_NAME_GRIDFS) {
         ret = new this.classFileStoreStrategyGridFs(this.gridFsBucket, fileObj, versionName);
       }

models/lib/s3/createOnAfterUpload.js

@@ -0,0 +1,187 @@
+import { Meteor } from 'meteor/meteor';
+import { _ } from 'meteor/underscore';
+import { Random } from 'meteor/random';
+import { FilesCollection } from 'meteor/ostrio:files';
+import stream from 'stream';
+
+import S3 from 'aws-sdk/clients/s3'; /* http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html */
+/* See fs-extra and graceful-fs NPM packages */
+/* For better i/o performance */
+import fs from 'fs';
+
+/* Example: S3='{"s3":{"key": "xxx", "secret": "xxx", "bucket": "xxx", "region": "xxx""}}' meteor */
+if (process.env.S3) {
+  Meteor.settings.s3 = JSON.parse(process.env.S3).s3;
+
+const s3Conf = Meteor.settings.s3 || {};
+const bound  = Meteor.bindEnvironment((callback) => {
+  return callback();
+});
+
+/* Check settings existence in `Meteor.settings` */
+/* This is the best practice for app security */
+if (s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && s3Conf.region) {
+  // Create a new S3 object
+  const s3 = new S3({
+    secretAccessKey: s3Conf.secret,
+    accessKeyId: s3Conf.key,
+    region: s3Conf.region,
+    // sslEnabled: true, // optional
+    httpOptions: {
+      timeout: 6000,
+      agent: false
+    }
+  });
+
+  // Declare the Meteor file collection on the Server
+  const UserFiles = new FilesCollection({
+    debug: false, // Change to `true` for debugging
+    storagePath: 'assets/app/uploads/uploadedFiles',
+    collectionName: 'userFiles',
+    // Disallow Client to execute remove, use the Meteor.method
+    allowClientCode: false,
+
+    // Start moving files to AWS:S3
+    // after fully received by the Meteor server
+    onAfterUpload(fileRef) {
+      // Run through each of the uploaded file
+      _.each(fileRef.versions, (vRef, version) => {
+        // We use Random.id() instead of real file's _id
+        // to secure files from reverse engineering on the AWS client
+        const filePath = 'files/' + (Random.id()) + '-' + version + '.' + fileRef.extension;
+
+        // Create the AWS:S3 object.
+        // Feel free to change the storage class from, see the documentation,
+        // `STANDARD_IA` is the best deal for low access files.
+        // Key is the file name we are creating on AWS:S3, so it will be like files/XXXXXXXXXXXXXXXXX-original.XXXX
+        // Body is the file stream we are sending to AWS
+        s3.putObject({
+          // ServerSideEncryption: 'AES256', // Optional
+          StorageClass: 'STANDARD',
+          Bucket: s3Conf.bucket,
+          Key: filePath,
+          Body: fs.createReadStream(vRef.path),
+          ContentType: vRef.type,
+        }, (error) => {
+          bound(() => {
+            if (error) {
+              console.error(error);
+            } else {
+              // Update FilesCollection with link to the file at AWS
+              const upd = { $set: {} };
+              upd['$set']['versions.' + version + '.meta.pipePath'] = filePath;
+
+              this.collection.update({
+                _id: fileRef._id
+              }, upd, (updError) => {
+                if (updError) {
+                  console.error(updError);
+                } else {
+                  // Unlink original files from FS after successful upload to AWS:S3
+                  this.unlink(this.collection.findOne(fileRef._id), version);
+                }
+              });
+            }
+          });
+        });
+      });
+    },
+
+
+    // Intercept access to the file
+    // And redirect request to AWS:S3
+    interceptDownload(http, fileRef, version) {
+      let path;
+
+      if (fileRef && fileRef.versions && fileRef.versions[version] && fileRef.versions[version].meta && fileRef.versions[version].meta.pipePath) {
+        path = fileRef.versions[version].meta.pipePath;
+      }
+
+      if (path) {
+        // If file is successfully moved to AWS:S3
+        // We will pipe request to AWS:S3
+        // So, original link will stay always secure
+
+        // To force ?play and ?download parameters
+        // and to keep original file name, content-type,
+        // content-disposition, chunked "streaming" and cache-control
+        // we're using low-level .serve() method
+        const opts = {
+          Bucket: s3Conf.bucket,
+          Key: path
+        };
+
+        if (http.request.headers.range) {
+          const vRef  = fileRef.versions[version];
+          let range   = _.clone(http.request.headers.range);
+          const array = range.split(/bytes=([0-9]*)-([0-9]*)/);
+          const start = parseInt(array[1]);
+          let end     = parseInt(array[2]);
+          if (isNaN(end)) {
+            // Request data from AWS:S3 by small chunks
+            end       = (start + this.chunkSize) - 1;
+            if (end >= vRef.size) {
+              end     = vRef.size - 1;
+            }
+          }
+          opts.Range   = `bytes=${start}-${end}`;
+          http.request.headers.range = `bytes=${start}-${end}`;
+        }
+
+        const fileColl = this;
+        s3.getObject(opts, function (error) {
+          if (error) {
+            console.error(error);
+            if (!http.response.finished) {
+              http.response.end();
+            }
+          } else {
+            if (http.request.headers.range && this.httpResponse.headers['content-range']) {
+              // Set proper range header in according to what is returned from AWS:S3
+              http.request.headers.range = this.httpResponse.headers['content-range'].split('/')[0].replace('bytes ', 'bytes=');
+            }
+
+            const dataStream = new stream.PassThrough();
+            fileColl.serve(http, fileRef, fileRef.versions[version], version, dataStream);
+            dataStream.end(this.data.Body);
+          }
+        });
+
+        return true;
+      }
+      // While file is not yet uploaded to AWS:S3
+      // It will be served file from FS
+      return false;
+    }
+  });
+
+  // Intercept FilesCollection's remove method to remove file from AWS:S3
+  const _origRemove = UserFiles.remove;
+  UserFiles.remove = function (selector, callback) {
+    const cursor = this.collection.find(selector);
+    cursor.forEach((fileRef) => {
+      _.each(fileRef.versions, (vRef) => {
+        if (vRef && vRef.meta && vRef.meta.pipePath) {
+          // Remove the object from AWS:S3 first, then we will call the original FilesCollection remove
+          s3.deleteObject({
+            Bucket: s3Conf.bucket,
+            Key: vRef.meta.pipePath,
+          }, (error) => {
+            bound(() => {
+              if (error) {
+                console.error(error);
+              }
+            });
+          });
+        }
+      });
+    });
+
+    // Remove original file from database
+    _origRemove.call(this, selector, callback);
+  };
+} else {
+  throw new Meteor.Error(401, 'Missing Meteor file settings');
+}
+
+}