Move In Progress ostrio-files changes to separate branch, and revert ostrio-files changes, so that:

- Export to CSV/TSV with custom fields works
- Attachments are not exported to disk
- It is possible to build arm64/s390x versions again.

Thanks to xet7 !

Related #3110
This commit is contained in:
Lauri Ojansivu 2020-05-25 17:54:51 +03:00
parent 23dcd084a4
commit d52affe658
22 changed files with 385 additions and 757 deletions

View file

@ -217,7 +217,7 @@ if (Meteor.isServer) {
}
if (activity.attachmentId) {
const attachment = activity.attachment();
params.attachment = attachment.name;
params.attachment = attachment.original.name;
params.attachmentId = attachment._id;
}
if (activity.checklistId) {

View file

@ -1,127 +1,263 @@
import { FilesCollection } from 'meteor/ostrio:files';
const fs = require('fs');
const localFSStore = process.env.ATTACHMENTS_STORE_PATH;
const storeName = 'attachments';
const defaultStoreOptions = {
beforeWrite: fileObj => {
if (!fileObj.isImage()) {
return {
type: 'application/octet-stream',
};
}
return {};
},
};
let store;
if (localFSStore) {
// have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem
const fs = Npm.require('fs');
const path = Npm.require('path');
const mongodb = Npm.require('mongodb');
const Grid = Npm.require('gridfs-stream');
// calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :(
let pathname = localFSStore;
/*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */
const collectionName = 'attachments2';
if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) {
pathname = path.join(
__meteor_bootstrap__.serverDir,
`../../../cfs/files/${storeName}`,
);
}
Attachments = new FilesCollection({
storagePath: storagePath(),
debug: false,
// allowClientCode: true,
collectionName: 'attachments2',
onAfterUpload: onAttachmentUploaded,
onBeforeRemove: onAttachmentRemoving
if (!pathname)
throw new Error('FS.Store.FileSystem unable to determine path');
// Check if we have '~/foo/bar'
if (pathname.split(path.sep)[0] === '~') {
const homepath =
process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
if (homepath) {
pathname = pathname.replace('~', homepath);
} else {
throw new Error('FS.Store.FileSystem unable to resolve "~" in path');
}
}
// Set absolute path
const absolutePath = path.resolve(pathname);
const _FStore = new FS.Store.FileSystem(storeName, {
path: localFSStore,
...defaultStoreOptions,
});
const GStore = {
fileKey(fileObj) {
const key = {
_id: null,
filename: null,
};
// If we're passed a fileObj, we retrieve the _id and filename from it.
if (fileObj) {
const info = fileObj._getInfo(storeName, {
updateFileRecordFirst: false,
});
key._id = info.key || null;
key.filename =
info.name ||
fileObj.name({ updateFileRecordFirst: false }) ||
`${fileObj.collectionName}-${fileObj._id}`;
}
// If key._id is null at this point, createWriteStream will let GridFS generate a new ID
return key;
},
db: undefined,
mongoOptions: { useNewUrlParser: true },
mongoUrl: process.env.MONGO_URL,
init() {
this._init(err => {
this.inited = !err;
});
},
_init(callback) {
const self = this;
mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function(
err,
db,
) {
if (err) {
return callback(err);
}
self.db = db;
return callback(null);
});
return;
},
createReadStream(fileKey, options) {
const self = this;
if (!self.inited) {
self.init();
return undefined;
}
options = options || {};
// Init GridFS
const gfs = new Grid(self.db, mongodb);
// Set the default streamning settings
const settings = {
_id: new mongodb.ObjectID(fileKey._id),
root: `cfs_gridfs.${storeName}`,
};
// Check if this should be a partial read
if (
typeof options.start !== 'undefined' &&
typeof options.end !== 'undefined'
) {
// Add partial info
settings.range = {
startPos: options.start,
endPos: options.end,
};
}
return gfs.createReadStream(settings);
},
};
GStore.init();
const CRS = 'createReadStream';
const _CRS = `_${CRS}`;
const FStore = _FStore._transform;
FStore[_CRS] = FStore[CRS].bind(FStore);
FStore[CRS] = function(fileObj, options) {
let stream;
try {
const localFile = path.join(
absolutePath,
FStore.storage.fileKey(fileObj),
);
const state = fs.statSync(localFile);
if (state) {
stream = FStore[_CRS](fileObj, options);
}
} catch (e) {
// file is not there, try GridFS ?
stream = undefined;
}
if (stream) return stream;
else {
try {
const stream = GStore[CRS](GStore.fileKey(fileObj), options);
return stream;
} catch (e) {
return undefined;
}
}
}.bind(FStore);
store = _FStore;
} else {
store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, {
// XXX Add a new store for cover thumbnails so we don't load big images in
// the general board view
// If the uploaded document is not an image we need to enforce browser
// download instead of execution. This is particularly important for HTML
// files that the browser will just execute if we don't serve them with the
// appropriate `application/octet-stream` MIME header which can lead to user
// data leaks. I imagine other formats (like PDF) can also be attack vectors.
// See https://github.com/wekan/wekan/issues/99
// XXX Should we use `beforeWrite` option of CollectionFS instead of
// collection-hooks?
// We should use `beforeWrite`.
...defaultStoreOptions,
});
}
Attachments = new FS.Collection('attachments', {
stores: [store],
});
if (Meteor.isServer) {
Meteor.startup(() => {
Attachments.collection._ensureIndex({ cardId: 1 });
Attachments.files._ensureIndex({ cardId: 1 });
});
// TODO: Permission related
Attachments.allow({
insert() {
return false;
insert(userId, doc) {
return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
},
update() {
return true;
update(userId, doc) {
return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
},
remove() {
return true;
}
});
Meteor.methods({
cloneAttachment(file, overrides) {
check(file, Object);
check(overrides, Match.Maybe(Object));
const path = file.path;
const opts = {
fileName: file.name,
type: file.type,
meta: file.meta,
userId: file.userId
};
for (let key in overrides) {
if (key === 'meta') {
for (let metaKey in overrides.meta) {
opts.meta[metaKey] = overrides.meta[metaKey];
}
} else {
opts[key] = overrides[key];
}
remove(userId, doc) {
return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
},
// We authorize the attachment download either:
// - if the board is public, everyone (even unconnected) can download it
// - if the board is private, only board members can download it
download(userId, doc) {
const board = Boards.findOne(doc.boardId);
if (board.isPublic()) {
return true;
} else {
return board.hasMember(userId);
}
const buffer = fs.readFileSync(path);
Attachments.write(buffer, opts, (err, fileRef) => {
if (err) {
console.log('Error when cloning record', err);
}
},
fetch: ['boardId'],
});
}
// XXX Enforce a schema for the Attachments CollectionFS
if (Meteor.isServer) {
Attachments.files.after.insert((userId, doc) => {
// If the attachment doesn't have a source field
// or its source is different than import
if (!doc.source || doc.source !== 'import') {
// Add activity about adding the attachment
Activities.insert({
userId,
type: 'card',
activityType: 'addAttachment',
attachmentId: doc._id,
// this preserves the name so that notifications can be meaningful after
// this file is removed
attachmentName: doc.original.name,
boardId: doc.boardId,
cardId: doc.cardId,
listId: doc.listId,
swimlaneId: doc.swimlaneId,
});
return true;
} else {
// Don't add activity about adding the attachment as the activity
// be imported and delete source field
Attachments.update(
{
_id: doc._id,
},
{
$unset: {
source: '',
},
},
);
}
});
Meteor.publish(collectionName, function() {
return Attachments.find().cursor;
});
} else {
Meteor.subscribe(collectionName);
}
function storagePath(defaultPath) {
const storePath = process.env.ATTACHMENTS_STORE_PATH;
return storePath ? storePath : defaultPath;
}
function onAttachmentUploaded(fileRef) {
Attachments.update({_id:fileRef._id}, {$set: {"meta.uploading": false}});
if (!fileRef.meta.source || fileRef.meta.source !== 'import') {
// Add activity about adding the attachment
Attachments.files.before.remove((userId, doc) => {
Activities.insert({
userId: fileRef.userId,
userId,
type: 'card',
activityType: 'addAttachment',
attachmentId: fileRef._id,
activityType: 'deleteAttachment',
attachmentId: doc._id,
// this preserves the name so that notifications can be meaningful after
// this file is removed
attachmentName: fileRef.name,
boardId: fileRef.meta.boardId,
cardId: fileRef.meta.cardId,
listId: fileRef.meta.listId,
swimlaneId: fileRef.meta.swimlaneId,
// this file is removed
attachmentName: doc.original.name,
boardId: doc.boardId,
cardId: doc.cardId,
listId: doc.listId,
swimlaneId: doc.swimlaneId,
});
} else {
// Don't add activity about adding the attachment as the activity
// be imported and delete source field
Attachments.collection.update(
{
_id: fileRef._id,
},
{
$unset: {
'meta.source': '',
},
},
);
}
}
function onAttachmentRemoving(cursor) {
const file = cursor.get()[0];
const meta = file.meta;
Activities.insert({
userId: this.userId,
type: 'card',
activityType: 'deleteAttachment',
attachmentId: file._id,
// this preserves the name so that notifications can be meaningful after
// this file is removed
attachmentName: file.name,
boardId: meta.boardId,
cardId: meta.cardId,
listId: meta.listId,
swimlaneId: meta.swimlaneId,
});
return true;
}
export default Attachments;

View file

@ -412,14 +412,10 @@ Cards.helpers({
const _id = Cards.insert(this);
// Copy attachments
oldCard.attachments().forEach((file) => {
Meteor.call('cloneAttachment', file,
{
meta: {
cardId: _id
}
}
);
oldCard.attachments().forEach(att => {
att.cardId = _id;
delete att._id;
return Attachments.insert(att);
});
// copy checklists
@ -522,15 +518,14 @@ Cards.helpers({
attachments() {
if (this.isLinkedCard()) {
return Attachments.find(
{ 'meta.cardId': this.linkedId },
{ cardId: this.linkedId },
{ sort: { uploadedAt: -1 } },
);
} else {
let ret = Attachments.find(
{ 'meta.cardId': this._id },
return Attachments.find(
{ cardId: this._id },
{ sort: { uploadedAt: -1 } },
);
return ret;
}
},
@ -539,7 +534,7 @@ Cards.helpers({
const cover = Attachments.findOne(this.coverId);
// if we return a cover before it is fully stored, we will get errors when we try to display it
// todo XXX we could return a default "upload pending" image in the meantime?
return cover && cover.link();
return cover && cover.url() && cover;
},
checklists() {

View file

@ -1,3 +1,4 @@
import { Exporter } from './exporter';
/* global JsonRoutes */
if (Meteor.isServer) {
// todo XXX once we have a real API in place, move that route there
@ -7,10 +8,10 @@ if (Meteor.isServer) {
// on the client instead of copy/pasting the route path manually between the
// client and the server.
/**
* @operation export
* @operation exportJson
* @tag Boards
*
* @summary This route is used to export the board.
* @summary This route is used to export the board to a json file format.
*
* @description If user is already logged-in, pass loginToken as param
* "authToken": '/api/boards/:boardId/export?authToken=:token'
@ -46,199 +47,52 @@ if (Meteor.isServer) {
JsonRoutes.sendResult(res, 403);
}
});
}
// exporter maybe is broken since Gridfs introduced, add fs and path
export class Exporter {
constructor(boardId) {
this._boardId = boardId;
}
build() {
const fs = Npm.require('fs');
const os = Npm.require('os');
const path = Npm.require('path');
const byBoard = { boardId: this._boardId };
const byBoardNoLinked = {
boardId: this._boardId,
linkedId: { $in: ['', null] },
};
// we do not want to retrieve boardId in related elements
const noBoardId = {
fields: {
boardId: 0,
},
};
const result = {
_format: 'wekan-board-1.0.0',
};
_.extend(
result,
Boards.findOne(this._boardId, {
fields: {
stars: 0,
},
}),
);
result.lists = Lists.find(byBoard, noBoardId).fetch();
result.cards = Cards.find(byBoardNoLinked, noBoardId).fetch();
result.swimlanes = Swimlanes.find(byBoard, noBoardId).fetch();
result.customFields = CustomFields.find(
{ boardIds: { $in: [this.boardId] } },
{ fields: { boardId: 0 } },
).fetch();
result.comments = CardComments.find(byBoard, noBoardId).fetch();
result.activities = Activities.find(byBoard, noBoardId).fetch();
result.rules = Rules.find(byBoard, noBoardId).fetch();
result.checklists = [];
result.checklistItems = [];
result.subtaskItems = [];
result.triggers = [];
result.actions = [];
result.cards.forEach(card => {
result.checklists.push(
...Checklists.find({
cardId: card._id,
}).fetch(),
);
result.checklistItems.push(
...ChecklistItems.find({
cardId: card._id,
}).fetch(),
);
result.subtaskItems.push(
...Cards.find({
parentId: card._id,
}).fetch(),
);
});
result.rules.forEach(rule => {
result.triggers.push(
...Triggers.find(
{
_id: rule.triggerId,
},
noBoardId,
).fetch(),
);
result.actions.push(
...Actions.find(
{
_id: rule.actionId,
},
noBoardId,
).fetch(),
);
});
// [Old] for attachments we only export IDs and absolute url to original doc
// [New] Encode attachment to base64
const getBase64Data = function(doc, callback) {
let buffer = Buffer.allocUnsafe(0);
buffer.fill(0);
// callback has the form function (err, res) {}
const tmpFile = path.join(
os.tmpdir(),
`tmpexport${process.pid}${Math.random()}`,
);
const tmpWriteable = fs.createWriteStream(tmpFile);
const readStream = fs.createReadStream(doc.path);
readStream.on('data', function(chunk) {
buffer = Buffer.concat([buffer, chunk]);
});
readStream.on('error', function(err) {
callback(null, null);
});
readStream.on('end', function() {
// done
fs.unlink(tmpFile, () => {
//ignored
});
callback(null, buffer.toString('base64'));
});
readStream.pipe(tmpWriteable);
};
const getBase64DataSync = Meteor.wrapAsync(getBase64Data);
result.attachments = Attachments.find({ 'meta.boardId': byBoard.boardId })
.fetch()
.map(attachment => {
let filebase64 = null;
filebase64 = getBase64DataSync(attachment);
return {
_id: attachment._id,
cardId: attachment.meta.cardId,
//url: FlowRouter.url(attachment.url()),
file: filebase64,
name: attachment.name,
type: attachment.type,
};
});
// we also have to export some user data - as the other elements only
// include id but we have to be careful:
// 1- only exports users that are linked somehow to that board
// 2- do not export any sensitive information
const users = {};
result.members.forEach(member => {
users[member.userId] = true;
});
result.lists.forEach(list => {
users[list.userId] = true;
});
result.cards.forEach(card => {
users[card.userId] = true;
if (card.members) {
card.members.forEach(memberId => {
users[memberId] = true;
});
}
});
result.comments.forEach(comment => {
users[comment.userId] = true;
});
result.activities.forEach(activity => {
users[activity.userId] = true;
});
result.checklists.forEach(checklist => {
users[checklist.userId] = true;
});
const byUserIds = {
_id: {
$in: Object.getOwnPropertyNames(users),
},
};
// we use whitelist to be sure we do not expose inadvertently
// some secret fields that gets added to User later.
const userFields = {
fields: {
_id: 1,
username: 1,
'profile.fullname': 1,
'profile.initials': 1,
'profile.avatarUrl': 1,
},
};
result.users = Users.find(byUserIds, userFields)
.fetch()
.map(user => {
// user avatar is stored as a relative url, we export absolute
if ((user.profile || {}).avatarUrl) {
user.profile.avatarUrl = FlowRouter.url(user.profile.avatarUrl);
}
return user;
});
return result;
}
canExport(user) {
const board = Boards.findOne(this._boardId);
return board && board.isVisibleBy(user);
}
/**
* @operation exportCSV/TSV
* @tag Boards
*
* @summary This route is used to export the board to a CSV or TSV file format.
*
* @description If user is already logged-in, pass loginToken as param
*
* See https://blog.kayla.com.au/server-side-route-authentication-in-meteor/
* for detailed explanations
*
* @param {string} boardId the ID of the board we are exporting
* @param {string} authToken the loginToken
* @param {string} delimiter delimiter to use while building export. Default is comma ','
*/
Picker.route('/api/boards/:boardId/export/csv', function(params, req, res) {
const boardId = params.boardId;
let user = null;
const loginToken = params.query.authToken;
if (loginToken) {
const hashToken = Accounts._hashLoginToken(loginToken);
user = Meteor.users.findOne({
'services.resume.loginTokens.hashedToken': hashToken,
});
} else if (!Meteor.settings.public.sandstorm) {
Authentication.checkUserId(req.userId);
user = Users.findOne({
_id: req.userId,
isAdmin: true,
});
}
const exporter = new Exporter(boardId);
if (exporter.canExport(user)) {
body = params.query.delimiter
? exporter.buildCsv(params.query.delimiter)
: exporter.buildCsv();
res.writeHead(200, {
'Content-Length': body[0].length,
'Content-Type': params.query.delimiter ? 'text/csv' : 'text/tsv',
});
res.write(body[0]);
res.end();
} else {
res.writeHead(403);
res.end('Permission Error');
}
});
}

View file

@ -369,7 +369,6 @@ export class TrelloCreator {
// so we make it server only, and let UI catch up once it is done, forget about latency comp.
const self = this;
if (Meteor.isServer) {
// FIXME: Change to new model
file.attachData(att.url, function(error) {
file.boardId = boardId;
file.cardId = cardId;

View file

@ -415,7 +415,6 @@ export class WekanCreator {
const self = this;
if (Meteor.isServer) {
if (att.url) {
// FIXME: Change to new file library
file.attachData(att.url, function(error) {
file.boardId = boardId;
file.cardId = cardId;
@ -441,7 +440,6 @@ export class WekanCreator {
}
});
} else if (att.file) {
// FIXME: Change to new file library
file.attachData(
Buffer.from(att.file, 'base64'),
{