add: export board/cards/lists to CSV/TSV

This commit is contained in:
Bryan Mutai 2020-05-10 23:58:15 +03:00
parent 1742bcd9b1
commit a570c4a861
9 changed files with 474 additions and 217 deletions

View file

@ -302,7 +302,7 @@ template(name="boardMenuPopup")
ul.pop-over-list ul.pop-over-list
if withApi if withApi
li li
a(href="{{exportUrl}}", download="{{exportFilename}}") a.js-export-board
i.fa.fa-share-alt i.fa.fa-share-alt
| {{_ 'export-board'}} | {{_ 'export-board'}}
li li
@ -362,6 +362,21 @@ template(name="boardMenuPopup")
i.fa.fa-sitemap i.fa.fa-sitemap
| {{_ 'subtask-settings'}} | {{_ 'subtask-settings'}}
template(name="exportBoard")
ul.pop-over-list
li
a(href="{{exportUrl}}", download="{{exportJsonFilename}}")
i.fa.fa-share-alt
| {{_ 'export-board-json'}}
li
a(href="{{exportCsvUrl}}", download="{{exportCsvFilename}}")
i.fa.fa-share-alt
| {{_ 'export-board-csv'}}
li
a(href="{{exportTsvUrl}}", download="{{exportTsvFilename}}")
i.fa.fa-share-alt
| {{_ 'export-board-tsv'}}
template(name="labelsWidget") template(name="labelsWidget")
.board-widget.board-widget-labels .board-widget.board-widget-labels
h3 h3

View file

@ -1,3 +1,4 @@
sidebar.js;
import { Cookies } from 'meteor/ostrio:cookies'; import { Cookies } from 'meteor/ostrio:cookies';
const cookies = new Cookies(); const cookies = new Cookies();
Sidebar = null; Sidebar = null;
@ -213,6 +214,7 @@ Template.boardMenuPopup.events({
'click .js-import-board': Popup.open('chooseBoardSource'), 'click .js-import-board': Popup.open('chooseBoardSource'),
'click .js-subtask-settings': Popup.open('boardSubtaskSettings'), 'click .js-subtask-settings': Popup.open('boardSubtaskSettings'),
'click .js-card-settings': Popup.open('boardCardSettings'), 'click .js-card-settings': Popup.open('boardCardSettings'),
'click .js-export-board': Popup.open('exportBoard'),
}); });
Template.boardMenuPopup.onCreated(function() { Template.boardMenuPopup.onCreated(function() {
@ -405,6 +407,63 @@ BlazeComponent.extendComponent({
}, },
}).register('chooseBoardSourcePopup'); }).register('chooseBoardSourcePopup');
BlazeComponent.extendComponent({
template() {
return 'exportBoard';
},
withApi() {
return Template.instance().apiEnabled.get();
},
exportUrl() {
const params = {
boardId: Session.get('currentBoard'),
};
const queryParams = {
authToken: Accounts._storedLoginToken(),
};
return FlowRouter.path('/api/boards/:boardId/export', params, queryParams);
},
exportCsvUrl() {
const params = {
boardId: Session.get('currentBoard'),
};
const queryParams = {
authToken: Accounts._storedLoginToken(),
};
return FlowRouter.path(
'/api/boards/:boardId/export/csv',
params,
queryParams,
);
},
exportTsvUrl() {
const params = {
boardId: Session.get('currentBoard'),
};
const queryParams = {
authToken: Accounts._storedLoginToken(),
delimiter: '\t',
};
return FlowRouter.path(
'/api/boards/:boardId/export/csv',
params,
queryParams,
);
},
exportJsonFilename() {
const boardId = Session.get('currentBoard');
return `wekan-export-board-${boardId}.json`;
},
exportCsvFilename() {
const boardId = Session.get('currentBoard');
return `wekan-export-board-${boardId}.csv`;
},
exportTsvFilename() {
const boardId = Session.get('currentBoard');
return `wekan-export-board-${boardId}.tsv`;
},
}).register('exportBoardPopup');
Template.labelsWidget.events({ Template.labelsWidget.events({
'click .js-label': Popup.open('editLabel'), 'click .js-label': Popup.open('editLabel'),
'click .js-add-label': Popup.open('createLabel'), 'click .js-add-label': Popup.open('createLabel'),

View file

@ -315,6 +315,10 @@
"error-username-taken": "This username is already taken", "error-username-taken": "This username is already taken",
"error-email-taken": "Email has already been taken", "error-email-taken": "Email has already been taken",
"export-board": "Export board", "export-board": "Export board",
"export-board-json": "Export board to JSON",
"export-board-csv": "Export board to CSV",
"export-board-tsv": "Export board to TSV",
"exportBoardPopup-title": "Export board",
"sort": "Sort", "sort": "Sort",
"sort-desc": "Click to Sort List", "sort-desc": "Click to Sort List",
"list-sort-by": "Sort the List By:", "list-sort-by": "Sort the List By:",

View file

@ -128,10 +128,9 @@ export class CsvCreator {
}; };
// create labels // create labels
const labelsToCreate = new Set();
for (let i = 1; i < csvData.length; i++) { for (let i = 1; i < csvData.length; i++) {
//get the label column
if (csvData[i][this.fieldIndex.labels]) { if (csvData[i][this.fieldIndex.labels]) {
const labelsToCreate = new Set();
for (const importedLabel of csvData[i][this.fieldIndex.labels].split( for (const importedLabel of csvData[i][this.fieldIndex.labels].split(
' ', ' ',
)) { )) {
@ -139,23 +138,23 @@ export class CsvCreator {
labelsToCreate.add(importedLabel); labelsToCreate.add(importedLabel);
} }
} }
for (const label of labelsToCreate) {
let labelName, labelColor;
if (label.indexOf('-') > -1) {
labelName = label.split('-')[0];
labelColor = label.split('-')[1];
} else {
labelName = label;
}
const labelToCreate = {
_id: Random.id(6),
color: labelColor ? labelColor : 'black',
name: labelName,
};
boardToCreate.labels.push(labelToCreate);
}
} }
} }
for (const label of labelsToCreate) {
let labelName, labelColor;
if (label.indexOf('-') > -1) {
labelName = label.split('-')[0];
labelColor = label.split('-')[1];
} else {
labelName = label;
}
const labelToCreate = {
_id: Random.id(6),
color: labelColor ? labelColor : 'black',
name: labelName,
};
boardToCreate.labels.push(labelToCreate);
}
const boardId = Boards.direct.insert(boardToCreate); const boardId = Boards.direct.insert(boardToCreate);
Boards.direct.update(boardId, { Boards.direct.update(boardId, {

View file

@ -1,3 +1,4 @@
import { Exporter } from './exporter';
/* global JsonRoutes */ /* global JsonRoutes */
if (Meteor.isServer) { if (Meteor.isServer) {
// todo XXX once we have a real API in place, move that route there // todo XXX once we have a real API in place, move that route there
@ -7,10 +8,10 @@ if (Meteor.isServer) {
// on the client instead of copy/pasting the route path manually between the // on the client instead of copy/pasting the route path manually between the
// client and the server. // client and the server.
/** /**
* @operation export * @operation exportJson
* @tag Boards * @tag Boards
* *
* @summary This route is used to export the board. * @summary This route is used to export the board to a json file format.
* *
* @description If user is already logged-in, pass loginToken as param * @description If user is already logged-in, pass loginToken as param
* "authToken": '/api/boards/:boardId/export?authToken=:token' * "authToken": '/api/boards/:boardId/export?authToken=:token'
@ -46,199 +47,52 @@ if (Meteor.isServer) {
JsonRoutes.sendResult(res, 403); JsonRoutes.sendResult(res, 403);
} }
}); });
}
/**
// exporter maybe is broken since Gridfs introduced, add fs and path * @operation exportCSV/TSV
* @tag Boards
export class Exporter { *
constructor(boardId) { * @summary This route is used to export the board to a CSV or TSV file format.
this._boardId = boardId; *
} * @description If user is already logged-in, pass loginToken as param
*
build() { * See https://blog.kayla.com.au/server-side-route-authentication-in-meteor/
const fs = Npm.require('fs'); * for detailed explanations
const os = Npm.require('os'); *
const path = Npm.require('path'); * @param {string} boardId the ID of the board we are exporting
* @param {string} authToken the loginToken
const byBoard = { boardId: this._boardId }; * @param {string} delimiter delimiter to use while building export. Default is comma ','
const byBoardNoLinked = { */
boardId: this._boardId, Picker.route('/api/boards/:boardId/export/csv', function(params, req, res) {
linkedId: { $in: ['', null] }, const boardId = params.boardId;
}; let user = null;
// we do not want to retrieve boardId in related elements const loginToken = params.query.authToken;
const noBoardId = { if (loginToken) {
fields: { const hashToken = Accounts._hashLoginToken(loginToken);
boardId: 0, user = Meteor.users.findOne({
}, 'services.resume.loginTokens.hashedToken': hashToken,
}; });
const result = { } else if (!Meteor.settings.public.sandstorm) {
_format: 'wekan-board-1.0.0', Authentication.checkUserId(req.userId);
}; user = Users.findOne({
_.extend( _id: req.userId,
result, isAdmin: true,
Boards.findOne(this._boardId, { });
fields: { }
stars: 0, const exporter = new Exporter(boardId);
}, if (exporter.canExport(user)) {
}), body = params.query.delimiter
); ? exporter.buildCsv(params.query.delimiter)
result.lists = Lists.find(byBoard, noBoardId).fetch(); : exporter.buildCsv();
result.cards = Cards.find(byBoardNoLinked, noBoardId).fetch(); res.writeHead(200, {
result.swimlanes = Swimlanes.find(byBoard, noBoardId).fetch(); 'Content-Length': body[0].length,
result.customFields = CustomFields.find( 'Content-Type': params.query.delimiter ? 'text/csv' : 'text/tsv',
{ boardIds: { $in: [this.boardId] } }, });
{ fields: { boardId: 0 } }, res.write(body[0]);
).fetch(); res.end();
result.comments = CardComments.find(byBoard, noBoardId).fetch(); } else {
result.activities = Activities.find(byBoard, noBoardId).fetch(); res.writeHead(403);
result.rules = Rules.find(byBoard, noBoardId).fetch(); res.end('Permission Error');
result.checklists = []; }
result.checklistItems = []; });
result.subtaskItems = [];
result.triggers = [];
result.actions = [];
result.cards.forEach(card => {
result.checklists.push(
...Checklists.find({
cardId: card._id,
}).fetch(),
);
result.checklistItems.push(
...ChecklistItems.find({
cardId: card._id,
}).fetch(),
);
result.subtaskItems.push(
...Cards.find({
parentId: card._id,
}).fetch(),
);
});
result.rules.forEach(rule => {
result.triggers.push(
...Triggers.find(
{
_id: rule.triggerId,
},
noBoardId,
).fetch(),
);
result.actions.push(
...Actions.find(
{
_id: rule.actionId,
},
noBoardId,
).fetch(),
);
});
// [Old] for attachments we only export IDs and absolute url to original doc
// [New] Encode attachment to base64
const getBase64Data = function(doc, callback) {
let buffer = Buffer.allocUnsafe(0);
buffer.fill(0);
// callback has the form function (err, res) {}
const tmpFile = path.join(
os.tmpdir(),
`tmpexport${process.pid}${Math.random()}`,
);
const tmpWriteable = fs.createWriteStream(tmpFile);
const readStream = doc.createReadStream();
readStream.on('data', function(chunk) {
buffer = Buffer.concat([buffer, chunk]);
});
readStream.on('error', function(err) {
callback(null, null);
});
readStream.on('end', function() {
// done
fs.unlink(tmpFile, () => {
//ignored
});
callback(null, buffer.toString('base64'));
});
readStream.pipe(tmpWriteable);
};
const getBase64DataSync = Meteor.wrapAsync(getBase64Data);
result.attachments = Attachments.find(byBoard)
.fetch()
.map(attachment => {
let filebase64 = null;
filebase64 = getBase64DataSync(attachment);
return {
_id: attachment._id,
cardId: attachment.cardId,
//url: FlowRouter.url(attachment.url()),
file: filebase64,
name: attachment.original.name,
type: attachment.original.type,
};
});
// we also have to export some user data - as the other elements only
// include id but we have to be careful:
// 1- only exports users that are linked somehow to that board
// 2- do not export any sensitive information
const users = {};
result.members.forEach(member => {
users[member.userId] = true;
});
result.lists.forEach(list => {
users[list.userId] = true;
});
result.cards.forEach(card => {
users[card.userId] = true;
if (card.members) {
card.members.forEach(memberId => {
users[memberId] = true;
});
}
});
result.comments.forEach(comment => {
users[comment.userId] = true;
});
result.activities.forEach(activity => {
users[activity.userId] = true;
});
result.checklists.forEach(checklist => {
users[checklist.userId] = true;
});
const byUserIds = {
_id: {
$in: Object.getOwnPropertyNames(users),
},
};
// we use whitelist to be sure we do not expose inadvertently
// some secret fields that gets added to User later.
const userFields = {
fields: {
_id: 1,
username: 1,
'profile.fullname': 1,
'profile.initials': 1,
'profile.avatarUrl': 1,
},
};
result.users = Users.find(byUserIds, userFields)
.fetch()
.map(user => {
// user avatar is stored as a relative url, we export absolute
if ((user.profile || {}).avatarUrl) {
user.profile.avatarUrl = FlowRouter.url(user.profile.avatarUrl);
}
return user;
});
return result;
}
canExport(user) {
const board = Boards.findOne(this._boardId);
return board && board.isVisibleBy(user);
}
} }

320
models/exporter.js Normal file
View file

@ -0,0 +1,320 @@
models / exporter.js;
const stringify = require('csv-stringify');
// exporter maybe is broken since Gridfs introduced, add fs and path
export class Exporter {
constructor(boardId) {
this._boardId = boardId;
}
build() {
const fs = Npm.require('fs');
const os = Npm.require('os');
const path = Npm.require('path');
const byBoard = { boardId: this._boardId };
const byBoardNoLinked = {
boardId: this._boardId,
linkedId: { $in: ['', null] },
};
// we do not want to retrieve boardId in related elements
const noBoardId = {
fields: {
boardId: 0,
},
};
const result = {
_format: 'wekan-board-1.0.0',
};
_.extend(
result,
Boards.findOne(this._boardId, {
fields: {
stars: 0,
},
}),
);
result.lists = Lists.find(byBoard, noBoardId).fetch();
result.cards = Cards.find(byBoardNoLinked, noBoardId).fetch();
result.swimlanes = Swimlanes.find(byBoard, noBoardId).fetch();
result.customFields = CustomFields.find(
{ boardIds: { $in: [this.boardId] } },
{ fields: { boardId: 0 } },
).fetch();
result.comments = CardComments.find(byBoard, noBoardId).fetch();
result.activities = Activities.find(byBoard, noBoardId).fetch();
result.rules = Rules.find(byBoard, noBoardId).fetch();
result.checklists = [];
result.checklistItems = [];
result.subtaskItems = [];
result.triggers = [];
result.actions = [];
result.cards.forEach(card => {
result.checklists.push(
...Checklists.find({
cardId: card._id,
}).fetch(),
);
result.checklistItems.push(
...ChecklistItems.find({
cardId: card._id,
}).fetch(),
);
result.subtaskItems.push(
...Cards.find({
parentId: card._id,
}).fetch(),
);
});
result.rules.forEach(rule => {
result.triggers.push(
...Triggers.find(
{
_id: rule.triggerId,
},
noBoardId,
).fetch(),
);
result.actions.push(
...Actions.find(
{
_id: rule.actionId,
},
noBoardId,
).fetch(),
);
});
// [Old] for attachments we only export IDs and absolute url to original doc
// [New] Encode attachment to base64
const getBase64Data = function(doc, callback) {
let buffer = Buffer.allocUnsafe(0);
buffer.fill(0);
// callback has the form function (err, res) {}
const tmpFile = path.join(
os.tmpdir(),
`tmpexport${process.pid}${Math.random()}`,
);
const tmpWriteable = fs.createWriteStream(tmpFile);
const readStream = doc.createReadStream();
readStream.on('data', function(chunk) {
buffer = Buffer.concat([buffer, chunk]);
});
readStream.on('error', function() {
callback(null, null);
});
readStream.on('end', function() {
// done
fs.unlink(tmpFile, () => {
//ignored
});
callback(null, buffer.toString('base64'));
});
readStream.pipe(tmpWriteable);
};
const getBase64DataSync = Meteor.wrapAsync(getBase64Data);
result.attachments = Attachments.find(byBoard)
.fetch()
.map(attachment => {
let filebase64 = null;
filebase64 = getBase64DataSync(attachment);
return {
_id: attachment._id,
cardId: attachment.cardId,
//url: FlowRouter.url(attachment.url()),
file: filebase64,
name: attachment.original.name,
type: attachment.original.type,
};
});
// we also have to export some user data - as the other elements only
// include id but we have to be careful:
// 1- only exports users that are linked somehow to that board
// 2- do not export any sensitive information
const users = {};
result.members.forEach(member => {
users[member.userId] = true;
});
result.lists.forEach(list => {
users[list.userId] = true;
});
result.cards.forEach(card => {
users[card.userId] = true;
if (card.members) {
card.members.forEach(memberId => {
users[memberId] = true;
});
}
});
result.comments.forEach(comment => {
users[comment.userId] = true;
});
result.activities.forEach(activity => {
users[activity.userId] = true;
});
result.checklists.forEach(checklist => {
users[checklist.userId] = true;
});
const byUserIds = {
_id: {
$in: Object.getOwnPropertyNames(users),
},
};
// we use whitelist to be sure we do not expose inadvertently
// some secret fields that gets added to User later.
const userFields = {
fields: {
_id: 1,
username: 1,
'profile.fullname': 1,
'profile.initials': 1,
'profile.avatarUrl': 1,
},
};
result.users = Users.find(byUserIds, userFields)
.fetch()
.map(user => {
// user avatar is stored as a relative url, we export absolute
if ((user.profile || {}).avatarUrl) {
user.profile.avatarUrl = FlowRouter.url(user.profile.avatarUrl);
}
return user;
});
return result;
}
buildCsv(delimiter = ',') {
const result = this.build();
const columnHeaders = [];
const cardRows = [];
columnHeaders.push(
'Title',
'Description',
'Status',
'Swimlane',
'Owner',
'Requested by',
'Assigned by',
'Members',
'Assignees',
'Labels',
'Start at',
'Due at',
'End at',
'Over time',
'Spent time (hours)',
'Created at',
'Last modified at',
'Last activity',
'Vote',
'Archived',
);
const stringifier = stringify({
header: true,
delimiter,
columns: columnHeaders,
});
stringifier.on('readable', function() {
let row;
while ((row = stringifier.read())) {
cardRows.push(row);
}
});
stringifier.on('error', function(err) {
// eslint-disable-next-line no-console
console.error(err.message);
});
result.cards.forEach(card => {
const currentRow = [];
currentRow.push(card.title);
currentRow.push(card.description);
currentRow.push(
result.lists.find(({ _id }) => _id === card.listId).title,
);
currentRow.push(
result.swimlanes.find(({ _id }) => _id === card.swimlaneId).title,
);
currentRow.push(
result.users.find(({ _id }) => _id === card.userId).username,
);
currentRow.push(card.requestedBy ? card.requestedBy : ' ');
currentRow.push(card.assignedBy ? card.assignedBy : ' ');
let usernames = '';
card.members.forEach(memberId => {
const user = result.users.find(({ _id }) => _id === memberId);
usernames = `${usernames + user.username} `;
});
currentRow.push(usernames.trim());
let assignees = '';
card.assignees.forEach(assigneeId => {
const user = result.users.find(({ _id }) => _id === assigneeId);
assignees = `${assignees + user.username} `;
});
currentRow.push(assignees.trim());
let labels = '';
card.labelIds.forEach(labelId => {
const label = result.labels.find(({ _id }) => _id === labelId);
labels = `${labels + label.name}-${label.color} `;
});
currentRow.push(labels.trim());
currentRow.push(card.startAt ? moment(card.startAt).format('LLLL') : ' ');
currentRow.push(card.dueAt ? moment(card.dueAt).format('LLLL') : ' ');
currentRow.push(card.endAt ? moment(card.endAt).format('LLLL') : ' ');
currentRow.push(card.isOvertime ? 'true' : 'false');
currentRow.push(card.spentTime);
currentRow.push(
card.createdAt ? moment(card.createdAt).format('LLLL') : ' ',
);
currentRow.push(
card.modifiedAt ? moment(card.modifiedAt).format('LLLL') : ' ',
);
currentRow.push(
card.dateLastActivity
? moment(card.dateLastActivity).format('LLLL')
: ' ',
);
if (card.vote.question) {
let positiveVoters = '';
let negativeVoters = '';
card.vote.positive.forEach(userId => {
const user = result.users.find(({ _id }) => _id === userId);
positiveVoters = `${positiveVoters + user.username} `;
});
card.vote.negative.forEach(userId => {
const user = result.users.find(({ _id }) => _id === userId);
negativeVoters = `${negativeVoters + user.username} `;
});
const votingResult = `${
card.vote.public
? `yes-${
card.vote.positive.length
}-${positiveVoters.trimRight()}-no-${
card.vote.negative.length
}-${negativeVoters.trimRight()}`
: `yes-${card.vote.positive.length}-no-${card.vote.negative.length}`
}`;
currentRow.push(`${card.vote.question}-${votingResult}`);
} else {
currentRow.push(' ');
}
currentRow.push(card.archived ? 'true' : 'false');
stringifier.write(currentRow);
});
stringifier.end();
return cardRows;
}
canExport(user) {
const board = Boards.findOne(this._boardId);
return board && board.isVisibleBy(user);
}
}

View file

@ -1,8 +1,8 @@
import { TrelloCreator } from './trelloCreator'; import { TrelloCreator } from './trelloCreator';
import { WekanCreator } from './wekanCreator'; import { WekanCreator } from './wekanCreator';
import { Exporter } from './export';
import wekanMembersMapper from './wekanmapper';
import { CsvCreator } from './csvCreator'; import { CsvCreator } from './csvCreator';
import { Exporter } from './exporter';
import wekanMembersMapper from './wekanmapper';
Meteor.methods({ Meteor.methods({
importBoard(board, data, importSource, currentBoard) { importBoard(board, data, importSource, currentBoard) {

5
package-lock.json generated
View file

@ -894,6 +894,11 @@
"resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz",
"integrity": "sha1-xtJnJjKi5cg+AT5oZKQs6N79IK4=" "integrity": "sha1-xtJnJjKi5cg+AT5oZKQs6N79IK4="
}, },
"csv-stringify": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-5.5.0.tgz",
"integrity": "sha512-G05575DSO/9vFzQxZN+Srh30cNyHk0SM0ePyiTChMD5WVt7GMTVPBQf4rtgMF6mqhNCJUPw4pN8LDe8MF9EYOA=="
},
"dashdash": { "dashdash": {
"version": "1.14.1", "version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",

View file

@ -61,6 +61,7 @@
"bcrypt": "^3.0.7", "bcrypt": "^3.0.7",
"bson": "^4.0.3", "bson": "^4.0.3",
"bunyan": "^1.8.12", "bunyan": "^1.8.12",
"csv-stringify": "^5.5.0",
"es6-promise": "^4.2.4", "es6-promise": "^4.2.4",
"flatted": "^2.0.1", "flatted": "^2.0.1",
"gridfs-stream": "^0.5.3", "gridfs-stream": "^0.5.3",