refactor dataexport controller

package.json

@@ -29,6 +29,7 @@
     "firebase": "^2.2.9",
     "firebase-token-generator": "^2.0.0",
     "glob": "^4.3.5",
+    "got": "^6.1.1",
     "grunt": "~0.4.1",
     "grunt-cli": "~0.1.9",
     "grunt-contrib-clean": "~0.6.0",
@@ -52,7 +53,7 @@
     "image-size": "~0.3.2",
     "in-app-purchase": "^0.2.0",
     "jade": "~1.11.0",
-    "js2xmlparser": "~0.1.2",
+    "js2xmlparser": "~1.0.0",
     "lodash": "^3.10.1",
     "loggly": "~1.0.8",
     "marked": "^0.3.5",
@@ -67,7 +68,7 @@
     "nib": "~1.0.1",
     "nodemailer": "^1.9.0",
     "object-path": "^0.9.2",
-    "pageres": "^1.0.1",
+    "pageres": "^4.1.0",
     "passport": "~0.2.1",
     "passport-facebook": "2.0.0",
     "paypal-ipn": "2.1.0",
@@ -77,7 +78,6 @@
     "push-notify": "^1.1.1",
     "q": "^1.4.1",
     "request": "~2.44.0",
-    "s3-upload-stream": "^1.0.6",
     "serve-favicon": "^2.3.0",
     "stripe": "*",
     "superagent": "~1.4.0",

website/src/controllers/api-v2/dataexport.js

@@ -22,21 +22,6 @@ var request = require('request');
   ------------------------------------------------------------------------
 */
 
-dataexport.history = function(req, res) {
-  var user = res.locals.user;
-  var output = [
-    ["Task Name", "Task ID", "Task Type", "Date", "Value"]
-  ];
-  _.each(user.tasks, function(task) {
-    _.each(task.history, function(history) {
-      output.push(
-        [task.text, task.id, task.type, moment(history.date).format("MM-DD-YYYY HH:mm:ss"), history.value]
-      );
-    });
-  });
-  return res.csv(output);
-}
-
 var userdata = function(user) {
   if(user.auth && user.auth.local) {
     delete user.auth.local.salt;

website/src/controllers/api-v3/dataexport.js

@@ -0,0 +1,225 @@
+import { authWithSession } from '../../middlewares/api-v3/auth';
+import cron from '../../middlewares/api-v3/cron';
+import { model as User } from '../../models/user';
+import * as Tasks from '../../models/task';
+import {
+  NotFound,
+} from '../../libs/api-v3/errors';
+import _ from 'lodash';
+import csvStringify from '../../libs/api-v3/csvStringify';
+import moment from 'moment';
+import js2xml from 'js2xmlparser';
+import Pageres from 'pageres';
+import AWS from 'aws-sdk';
+import nconf from 'nconf';
+import got from 'got';
+import Q from 'q';
+
+let S3 = new AWS.S3({
+  accessKeyId: nconf.get('S3:accessKeyId'),
+  secretAccessKey: nconf.get('S3:secretAccessKey'),
+});
+const S3_BUCKET = nconf.get('S3:bucket');
+
+const BASE_URL = nconf.get('BASE_URL');
+
+let api = {};
+
+/**
+ * @api {get} /export/history.csv Export user tasks history in CSV format. History is only available for habits and dailys so todos and rewards won't be included
+ * @apiVersion 3.0.0
+ * @apiName ExportUserHistory
+ * @apiGroup DataExport
+ *
+ * @apiSuccess {string} A cvs file
+ */
+api.exportUserHistory = {
+  method: 'GET',
+  url: '/export/history.csv',
+  middlewares: [authWithSession(), cron],
+  async handler (req, res) {
+    let user = res.locals.user;
+
+    let tasks = await Tasks.Task.find({
+      userId: user._id,
+      type: {$in: ['habit', 'daily']},
+    }).exec();
+
+    let output = [
+      ['Task Name', 'Task ID', 'Task Type', 'Date', 'Value'],
+    ];
+
+    tasks.forEach(task => {
+      task.history.forEach(history => {
+        output.push([
+          task.text,
+          task._id,
+          task.type,
+          moment(history.date).format('YYYY-MM-DD HH:mm:ss'),
+          history.value,
+        ]);
+      });
+    });
+
+    res.set({
+      'Content-Type': 'text/csv',
+      'Content-disposition': `attachment; filename=habitica-tasks-history.csv`,
+    });
+
+    let csvRes = await csvStringify(output);
+    res.status(200).send(csvRes);
+  },
+};
+
+// Convert user to json and attach tasks divided by type
+// at user.tasks[`${taskType}s`] (user.tasks.{dailys/habits/...})
+async function _getUserDataForExport (user) {
+  let userData = user.toJSON();
+  userData.tasks = {};
+
+  let tasks = await Tasks.Task.find({
+    userId: user._id,
+  }).exec();
+
+  tasks = _.chain(tasks)
+    .map(task => task.toJSON())
+    .groupBy(task => task.type)
+    .each((tasksPerType, taskType) => {
+      userData.tasks[`${taskType}s`] = tasksPerType;
+    })
+    .value();
+
+  return userData;
+}
+
+// TODO export tasks too
+/**
+ * @api {get} /export/userdata.json Export user data in JSON format.
+ * @apiVersion 3.0.0
+ * @apiName ExportUserDataJson
+ * @apiGroup DataExport
+ *
+ * @apiSuccess {string} A json file
+ */
+api.exportUserDataJson = {
+  method: 'GET',
+  url: '/export/userdata.json',
+  middlewares: [authWithSession(), cron],
+  async handler (req, res) {
+    let userData = await _getUserDataForExport(res.locals.user);
+
+    res.set({
+      'Content-Type': 'application/json',
+      'Content-disposition': `attachment; filename=habitica-user-data.json`,
+    });
+    let jsonRes = JSON.stringify(userData);
+
+    res.status(200).send(jsonRes);
+  },
+};
+
+/**
+ * @api {get} /export/userdata.xml Export user data in XML format
+ * @apiVersion 3.0.0
+ * @apiName ExportUserDataXml
+ * @apiGroup DataExport
+ *
+ * @apiSuccess {string} A xml file
+ */
+api.exportUserDataXml = {
+  method: 'GET',
+  url: '/export/userdata.xml',
+  middlewares: [authWithSession(), cron],
+  async handler (req, res) {
+    let userData = await _getUserDataForExport(res.locals.user);
+
+    res.set({
+      'Content-Type': 'text/xml',
+      'Content-disposition': `attachment; filename=habitica-user-data.xml`,
+    });
+    res.status(200).send(js2xml('user', userData));
+  },
+};
+
+/**
+ * @api {get} /export/avatar-:uuid.html Render a user avatar as an HTML page
+ * @apiVersion 3.0.0
+ * @apiName ExportUserAvatarHtml
+ * @apiGroup DataExport
+ *
+ * @apiSuccess {string} An html page
+ */
+api.exportUserAvatarHtml = {
+  method: 'GET',
+  url: '/export/avatar-:memberId.html',
+  async handler (req, res) {
+    req.checkParams('memberId', res.t('memberIdRequired')).notEmpty().isUUID();
+
+    let validationErrors = req.validationErrors();
+    if (validationErrors) throw validationErrors;
+
+    let memberId = req.params.memberId;
+    let member = await User
+      .findById(memberId)
+      .select('stats profile items achievements preferences backer contributor')
+      .exec();
+
+    if (!member) throw new NotFound(res.t('userWithIDNotFound', {userId: memberId}));
+    res.render('avatar-static', {
+      title: member.profile.name,
+      env: _.defaults({member}, res.locals.habitrpg), // TODO review once static pages are done
+    });
+  },
+};
+
+/**
+ * @api {get} /export/avatar-:uuid.html Export a user avatar as a PNG file
+ * @apiVersion 3.0.0
+ * @apiName ExportUserAvatarPng
+ * @apiGroup DataExport
+ *
+ * @apiSuccess {string} A png file
+ */
+api.exportUserAvatarPng = {
+  method: 'GET',
+  url: '/export/avatar-:memberId.png',
+  async handler (req, res) {
+    req.checkParams('memberId', res.t('memberIdRequired')).notEmpty().isUUID();
+
+    let validationErrors = req.validationErrors();
+    if (validationErrors) throw validationErrors;
+
+    let memberId = req.params.memberId;
+
+    let filename = `avatars/${memberId}/.png`;
+    let s3url = `https://${S3_BUCKET}+'.s3.amazonaws.com/${filename}`;
+    let response = await got.head(s3url);
+
+    // cache images for 10 minutes on aws, else upload a new one
+    if (response.statusCode === 200 && moment().diff(response.headers['last-modified'], 'minutes') < 10) {
+      return res.redirect(301, s3url);
+    }
+
+    let [stream] = await new Pageres()
+    .src(`${BASE_URL}/export/avatar-${memberId}.html`, ['140x147'], {
+      crop: true,
+      filename: filename.replace('.png', ''),
+    })
+    .run();
+
+    let s3upload = S3.upload({
+      Bucket: S3_BUCKET,
+      Key: filename,
+      ACL: 'public-read',
+      StorageClass: 'REDUCED_REDUNDANCY',
+      ContentType: 'image/png',
+      Expires: moment().add({minutes: 3}),
+      Body: stream,
+    });
+
+    let s3res = await Q.ninvoke(s3upload, 'send');
+    res.redirect(s3res.Location);
+  },
+};
+
+export default api;

website/src/libs/api-v3/webhook.js

@@ -7,7 +7,7 @@ let _sendWebhook = (url, body) => {
     url,
     body,
     json: true,
-  });
+  }); // TODO use promises and handle errors
 };
 
 let _isInvalidWebhook = (hook) => {

website/src/routes/dataexport.js

@@ -1,6 +1,6 @@
 var express = require('express');
 var router = new express.Router();
-var dataexport = require('../controllers/dataexport');
+var dataexport = require('../controllers/api-v2/dataexport');
 var auth = require('../controllers/api-v2/auth');
 var nconf = require('nconf');
 var i18n = require('../libs/api-v2/i18n');