chore: Add quest collection migration and temporary code to catch data inconsistencies

migrations/20160602_convert_quest_collection.js

@@ -0,0 +1,140 @@
+'use strict';
+
+/****************************************
+ * Author: Blade Barringer @crookedneighbor
+ *
+ * Reason: Collection quest data on the client is unreliable
+ * because the quest key on the user.party.quest.key property
+ * is unreliable. We were calculating the quest items found
+ * at the time a drop was created, when instead we could
+ * just calculate it from the party on the server. This
+ * necessitates changing the property type of party.quest.progress.collect
+ * from an object to a number, hence this migration.
+ ***************************************/
+
+global.Promise = require('bluebird');
+const TaskQueue = require('cwait').TaskQueue;
+const logger = require('./utils/logger');
+const Timer = require('./utils/timer');
+const connectToDb = require('./utils/connect').connectToDb;
+const closeDb = require('./utils/connect').closeDb;
+
+const timer = new Timer();
+
+// PROD: Enable prod db
+// const DB_URI = 'mongodb://username:password@dsXXXXXX-a0.mlab.com:XXXXX,dsXXXXXX-a1.mlab.com:XXXXX/habitica?replicaSet=rs-dsXXXXXX';
+const DB_URI = 'mongodb://localhost/new-prod-copy';
+
+const COLLECTION_QUESTS = [
+  'evilsanta2',
+  'vice2',
+  'egg',
+  'atom1',
+  'moonstone1',
+  'goldenknight1',
+  'dilatoryDistress1',
+]
+
+let Users, Groups;
+
+connectToDb(DB_URI).then((db) => {
+  Users = db.collection('users_backup');
+  Groups = db.collection('groups_backup');
+
+  return Promise.resolve();
+})
+.then(findUsersWithCollectionData)
+.then(getUsersCollectionData)
+.then(transformCollectionData)
+.then(cleanUpEmptyCollectionData)
+.then(() => {
+  timer.stop();
+  closeDb();
+}).catch(reportError);
+
+function reportError (err) {
+  logger.error('Uh oh, an error occurred');
+  closeDb();
+  timer.stop();
+  throw err;
+}
+
+function findUsersWithCollectionData () {
+  logger.info('Looking up groups on collection quests...');
+
+  return Groups.find({'quest.key': {$in: COLLECTION_QUESTS }}, ['quest.members']).toArray().then((groups) => {
+    logger.success('Found', groups.length, 'parties on collection quests');
+    logger.info('Parsing member data...');
+
+    let members = groups.reduce((array, party) => {
+      let questers = Object.keys(party.quest.members);
+      array.push.apply(array, questers);
+      return array;
+    }, []);
+
+    logger.success('Found', members.length, 'users on collection quests');
+
+    return Promise.resolve(members);
+  })
+}
+
+function getUsersCollectionData (users) {
+  logger.info('Fetching collection data from users...');
+
+  return Users.find({_id: {$in: users}}, ['party.quest.progress']).toArray().then((docs) => {
+    let items = docs.reduce((array, user) => {
+      let total = 0;
+      let collect = user.party && user.party.quest && user.party.quest.progress && user.party.quest.progress.collect;
+
+      if (!collect) return array;
+      if (typeof collect === 'number') return array;
+
+      for (var i in collect) {
+        if (collect.hasOwnProperty(i)) {
+          total += collect[i];
+        }
+      }
+
+      array.push({_id: user._id, collect: total});
+      return array;
+    }, []);
+
+    return Promise.resolve(items);
+  });
+}
+
+function updateUserById (user) {
+  return Users.findOneAndUpdate({_id: user._id}, {$set: {'party.quest.progress.collect': user.collect}}, {returnOriginal: false})
+}
+
+
+function transformCollectionData (users) {
+  let queue = new TaskQueue(Promise, 300);
+
+  logger.info('About to update', users.length, 'user collection items...');
+
+  return Promise.map(users, queue.wrap(updateUserById)).then((result) => {
+    let updates = result.filter(res => res.lastErrorObject && res.lastErrorObject.updatedExisting)
+    let failures = result.filter(res => !(res.lastErrorObject && res.lastErrorObject.updatedExisting));
+
+    logger.success(updates.length, 'users have been fixed');
+
+    if (failures.length > 0) {
+      logger.error(failures.length, 'users could not be found');
+    }
+
+    return Promise.resolve();
+  });
+}
+
+function cleanUpEmptyCollectionData () {
+  logger.info('Fetching users without collection data...');
+
+  return Users.updateMany({$or: [{'party.quest.progress.collect': { $type: 3}}, {'party.quest.progress.collect': { $exists: false}}]}, {$set: {'party.quest.progress.collect': 0}}).then((r) => {
+    let updates = r.result.n;
+
+    logger.success(updates, 'users have been fixed');
+
+    return Promise.resolve();
+  });
+}

website/server/controllers/api-v3/tasks.js

@@ -392,6 +392,17 @@ api.scoreTask = {
 
     let wasCompleted = task.completed;
 
+    // TEMPORARY, remove once collection migration completes
+    if (typeof user.party.quest.progress.collect === 'object') {
+      let totalItemsFound = _.reduce(user.party.quest.progress.collect, (total, amount) => {
+        return total + amount;
+      }, 0);
+
+      user.party.quest.progress.collect = totalItemsFound;
+    } else if (!user.party.quest.progress.collect) {
+      user.party.quest.progress.collect = 0;
+    }
+
     let [delta] = common.ops.scoreTask({task, user, direction}, req);
     // Drop system (don't run on the client, as it would only be discarded since ops are sent to the API, not the results)
     if (direction === 'up') user.fns.randomDrop({task, delta}, req);

website/server/models/group.js

@@ -589,6 +589,17 @@ schema.statics.processQuestProgress = async function processQuestProgress (user,
 
   if (!_isOnQuest(user, progress, group)) return;
 
+  // TEMPORARY, remove once collection migration completes
+  if (typeof progress.collect === 'object') {
+    let totalItemsFound = _.reduce(progress.collect, (total, amount) => {
+      return total + amount;
+    }, 0);
+
+    progress.collect = totalItemsFound;
+  } else if (!progress.collect) {
+    progress.collect = 0;
+  }
+
   let quest = shared.content.quests[group.quest.key];
 
   if (!quest) return; // TODO should this throw an error instead?

website/server/models/user.js

@@ -376,7 +376,8 @@ export let schema = new Schema({
       progress: {
         up: {type: Number, default: 0},
         down: {type: Number, default: 0},
-        collect: {type: Number, default: 0},
+        // TEMPORARY - Switch type to Number after migration
+        collect: {type: Schema.Types.Mixed, default: 0},
       },
       completed: String, // When quest is done, we move it from key => completed, and it's a one-time flag (for modal) that they unset by clicking "ok" in browser
       RSVPNeeded: {type: Boolean, default: false}, // Set to true when invite is pending, set to false when quest invite is accepted or rejected, quest starts, or quest is cancelled