fix migrations

migrations/20150906_groups_fix_leaders.js

@@ -3,8 +3,9 @@
  */
 
 var mongo = require('mongoskin');
+var async = require('async');
 
-var dbserver = 'mongodb://url';
+var dbserver = 'url';
 var dbname = 'dbname';
 var countGroups = 0;
 var countUsers = 0;
@@ -15,35 +16,64 @@ var dbGroups = db.collection('groups');
 
 console.log('Begins work on db');
 
-dbGroups.findEach({}, {_id: 1, members: 1, leader: 1}, {batchSize: 500}, function(err, group) {
-  if(err) throw err;
-  if(!group) return;
+function findGroups(gt){
+  var query = {};
+  if(gt) query._id = {$gt: gt};
 
-  countGroups++;
-  console.log('Group: ', countGroups);
+  console.log(query)
 
-  var members = group.members;
-  var leader = group.leader;
-
-  dbUsers.count({_id: group.leader}, function(err, count){
+  dbGroups.find(query, {
+    fields: {_id: 1, members: 1, leader: 1},
+    limit: 10000,
+    sort: {
+      _id: 1
+    }
+  }).toArray(function(err, groups){
     if(err) throw err;
 
-    // If leader has deleted account
-    if(count < 1 && (group._id !== 'habitrpg') && members && members[0]) {
-      dbGroups.update({
-        _id: group._id
-      }, {
-        $set: {
-          // Set first user as new leader
-          leader: members[0]
-        }
-      }, {
-        multi: false
-      }, function(err, res){
-        if(err) throw err;
-
-        console.log('Updated: ', res);
-      });
+    var lastGroup = null;
+    if(groups.length === 10000){
+      lastGroup = groups[groups.length - 1];
     }
-  })
-});
+
+    async.eachLimit(groups, 30, function(group, cb1){
+      countGroups++;
+      console.log('Group: ', countGroups, group._id);
+
+      var members = group.members;
+
+      dbUsers.findOne({_id: group.leader}, {fields: {_id: 1}}, function(err, user){
+        if(err) return cb1(err);
+
+        // If leader has deleted account
+        if(!user && (group._id !== 'habitrpg') && members && members[0]) {
+          dbGroups.update({
+            _id: group._id
+          }, {
+            $set: {
+              // Set first user as new leader
+              leader: members[0]
+            }
+          }, {
+            multi: false
+          }, function(err, res){
+            if(err) return cb1(err);
+
+            console.log('Updated: ', res);
+            return cb1();
+          });
+        }else{
+          return cb1();
+        }
+      });
+    }, function(err){
+      if(err) throw err;
+
+      if(lastGroup && lastGroup._id){
+        findGroups(lastGroup._id);
+      }
+    });
+  });
+};
+
+findGroups();

migrations/20150906_groups_remove_deleted_users.js

@@ -3,8 +3,9 @@
  */
 
 var mongo = require('mongoskin');
+var async = require('async');
 
-var dbserver = 'mongodb://url';
+var dbserver = 'url';
 var dbname = 'dbname';
 var countGroups = 0;
 var countUsers = 0;
@@ -15,38 +16,72 @@ var dbGroups = db.collection('groups');
 
 console.log('Begins work on db');
 
-dbGroups.findEach({}, {_id: 1, members: 1}, {batchSize: 500}, function(err, group) {
-  if(err) throw err;
-  if(!group) return;
+function findGroups(gt){
+  var query = {};
+  if(gt) query._id = {$gt: gt};
 
-  countGroups++;
-  console.log('Group: ', countGroups);
+  console.log(query)
 
-  var members = group.members;
+  dbGroups.find(query, {
+    fields: {_id: 1, members: 1},
+    limit: 10000,
+    sort: {
+      _id: 1
+    }
+  }).toArray(function(err, groups){
+    if(err) throw err;
 
-  // Remove users who deleted their account
-  members.forEach(function(member){
-    dbUsers.count({_id: member}, function(err, count){
+    var lastGroup = null;
+    if(groups.length === 10000){
+      lastGroup = groups[groups.length - 1];
+    }
+
+    async.eachLimit(groups, 3, function(group, cb1){
+      countGroups++;
+      console.log('Group: ', countGroups, group._id);
+
+      var members = group.members;
+
+      // Remove users who deleted their account
+      async.eachLimit(members, 15, function(member, cb2){
+        dbUsers.findOne({_id: member}, {fields: {_id: 1}}, function(err, user){
+          if(err) return cb2(err);
+
+          if(!user){
+            countUsers++;
+            console.log('User removed n. ', countUsers, 'user id ', member, 'group id ', group._id);
+
+            dbGroups.update({
+              _id: group._id
+            }, {
+              $pull: {members: member},
+              $inc: {memberCount: -1}
+            }, {
+              multi: false
+            }, function(err, res){
+              if(err) return cb2(err);
+
+              console.log('Updated: ', res);
+              return cb2();
+            });
+          }else{
+            cb2();
+          }
+        });
+      }, function(err){
+        if(err) return cb1(err);
+
+        cb1();
+      });
+
+    }, function(err){
       if(err) throw err;
 
-      if(count < 1) {
-        countUsers++;
-        console.log('User: ', countUsers);
-
-        // TODO updating the same group many times concurrently can cause problems?
-        dbGroups.update({
-          _id: group._id
-        }, {
-          $pull: {members: member},
-          $inc: {memberCount: -1}
-        }, {
-          multi: false
-        }, function(err, res){
-          if(err) throw err;
-
-          console.log('Updated: ', res);
-        });
+      if(lastGroup && lastGroup._id){
+        findGroups(lastGroup._id);
       }
     });
   });
-});
+};
+
+findGroups();

migrations/20150906_groups_remove_empty.js

@@ -4,21 +4,17 @@
 
 var mongo = require('mongoskin');
 
-var dbserver = 'mongodb://url';
-var dbname = 'db';
+var dbserver = 'url';
+var dbname = 'name';
 
 var db = mongo.db(dbserver + '/' + dbname + '?auto_reconnect');
 var dbGroups = db.collection('groups');
 
 console.log('Begins work on db');
 
-dbGroups.remove({
-  members: {$size: 0},
-  $or: [
-    {type: 'party'},
-    {privacy: 'private'}
-  ]
-}, function(err, res){
+dbGroups.findEach({
+  memberCount: 0,
+}, {_id: 1}, function(err, res){
   if(err) throw err;
 
   console.log(res);