From 2b2e1d4b9aa2ef18bc091cf54716e9b9b88f557e Mon Sep 17 00:00:00 2001 From: Matteo Pagliazzi Date: Fri, 9 Nov 2018 12:58:39 +0100 Subject: [PATCH 1/4] rewrite mongoose migration to avoid using recursion --- .../users/20181023_veteran_pet_ladder.js | 88 ++++++------------- 1 file changed, 28 insertions(+), 60 deletions(-) diff --git a/migrations/users/20181023_veteran_pet_ladder.js b/migrations/users/20181023_veteran_pet_ladder.js index cbe8044d53..265808d5aa 100644 --- a/migrations/users/20181023_veteran_pet_ladder.js +++ b/migrations/users/20181023_veteran_pet_ladder.js @@ -2,52 +2,10 @@ const MIGRATION_NAME = '20181023_veteran_pet_ladder'; import { model as User } from '../../website/server/models/user'; -function processUsers (lastId) { - let query = { - migration: {$ne: MIGRATION_NAME}, - 'flags.verifiedUsername': true, - }; - - const fields = { - 'items.pets': 1, - }; - - if (lastId) { - query._id = { - $gt: lastId, - }; - } - - return User.find(query) - .limit(250) - .sort({_id: 1}) - .select(fields) - .exec() - .then(updateUsers) - .catch((err) => { - console.log(err); - return exiting(1, `ERROR! ${err}`); - }); -} - const progressCount = 1000; let count = 0; -function updateUsers (users) { - if (!users || users.length === 0) { - console.warn('All appropriate users found and modified.'); - displayData(); - return; - } - - let userPromises = users.map(updateUser); - let lastUser = users[users.length - 1]; - - return Promise.all(userPromises) - .then(() => { - return processUsers(lastUser._id); - }); -} +const batchSize = 250; function updateUser (user) { count++; @@ -71,24 +29,34 @@ function updateUser (user) { return user.save(); } -function displayData () { - console.warn(`\n${count} users processed\n`); - return exiting(0); -} +module.exports = async function processUsers () { + let query = { + migration: {$ne: MIGRATION_NAME}, + 'flags.verifiedUsername': true, + }; -function exiting (code, msg) { - code = code || 0; // 0 = success - if (code && !msg) { - msg = 'ERROR!'; - } - if (msg) { - if (code) { - console.error(msg); + const fields = { + items: 1, + }; + + while (true) { // eslint-disable-line no-constant-condition + const users = await User // eslint-disable-line no-await-in-loop + .find(query) + .limit(batchSize) + .sort({_id: 1}) + .select(fields) + .exec(); + + if (users.length === 0) { + console.warn('All appropriate users found and modified.'); + console.warn(`\n${count} users processed\n`); + break; } else { - console.log(msg); + query._id = { + $gt: users[users.length - 1], + }; } - } - process.exit(code); -} -module.exports = processUsers; + await Promise.all(users.map(updateUser)); // eslint-disable-line no-await-in-loop + } +}; From 39a35f44ef94f1a28b2adac90b55e308b3a746cf Mon Sep 17 00:00:00 2001 From: Matteo Pagliazzi Date: Fri, 9 Nov 2018 13:01:19 +0100 Subject: [PATCH 2/4] fixes --- migrations/users/20181023_veteran_pet_ladder.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/migrations/users/20181023_veteran_pet_ladder.js b/migrations/users/20181023_veteran_pet_ladder.js index 265808d5aa..4f2013df5d 100644 --- a/migrations/users/20181023_veteran_pet_ladder.js +++ b/migrations/users/20181023_veteran_pet_ladder.js @@ -5,9 +5,7 @@ import { model as User } from '../../website/server/models/user'; const progressCount = 1000; let count = 0; -const batchSize = 250; - -function updateUser (user) { +async function updateUser (user) { count++; user.migration = MIGRATION_NAME; @@ -26,7 +24,7 @@ function updateUser (user) { if (count % progressCount === 0) console.warn(`${count} ${user._id}`); - return user.save(); + return await user.save(); } module.exports = async function processUsers () { @@ -42,7 +40,7 @@ module.exports = async function processUsers () { while (true) { // eslint-disable-line no-constant-condition const users = await User // eslint-disable-line no-await-in-loop .find(query) - .limit(batchSize) + .limit(250) .sort({_id: 1}) .select(fields) .exec(); From 808885425f4f908fb712d59b604d4273741d4844 Mon Sep 17 00:00:00 2001 From: Matteo Pagliazzi Date: Fri, 9 Nov 2018 13:04:25 +0100 Subject: [PATCH 3/4] select more fields --- migrations/users/20181023_veteran_pet_ladder.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/migrations/users/20181023_veteran_pet_ladder.js b/migrations/users/20181023_veteran_pet_ladder.js index 4f2013df5d..c70cd028c7 100644 --- a/migrations/users/20181023_veteran_pet_ladder.js +++ b/migrations/users/20181023_veteran_pet_ladder.js @@ -34,7 +34,10 @@ module.exports = async function processUsers () { }; const fields = { + _id: 1, items: 1, + migration: 1, + flags: 1, }; while (true) { // eslint-disable-line no-constant-condition From ce03f837c78929a81b0a3b958eb64759581d887d Mon Sep 17 00:00:00 2001 From: Matteo Pagliazzi Date: Fri, 9 Nov 2018 13:10:55 +0100 Subject: [PATCH 4/4] use lean and .update --- migrations/users/20181023_veteran_pet_ladder.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/migrations/users/20181023_veteran_pet_ladder.js b/migrations/users/20181023_veteran_pet_ladder.js index c70cd028c7..4f33820bba 100644 --- a/migrations/users/20181023_veteran_pet_ladder.js +++ b/migrations/users/20181023_veteran_pet_ladder.js @@ -8,23 +8,25 @@ let count = 0; async function updateUser (user) { count++; - user.migration = MIGRATION_NAME; + const set = {}; + + set.migration = MIGRATION_NAME; if (user.items.pets['Bear-Veteran']) { - user.items.pets['Fox-Veteran'] = 5; + set['items.pets.Fox-Veteran'] = 5; } else if (user.items.pets['Lion-Veteran']) { - user.items.pets['Bear-Veteran'] = 5; + set['items.pets.Bear-Veteran'] = 5; } else if (user.items.pets['Tiger-Veteran']) { - user.items.pets['Lion-Veteran'] = 5; + set['items.pets.Lion-Veteran'] = 5; } else if (user.items.pets['Wolf-Veteran']) { - user.items.pets['Tiger-Veteran'] = 5; + set['items.pets.Tiger-Veteran'] = 5; } else { - user.items.pets['Wolf-Veteran'] = 5; + set['items.pets.Wolf-Veteran'] = 5; } if (count % progressCount === 0) console.warn(`${count} ${user._id}`); - return await user.save(); + return await User.update({_id: user._id}, {$set: set}).exec(); } module.exports = async function processUsers () { @@ -46,6 +48,7 @@ module.exports = async function processUsers () { .limit(250) .sort({_id: 1}) .select(fields) + .lean() .exec(); if (users.length === 0) {