Created
August 3, 2017 14:31
-
-
Save TalhaAwan/92cf2f4668d2a66f7bc4fd76388249ca to your computer and use it in GitHub Desktop.
Batch processing of user documents. Node, mongoose, async (whilst, eachLimit, waterfall), moment.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| var async = require("async"); | |
| var moment = require('moment'); | |
| var User = require('/path/to/user.js'); | |
| module.exports = function(){ | |
| var skip = 0; | |
| var limit = 700; //to process the users in batches of 700 | |
| var newUsersFound = true; | |
| var userCount = 0; | |
| async.whilst( | |
| function() { return newUsersFound}, | |
| function(callback) { | |
| var updatedAt = new Date(moment().subtract(7, 'days').format('YYYY-MM-DD')); //last week | |
| User.find({email: {$exists: true}, updatedAt: {$gt: updatedAt}}, function(err, users){ | |
| if(err){ | |
| return callback(err) | |
| } | |
| else{ | |
| if(!users.length){ | |
| newUsersFound = false; | |
| return callback(); | |
| } | |
| else{ | |
| userCount += users.length; | |
| skip += limit; | |
| processUserBatch(users, function(err){ | |
| if(err){ | |
| return callback(err); | |
| } | |
| else{ | |
| return callback(); | |
| } | |
| }) | |
| } | |
| } | |
| }) | |
| .limit(limit) | |
| .skip(skip) | |
| }, | |
| function (err) { | |
| if(err){ | |
| console.log(err) | |
| } | |
| else{ | |
| console.log("Users Processed: ", userCount) | |
| } | |
| } | |
| ); | |
| } | |
| function processUserBatch(users, done){ | |
| async.eachLimit(users, 20, function(user, callback){ | |
| processUser(user, function(err){ | |
| if(err){ | |
| return callback(err); | |
| } | |
| else{ | |
| return callback(); | |
| } | |
| }); | |
| }, function(err){ | |
| if(err){ | |
| return done(err); | |
| } | |
| else{ | |
| return done(); | |
| } | |
| }); | |
| } | |
| function processUser(user, done){ | |
| async.waterfall([ | |
| //dummy functions | |
| findSettings, | |
| gatherCyclingData, | |
| dispatchEmail | |
| ],function(err){ | |
| if(err){ | |
| return done(err); | |
| } | |
| else{ | |
| return done(); | |
| } | |
| }); | |
| function findSettings(callback){ | |
| // return callback(null, settings); | |
| // OR return callback(err); | |
| } | |
| function gatherCyclingData(settings, callback){ | |
| // return callback(null, cd); | |
| // OR return callback(err); | |
| } | |
| function dispatchEmail(cd, callback){ | |
| // return callback(); | |
| // OR return callback(err); | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment