Skip to content

Instantly share code, notes, and snippets.

@cschlyter

cschlyter/mongodbInserter.js

Created November 25, 2013 17:06
Show Gist options
  • Save cschlyter/7644738 to your computer and use it in GitHub Desktop.
Save cschlyter/7644738 to your computer and use it in GitHub Desktop.
Download ZIP
[Nodejs] [Mongodb] Batch insert controller for mongodb using nodejs. Taken from: http://stackoverflow.com/questions/12313915/node-jsloop-insert-1000000-documents-to-mongodb-via-native-driver-why-node-ta
Raw
mongodbInserter.js
var mongo = require('mongodb');
var Inserter = function (collection) {
this.collection = collection;
this.data = [];
this.maxThreads = 6;
this.currentThreads = 0;
this.batchSize = 5000;
this.queue = 0;
this.inserted = 0;
this.startTime = Date.now();
};
Inserter.prototype.add = function(data) {
this.data.push(data);
};
// Use force=true for last insert
Inserter.prototype.insert = function(force) {
var that = this;
if (this.data.length >= this.batchSize || force) {
if (this.currentThreads >= this.maxThreads) {
this.queue++;
return;
}
this.currentThreads++;
console.log('Threads: ' + this.currentThreads);
this.collection.insert(this.data.splice(0, this.batchSize), {safe:true}, function() {
that.inserted += that.batchSize;
var currentTime = Date.now();
var workTime = Math.round((currentTime - that.startTime) / 1000)
console.log('Speed: ' + that.inserted / workTime + ' per sec');
that.currentThreads--;
if (that.queue > 0) {
that.queue--;
that.insert();
}
});
}
};
var db = new mongo.Db('test', new mongo.Server('localhost', 27017, {}), {native_parser:false});
db.open(function(err, db) {
db.collection('test', function(err, collection) {
var inserter = new Inserter(collection);
setInterval(function() {
for (var i = 0; i < 5000; i++) {
inserter.add({test:'test'});
}
inserter.insert();
}, 0);
});
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment