Saving a million documents in Mongo using Nodejs and mongodb module

/**
* Created by davidsaliba on 13/03/2017.
*/
var MongoClient = require('mongodb').MongoClient
, format = require('util').format;

var url = 'mongodb://localhost:27017/test';
var async = require ('async');

var entry = {
data : "skdlfjsdf",
array : [ {id:"arr_obj1"} , {id:"arr_obj2"} ]
};

var entries = [];

var total_entries = 1000000

for (var j = 0 ; j <= total_entries ; j ++){
console.log (j);
entries.push({"_id":j ,entry:entry});
}

console.log ("Number of entries", entries.length);

MongoClient.connect(url, function(err, db) {
// Get the collection
var col = db.collection('articles');
//

var bulk = col.initializeOrderedBulkOp();
var counter = 0;

async.whilst(
// Iterator condition
function() { return counter < total_entries },

// Do this in the iterator
function(callback) {
counter++;

bulk.insert(entries[counter] );

if ( counter % 1000 == 0 ) {
bulk.execute(function(err,result) {
bulk = col.initializeOrderedBulkOp();
callback(err);
});
} else {
callback();
}
},

// When all is done
function(err) {
if ( counter % 1000 != 0 )
bulk.execute(function(err,result) {
console.log( "inserted some more" );
});
console.log( "I'm finished now" );
db.close();
}
);
});

nJoy 😉

Leave a Reply

Your email address will not be published. Required fields are marked *