Saving a million documents in Mongo using Nodejs and mongodb module

/**
 * Created by davidsaliba on 13/03/2017.
 */
var MongoClient = require('mongodb').MongoClient
    , format = require('util').format;

var url = 'mongodb://localhost:27017/test';
var async = require ('async');

var entry = {
    data : "skdlfjsdf",
    array : [ {id:"arr_obj1"} , {id:"arr_obj2"} ]
};

var entries = [];

var total_entries = 1000000

for (var j = 0 ; j <= total_entries  ; j ++){
    console.log (j);
    entries.push({"_id":j ,entry:entry});
}


console.log ("Number of entries", entries.length);



MongoClient.connect(url, function(err, db) {
    // Get the collection
var col = db.collection('articles');
    //

var bulk = col.initializeOrderedBulkOp();
var counter = 0;

async.whilst(
    // Iterator condition
    function() { return counter < total_entries },

    // Do this in the iterator
    function(callback) {
        counter++;
        
        bulk.insert(entries[counter] );

        if ( counter % 1000 == 0 ) {
            bulk.execute(function(err,result) {
                bulk = col.initializeOrderedBulkOp();
                callback(err);
            });
        } else {
            callback();
        }
    },

    // When all is done
    function(err) {
        if ( counter % 1000 != 0 )
            bulk.execute(function(err,result) {
                console.log( "inserted some more" );
            });
        console.log( "I'm finished now" );
        db.close();
    }
);
});

nJoy 😉

Write a Reply or Comment

Your email address will not be published.