Inserting a big array of object in mongodb from nodejs

Question

I need to insert a big array of objects (about 1.5-2 millions) in mongodb from nodejs. How can i improve my inserting?

This is my code:

var sizeOfArray = arrayOfObjects.length; //sizeOfArray about 1.5-2 millions
for(var i = 0; i < sizeOfResult; ++i) {
  newKey = {
    field_1: result[i][1],
    field_2: result[i][2],
    field_3: result[i][3]
  };
  collection.insert(newKey, function(err, data) {
    if (err) {
      log.error('Error insert: ' + err);
    }
  });
}

Solution 1

You can use bulk inserts.

There are two types of bulk operations:

1. Ordered bulk operations. These operations execute all the operation in order and error out on the first write error.
2. Unordered bulk operations. These operations execute all the operations in parallel and aggregates up all the errors. Unordered bulk operations do not guarantee order of execution.

So you can do something like this:

var MongoClient = require('mongodb').MongoClient;

MongoClient.connect("mongodb://myserver:27017/test", function(err, db) {
    // Get the collection
    var col = db.collection('myColl');

    // Initialize the Ordered Batch
    // You can use initializeUnorderedBulkOp to initialize Unordered Batch
    var batch = col.initializeOrderedBulkOp();

    for (var i = 0; i < sizeOfResult; ++i) {
      var newKey = {
          field_1: result[i][1],
          field_2: result[i][2],
          field_3: result[i][3]
      };
      batch.insert(newKey);
    }

    // Execute the operations
    batch.execute(function(err, result) {
      console.dir(err);
      console.dir(result);
      db.close();
    });
});

Solution 2

As For Version > 3.2, insertMany has been introduced which uses bulkWrite under the hood for bulk insert purposes only.

• insertMany supports ordered and unordered inserts. Unordered being faster as mongo decides the ordering. Likewise your implementation for best through-put should be ::

     var sizeOfArray = arrayOfObjects.length;
     for(var i = 0; i < sizeOfResult; ++i) {
       newKey = {
        field_1: result[i][1],
        field_2: result[i][2],
        field_3: result[i][3]
       };
     }
     collection.insertMany(newKey, { ordered: false }).then((res) => {
     console.log("Number of records inserted: " + res.insertedCount);
     })