Storing Data From Multiple MongoDB Queries in NodeJS

Question

All right, I think it's easier if I just copy my whole node.js route so you can see what I'm talking about.

I'm trying to make multiple queries to my MongoDB database (one for each day). The queries are running fine, but by the time they fire their callbacks, my incrementor variable has already incremented.

So for this:

exports.graph = function(req, res) {

  function pad(num, size) {
    var s = num+"";
    while (s.length < size) s = "0" + s;
    return s;
  }

  database.collection('customerData', function(err, collection) {

    for (var i = 2; i < 18; i++) {

      sDay = pad(i, 2);
      eDay = pad(i + 1, 2);

      collection.find({ 
        DATE: { 
          $gte: sDay + 'APR13:00:00:00',
          $lt: eDay + 'APR13:00:00:00' 
        }
      }, function(err, cursor){
        cursor.toArray(function(err, data){
          var counter = 0;
          for (var point in data) {
            trans = parseInt(data[point].Total_Transaction * 100);
            counter += trans;
          }

          console.log(i, counter / 100);  
        });
      });
    }
  });
}   

I get an output of this:

18 22023.29
18 24483.03
18 22644.11
18 23194.31
18 21560.99
18 23024.32
18 24384.93
18 23138.34
18 24400.63
18 28418.6
18 31691.65
18 31111.62
18 42358.74
18 38355.76
18 36787.52
18 42870.19
18 22023.29
18 22644.11
18 24483.03
18 23194.31
18 21560.99
18 23024.32
18 24400.63
18 23138.34
18 24384.93
18 28418.6
18 31691.65
18 31111.62
18 42358.74
18 38355.76
18 36787.52
18 42870.19

This (obviously) isn't ideal- the numbers aren't necessarily coming back in the order the queries are being fired, so it's important to me to figure out which ones go where.

I can't figure out a way to store the results of my arithmetic in the query callback in a meaningful way so I can use it later.

Any thoughts?

Answer 1

On a side note, unlike your example, and my example below, you really should save the collection reference rather than reinstantiating it each time you need to use it.

As for your question, just wrap the logic of each call in its own function. That way you create a closure which remembers which day each specific query was for. Below I created an example where all of the results get put into an object, and then when all of the queries have completed, it outputs all of the results in order.

var _outstanding = 0,
    _results = {};

database.collection('customerData', function(err, collection) {
  for (var i = 2; i < 18; i++) {
    _outstanding++;
    getDay(collection, i, getDayCallback);
  }
});

function getDay (collection, day, callback) {
  sDay = pad(day, 2);
  eDay = pad(day + 1, 2);

  collection.find({ ...query obj... }).toArray(err, data) {
    var counter = 0;
    if (!err) {
      for (var point in data)
        counter += parseInt(data[point].Total_Transaction * 100);
    }

    callback(err, day, counter);
  });
}

function getDayCallback (err, day, count) {
  // ...actually handle any errors, of course...
  _results[day] = count;

  if (--_outstanding === 0) { //display the results in-order when done
    for (var i = 2; i < 18; i++)
      console.log(i, _results[i] / 100)
  }
}

In this contrived example, getDay and getDayCallback could easily be combined into one function, but in the real world you'll most likely want them kept separate. Also, not sure where 2 and 18 come from, but I'm guessing they shouldn't actually be hard coded multiple times like I did :).

Answer 2

@Bret's answer is 100% correct.

I thought to contribute a bit to show how you could transform your code with minimal modifications to include the closure. Since closures are extremely useful in situations like this (which occur often) if may help to see the pattern.

exports.graph = function(req, res) {

    function pad(num, size) {
        var s = num+"";
        while (s.length < size) s = "0" + s;
        return s;
    }

    database.collection('customerData', function(err, collection) {

        for (var i = 2; i < 18; i++) {

            sDay = pad(i, 2);
            eDay = pad(i + 1, 2);

            collection.find({ 
                DATE: { 
                    $gte: sDay + 'APR13:00:00:00',
                    $lt: eDay + 'APR13:00:00:00' 
                }
            }, (function(i){ //anonymous function which is sync-executed, bringing
                             // 'i' to closure-scope.
                return function(err, cursor){
                    cursor.toArray(function(err, data){
                        var counter = 0;
                        for (var point in data) {
                            trans = parseInt(data[point].Total_Transaction * 100);
                            counter += trans;
                        }

                        console.log(i, counter / 100);  
                    });
                });
            }(i))
        }
    });
}