nodejs parallel callback design pattern

Question

I\'m trying to find a good pattern to execute a bunch of parallel tasks.

Let me define some task to exemplify. Tasks a, b, c, d, e, f, g execute as

Answer 1

You should check out Step ( https://github.com/creationix/step ). It's just over a hundred lines of code, so you can read the whole thing if needed.

My preferred pattern looks something like this:

function doABunchOfCrazyAsyncStuff() {
  Step (
    function stepA() {
      a(arg1, arg2, arg3, this); // this is the callback, defined by Step
    }
    ,function stepB(err, data) {
      if(err) throw err; // causes error to percolate to the next step, all the way to the end.  same as calling "this(err, null); return;"
      b(data, arg2, arg3, this);
    }
    ,function stepC(err, data) {
      if(err) throw err;
      c(data, arg2, arg3, this);
    }
    ,function stepDEF(err, data) {
      if(err) throw err;
      d(data, this.parallel());
      e(data, this.parallel());
      f(data, this.parallel());
    }
    ,function stepGGG(err, dataD, dataE, dataF) {
      if(err) throw err;
      var combined = magick(dataD, dataE, dataF);
      var group = this.group();  // group() is how you get Step to merge multiple results into an array
      _.map(combined, function (element) {
        g(element, group()); 
      });
    }
    ,function stepPostprocess(err, results) {
      if(err) throw err;
      var processed = _.map(results, magick);
      return processed; // return is a convenient alternative to calling "this(null, result)"
    }
    ,cb // finally, the callback gets (err, result) from the previous function, and we are done
  );
}

Notes

• My example also uses the underscore library, "the tie to match JQuery's tux": http://documentcloud.github.com/underscore/
• Naming each step function stepXXXXX is a good habit so that the stack traces are clear and readable.
• Step lets you make powerful and elegant combinations of serial and parallel execution. These patterns are straightforward and comprehensible. If you need anything more complex like "when 3 of 5 of these methods completes, go to the next step", seriously RETHINK your design. do you really need such a complex pattern? (maybe you are waiting for a quorum set). Such a complex pattern deserves a function of it's own.