Asynchronous for cycle in JavaScript

Question

I need a loop that waits for an async call before continuing. Something like:

for ( /* ... */ ) {

  someFunction(param1, praram2, function(result) {

    //         


        

Answer 1

I have been using the "setTimeout(Func,0);" trick for about year. Here is some recent research i wrote up to explain how to speed it up a bit. If you just want the answer, skip to Step 4. Step 1 2 and 3 explain the reasoning and mechanics;

// In Depth Analysis of the setTimeout(Func,0) trick.

//////// setTimeout(Func,0) Step 1 ////////////
// setTimeout and setInterval impose a minimum 
// time limit of about 2 to 10 milliseconds.

  console.log("start");
  var workCounter=0;
  var WorkHard = function()
  {
    if(workCounter>=2000) {console.log("done"); return;}
    workCounter++;
    setTimeout(WorkHard,0);
  };

// this take about 9 seconds
// that works out to be about 4.5ms per iteration
// Now there is a subtle rule here that you can tweak
// This minimum is counted from the time the setTimeout was executed.
// THEREFORE:

  console.log("start");
  var workCounter=0;
  var WorkHard = function()
  {
    if(workCounter>=2000) {console.log("done"); return;}
    setTimeout(WorkHard,0);
    workCounter++;
  };

// This code is slightly faster because we register the setTimeout
// a line of code earlier. Actually, the speed difference is immesurable 
// in this case, but the concept is true. Step 2 shows a measurable example.
///////////////////////////////////////////////


//////// setTimeout(Func,0) Step 2 ////////////
// Here is a measurable example of the concept covered in Step 1.

  var StartWork = function()
  {
    console.log("start");
    var startTime = new Date();
    var workCounter=0;
    var sum=0;
    var WorkHard = function()
    {
      if(workCounter>=2000) 
      {
        var ms = (new Date()).getTime() - startTime.getTime();
        console.log("done: sum=" + sum + " time=" + ms + "ms"); 
        return;
      }
      for(var i=0; i<1500000; i++) {sum++;}
      workCounter++;
      setTimeout(WorkHard,0);
    };
    WorkHard();
  };

// This adds some difficulty to the work instead of just incrementing a number
// This prints "done: sum=3000000000 time=18809ms".
// So it took 18.8 seconds.

  var StartWork = function()
  {
    console.log("start");
    var startTime = new Date();
    var workCounter=0;
    var sum=0;
    var WorkHard = function()
    {
      if(workCounter>=2000) 
      {
        var ms = (new Date()).getTime() - startTime.getTime();
        console.log("done: sum=" + sum + " time=" + ms + "ms"); 
        return;
      }
      setTimeout(WorkHard,0);
      for(var i=0; i<1500000; i++) {sum++;}
      workCounter++;
    };
    WorkHard();
  };

// Now, as we planned, we move the setTimeout to before the difficult part
// This prints: "done: sum=3000000000 time=12680ms"
// So it took 12.6 seconds. With a little math, (18.8-12.6)/2000 = 3.1ms
// We have effectively shaved off 3.1ms of the original 4.5ms of dead time.
// Assuming some of that time may be attributed to function calls and variable 
// instantiations, we have eliminated the wait time imposed by setTimeout.

// LESSON LEARNED: If you want to use the setTimeout(Func,0) trick with high 
// performance in mind, make sure your function takes more than 4.5ms, and set 
// the next timeout at the start of your function, instead of the end.
///////////////////////////////////////////////


//////// setTimeout(Func,0) Step 3 ////////////
// The results of Step 2 are very educational, but it doesn't really tell us how to apply the
// concept to the real world.  Step 2 says "make sure your function takes more than 4.5ms".
// No one makes functions that take 4.5ms. Functions either take a few microseconds, 
// or several seconds, or several minutes. This magic 4.5ms is unattainable.

// To solve the problem, we introduce the concept of "Burn Time".
// Lets assume that you can break up your difficult function into pieces that take 
// a few milliseconds or less to complete. Then the concept of Burn Time says, 
// "crunch several of the individual pieces until we reach 4.5ms, then exit"

// Step 1 shows a function that is asyncronous, but takes 9 seconds to run. In reality
// we could have easilly incremented workCounter 2000 times in under a millisecond.
// So, duh, that should not be made asyncronous, its horrible. But what if you don't know
// how many times you need to increment the number, maybe you need to run the loop 20 times,
// maybe you need to run the loop 2 billion times.

  console.log("start");
  var startTime = new Date();
  var workCounter=0;
  for(var i=0; i<2000000000; i++) // 2 billion
  {
    workCounter++;
  }
  var ms = (new Date()).getTime() - startTime.getTime();
  console.log("done: workCounter=" + workCounter + " time=" + ms + "ms"); 

// prints: "done: workCounter=2000000000 time=7214ms"
// So it took 7.2 seconds. Can we break this up into smaller pieces? Yes.
// I know, this is a retarded example, bear with me.

  console.log("start");
  var startTime = new Date();
  var workCounter=0;
  var each = function()
  {
    workCounter++;
  };
  for(var i=0; i<20000000; i++) // 20 million
  {
    each();
  }
  var ms = (new Date()).getTime() - startTime.getTime();
  console.log("done: workCounter=" + workCounter + " time=" + ms + "ms"); 

// The easiest way is to break it up into 2 billion smaller pieces, each of which take 
// only several picoseconds to run. Ok, actually, I am reducing the number from 2 billion
// to 20 million (100x less).  Just adding a function call increases the complexity of the loop
// 100 fold. Good lesson for some other topic.
// prints: "done: workCounter=20000000 time=7648ms"
// So it took 7.6 seconds, thats a good starting point.
// Now, lets sprinkle in the async part with the burn concept

  console.log("start");
  var startTime = new Date();
  var workCounter=0;
  var index=0;
  var end = 20000000;
  var each = function()
  {
    workCounter++;
  };
  var Work = function()
  {
    var burnTimeout = new Date();
    burnTimeout.setTime(burnTimeout.getTime() + 4.5); // burnTimeout set to 4.5ms in the future
    while((new Date()) < burnTimeout)
    {
      if(index>=end) 
      {
        var ms = (new Date()).getTime() - startTime.getTime();
        console.log("done: workCounter=" + workCounter + " time=" + ms + "ms"); 
        return;
      }
      each();
      index++;
    }
    setTimeout(Work,0);
  };

// prints "done: workCounter=20000000 time=107119ms"
// Sweet Jesus, I increased my 7.6 second function to 107.1 seconds.
// But it does prevent the browser from locking up, So i guess thats a plus.
// Again, the actual objective here is just to increment workCounter, so the overhead of all
// the async garbage is huge in comparison. 
// Anyway, Lets start by taking advice from Step 2 and move the setTimeout above the hard part. 

  console.log("start");
  var startTime = new Date();
  var workCounter=0;
  var index=0;
  var end = 20000000;
  var each = function()
  {
    workCounter++;
  };
  var Work = function()
  {
    if(index>=end) {return;}
    setTimeout(Work,0);
    var burnTimeout = new Date();
    burnTimeout.setTime(burnTimeout.getTime() + 4.5); // burnTimeout set to 4.5ms in the future
    while((new Date()) < burnTimeout)
    {
      if(index>=end) 
      {
        var ms = (new Date()).getTime() - startTime.getTime();
        console.log("done: workCounter=" + workCounter + " time=" + ms + "ms"); 
        return;
      }
      each();
      index++;
    }
  };

// This means we also have to check index right away because the last iteration will have nothing to do
// prints "done: workCounter=20000000 time=52892ms"  
// So, it took 52.8 seconds. Improvement, but way slower than the native 7.6 seconds.
// The Burn Time is the number you tweak to get a nice balance between native loop speed
// and browser responsiveness. Lets change it from 4.5ms to 50ms, because we don't really need faster
// than 50ms gui response.

  console.log("start");
  var startTime = new Date();
  var workCounter=0;
  var index=0;
  var end = 20000000;
  var each = function()
  {
    workCounter++;
  };
  var Work = function()
  {
    if(index>=end) {return;}
    setTimeout(Work,0);
    var burnTimeout = new Date();
    burnTimeout.setTime(burnTimeout.getTime() + 50); // burnTimeout set to 50ms in the future
    while((new Date()) < burnTimeout)
    {
      if(index>=end) 
      {
        var ms = (new Date()).getTime() - startTime.getTime();
        console.log("done: workCounter=" + workCounter + " time=" + ms + "ms"); 
        return;
      }
      each();
      index++;
    }
  };

// prints "done: workCounter=20000000 time=52272ms"
// So it took 52.2 seconds. No real improvement here which proves that the imposed limits of setTimeout
// have been eliminated as long as the burn time is anything over 4.5ms
///////////////////////////////////////////////


//////// setTimeout(Func,0) Step 4 ////////////
// The performance numbers from Step 3 seem pretty grim, but GUI responsiveness is often worth it.
// Here is a short library that embodies these concepts and gives a descent interface.

  var WilkesAsyncBurn = function()
  {
    var Now = function() {return (new Date());};
    var CreateFutureDate = function(milliseconds)
    {
      var t = Now();
      t.setTime(t.getTime() + milliseconds);
      return t;
    };
    var For = function(start, end, eachCallback, finalCallback, msBurnTime)
    {
      var i = start;
      var Each = function()
      {
        if(i==-1) {return;} //always does one last each with nothing to do
        setTimeout(Each,0);
        var burnTimeout = CreateFutureDate(msBurnTime);
        while(Now() < burnTimeout)
        {
          if(i>=end) {i=-1; finalCallback(); return;}
          eachCallback(i);
          i++;
        }
      };
      Each();
    };
    var ForEach = function(array, eachCallback, finalCallback, msBurnTime)
    {
      var i = 0;
      var len = array.length;
      var Each = function()
      {
        if(i==-1) {return;}
        setTimeout(Each,0);
        var burnTimeout = CreateFutureDate(msBurnTime);
        while(Now() < burnTimeout)
        {
          if(i>=len) {i=-1; finalCallback(array); return;}
          eachCallback(i, array[i]);
          i++;
        }
      };
      Each();
    };

    var pub = {};
    pub.For = For;          //eachCallback(index); finalCallback();
    pub.ForEach = ForEach;  //eachCallback(index,value); finalCallback(array);
    WilkesAsyncBurn = pub;
  };

///////////////////////////////////////////////


//////// setTimeout(Func,0) Step 5 ////////////
// Here is an examples of how to use the library from Step 4.

  WilkesAsyncBurn(); // Init the library
  console.log("start");
  var startTime = new Date();
  var workCounter=0;
  var FuncEach = function()
  {
    if(workCounter%1000==0)
    {
      var s = "
";
      var div = jQuery("*[class~=r1]");
      div.append(s);
    }
    workCounter++;
  };
  var FuncFinal = function()
  {
    var ms = (new Date()).getTime() - startTime.getTime();
    console.log("done: workCounter=" + workCounter + " time=" + ms + "ms"); 
  };
  WilkesAsyncBurn.For(0,2000000,FuncEach,FuncFinal,50);

// prints: "done: workCounter=20000000 time=149303ms"
// Also appends a few thousand divs to the html page, about 20 at a time.
// The browser is responsive the entire time, mission accomplished

// LESSON LEARNED: If your code pieces are super tiny, like incrementing a number, or walking through 
// an array summing the numbers, then just putting it in an "each" function is going to kill you. 
// You can still use the concept here, but your "each" function should also have a for loop in it 
// where you burn a few hundred items manually.  
///////////////////////////////////////////////