How can I do pagination with Bluebird Promises?

Question

I have something like

new Promise (resolve, reject) ->
  trader.getTrades limit, skip, (err, trades) ->
    return reject err if err

    resolve trade         


        

Answer 1

First, lets conceal that ugly callback api:

var getTrades = Promise.promisify(trader.getTrades, trader);

Now, to traverse that pagination api we'll use a simple recursive descent:

function getAllTrades(limit, arr) {
    if (!arr) arr=[];
    return getTrades(limit, arr.length).then(function(results) {
         if (!results.length)
             return arr;
         else
             return getAllTrades(limit, arr.concat(results));
    });
}

^{Admittedly, concat is not super-fast as it creates a new array after each request, but this is the most elegant.}

This function will return a promise that resolves with a huge array of all results when all requests are made. This of course might not be what you want - maybe you want to show the first results immediately, and load more only lazily? Then a single promise is not the tool that you want, as this behaviour is more stream-like. It can however be written with promises nonetheless:

getTradeChunks = (limit, start = 0) ->
  getTrades limit, start
  .then (chunk) ->
    throw new Error("end of stream") if not chunk.length
    s = start+chunk.length
    [chunk, -> getTradeChunks limit, s]

rec = ([chunk, cont]) ->
  Promise.each chunk, doStuff
  .then -> waitForClick $ "#more"
  .then cont
  .then rec, err
end = (err) ->
  $ "#more"
  .text "no more trades"
getTradeChunks 15
.then rec, err

Answer 2

Here's my own solution to paging through promises: method page, as part of the spex library.

It also lets you throttle the processing and provide load balancing as needed.

Example

var promise = require('bluebird');
var spex = require('spex')(promise);

function source(index, data, delay) {
    // create and return an array/page of mixed values
    // dynamically, based on the index of the sequence;
    switch (index) {
        case 0:
            return [0, 1, promise.resolve(2)];
        case 1:
            return [3, 4, promise.resolve(5)];
        case 2:
            return [6, 7, promise.resolve(8)];
    }
    // returning nothing/undefined indicates the end of the sequence;
    // throwing an error will result in a reject;
}

function dest(idx, data, delay) {
    // data - resolved array data;
    console.log("LOG:", idx, data, delay);
}

spex.page(source, dest)
    .then(function (data) {
        console.log("DATA:", data); // print result;
    });

Output

LOG: 0 [ 0, 1, 2 ] undefined
LOG: 1 [ 3, 4, 5 ] 3
LOG: 2 [ 6, 7, 8 ] 0
DATA: { pages: 3, total: 9, duration: 7 }

Answer 3

You should be able to use promisify()/promisifyAll() to convert trader.getTrades() to an async version that returns a promise. Then, something like this should work well:

function getAllTrades(limit, offset, query) {

    var allTrades = [];

    function getTrades(limit, offset, query){
        return trader.getTradesAsync(limit, offset, query)
            .each(function(trade) {
                allTrades.push(trade)
                // or, doStuff(trade), etc.
            })
            .then(function(trades) {
                if (trades.length === limit) {
                    offset += limit;
                    return getTrades(limit, offset, query);
                } else {
                    return allTrades;
                }
            })
            .catch(function(e) {
                console.log(e.stack);
            })
    }

    return getTrades(limit, offset, query)
}

If you knew the total # of trades in advance you could use a different strategy with .map and {concurrency: N} to get N pages of trades at once.