CasperJS - How to open up all links in an array of links

问题

I'm trying to make it so that CasperJS will open up every link in an array of links. I have it so that after I open a link, it will display the title of that page. Yet when I run it, nothing is displayed.

I can use a for loop to display the links and it works perfectly.

This is the code for what I just explained:

var x;

casper.start(URL, function() {

    x = links.split(" "); // now x is an array of links

    for (var i = 0; j < x.length; i++) // for every link...
    {
        casper.thenOpen(partialURL + x[i], function() { // open that link
            console.log(this.getTitle() + '\n'); // display the title of page
        });
    }

    this.exit();
});

casper.run();

This is another method I tried:

var x;

casper.start(URL, function() {
    x = links.split(" "); // now x is an array of links
    this.exit();
});

for (var i = 0; j < x.length; i++) // for every link...
{
    casper.thenOpen(partialURL + x[i], function() { // open that link
        console.log(this.getTitle() + '\n'); // display the title of page
    });
}

casper.run();

It says that 'x' in undefined. Notice that I set x to be a global variable though. Any modifications that you could make would be great. Thanks.

回答1:

var x; var i = -1;

casper.start(URL, function() {
    x = links.split(" "); // now x is an array of links
});

casper.then(function() {
    this.each(x, function() { 
        i++; // change the link being opened (has to be here specifically)
        this.thenOpen((partialURL + x[i]), function() {
            this.echo(this.getTitle()); // display the title of page
        });
    });
});

casper.run();

回答2:

var i = 0;
var nTimes = x.length;

casper.repeat(nTimes, function() {
    //... do your stuff
    i++;
});

worked for me.

回答3:

casper.start('about:blank');

var urls = ['http://google.fr', 'http://yahoo.fr', 'http://amazon.fr'];

casper.each(urls, function(casper, url) {
  casper.thenOpen(url, function() {
        this.echo("I'm in your " + url + ".");
    });
});

回答4:

In my case, I had to scrape a site that had an unknown number of pages. Each page (except the last) had a <a class="next-page" href="/page/N">Next page</a> link (where N is the page number). There was no way for the scraper to know when it was finished except when the "Next Page" link was no longer present.

Of course you'll have to make adjustments depending on what type of pagination links might exist on your page.

Here's what I did. Ymmv.

// imports
var fs = require('fs');

// scraper state
var state = {page: 1, data: []};

// casper
var casper = require("casper").create();

// scraper function
function scrape() {
  this.echo('Scraping page ' + state.page + '...', 'INFO');

  state.data = state.data.concat(this.evaluate(function() {
    // get some stuff from the page
    return someData;
  });

  var nextUrl = this.evaluate(function() {
    var nextLink = document.querySelector("a.next-page");
    return nextLink && nextLink.href;
  });

  if (nextUrl) {
    state.page = state.page + 1;
    casper.thenOpen(nextUrl, scrape); // <- recursion
  }
});

// run
casper.run(function() {
  fs.write('./data.json', JSON.stringify(state.data, null, '\t'), 'w');
  this.echo('Done!', 'INFO');
});

Hope this helps someone. If you have other questions, I'll be happy to try to help.

回答5:

casper.start();
casper.each(Object.keys(array), function(casper, array_elem) {
    this.thenOpen(partialURL+array[attay_item], function() {
        ...
};

And as to "undefined" error. Try not to use this too much. I experience this error with CasperJS to often, so I prefer to write casper instead of this.

回答6:

Try something like this.

var x;

casper.start(URL, function() {
    x = links.split(" "); // now x is an array of links
});

casper.then(function() {
    this.eachThen(x, function(response) {
        this.thenOpen((partialURL + response.data), function() {
            this.echo(this.getTitle()); // display the title of page
        });
    });
});

casper.run();

x was undefined because the for loop was being executed before casper.start. In the above code, the eachThen() block is nested inside of a casper.then block in order to delay its execution.

回答7:

I have solved the same issue with this code:

casper.then(function () {
    var i = -1;
    this.eachThen(locations, function () {
        i++;
        //Do stuff here like for example:
        this.thenOpen(YOUR_URL, function () {
            this.waitForSelector("MYSELECTOR", 
            function () {

            },                
            function () {

            })
        });
    })
});

来源：https://stackoverflow.com/questions/17926532/casperjs-how-to-open-up-all-links-in-an-array-of-links