Scraping with nightmare has been a breeze until recently , i started encountering errors with no details and the title "navigation error" and the error code 118 as shown below.
{ [Error: navigation error] '0': { message: 'navigation error', code: -118, details: '', url: 'http://markets.ft.com/research/Browse-Companies' }, length: 1, errors: [ { message: 'navigation error', code: -118, details: '', url: 'http://markets.ft.com/research/Browse-Companies' } ] }
My nightmare code (Node.Js) :
function *run(){ var nightmare = Nightmare({show : true }); nightmare.useragent("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36") var url = "http://markets.ft.com/research/Browse-Companies"; var industry = []; //fetching data in { INDUSTRY , LINK , SECTOR } format . yield nightmare.goto(url).inject('js' , './jquery-2.2.3.min.js').wait('#wsod').evaluate(function () { var arr = []; $('.simpleLinkList.wsodModuleContent li').each(function(){ arr.push({SECTOR : $(this).parents('ul').prev().text().replace('Sectors & Industries' , '') , INDUSTRY : $(this).text() , LINK : $(this).find('a').attr('href')}); }) return arr; }).then(function (data) { industry = data; }); //using {LINK} var companies = []; for(var i = 0 ; i<2; i++) { yield nightmare.goto(industry[i].LINK).inject('js' , './jquery-2.2.3.min.js').wait('#wsod'); var nextExists = yield nightmare.visible('.wsod-icon-paging-next-active'); var maxpage = 3; var currentpage = 1; var data = []; /* Object({ Name: "" , Link : ""})*/ while(nextExists && currentpage < maxpage) { //pagination / checking if next page exists and looping the scraper for each page yield nightmare.evaluate(function(a , b){ var obj = []; $('.company-link').each(function () { obj.push({Sector : a , Industry: b , Name: $(this).text() , Link: $(this).attr('href')}); }); return obj; },industry[i].SECTOR , industry[i].INDUSTRY).then(function (obj) { data.push(obj); }); yield nightmare.click('.wsod-icon-paging-next-active').wait(2000); currentpage++; nextExists = yield nightmare.visible('.wsod-icon-paging-next-active'); } //data is an array of arrays and needs to be flattened. var x = [].concat.apply([] , data); //now pushing data to companies list (entire container) companies.push(x); } companies = [].concat().apply([], companies); //now companies is an array of entire list of all companies in every single sector->industry with sector name included for ease console.log(companies);*/ console.log(companies); yield nightmare.end(); } If anyone could provide more info regarding this error , it would be great . The program works some times but most times i get the "Navigation-error"