How to download a file through a custom POST request with CasperJS

Question

I am writing a crawler and needs to download file generated after a form request using POST.

I have successfully used this.download(url,'POST',Params) for regular forms. One of the sites has many fields using the same name, thus preventing me from using the regular download method.

After trying a lot of things, I tried with $.ajax() and __utils.sendAJAX() to process the form like this:

response = this.evaluate(function(){
  url=...
  params = $('form#theirForm').serialize();
  data = __utils__.sendAJAX(url, 'POST', params,false,{contentType:"application/x-www-form-urlencoded"});
return __utils__.encode(data);
});
function decode_base64(s) { var e={},i,k,v=[],r='',w=String.fromCharCode; var n=[[65,91],[97,123],[48,58],[43,44],[47,48]]; for(z in n){for(i=n[z][0];i<n[z][1];i++){v.push(w(i));}} for(i=0;i<64;i++){e[v[i]]=i;} for(i=0;i<s.length;i+=72){ var b=0,c,x,l=0,o=s.substring(i,i+72); for(x=0;x<o.length;x++){ c=e[o.charAt(x)];b=(b<<6)+c;l+=6; while(l>=8){r+=w((b>>>(l-=8))%256);} } } return r; }
casper.then(function() {
    utils.dump(response);
    fs.write("test.zip",decode_base64(response),'w');
});

The codes returns me base64 data which I convert and store in a test.zip file. But I juste can't uncompress it, says it is corrupted. I dump the data of a correct zip file =>

PK^C^D^T^@^H^@^H^@<F4><89><96>F^@^@^@^@^@^@^@^@^@^@^@^@?^@^@^@fourniture denr<E9>es alimentaires - dietetique infantile\CCAP.pdf<AC><BC>^ET\K<D3><F7>;^D<B7><U+0B81>^@<C1><99>^Y^F'^D<B7><E0>^D^ON<90><E0><EE><EE><EE>^Dwww'^P<9C>^D^H<EE>^^܂<C3>%'<CF>9<E7><C9><F7><U+07B5><BE>7<F7>f^SVOzf

Compared it with the first line of my file :

PK^C^D^T^@^H^@^H^@)_^M^@^@^@^@^@^@^@^@^@^@^@^@^@b^@^@^@fourniture denr<FD>es alimentaires - dietetique infantile\Bordereau de prix dietetique infantile.xlsx<FD>zuT<FD>I<FD><FD><FD><FD>^^4hp^M^D^M^R^H<FD>.<FD><FD>}p<FD>3<FD>kpw<FD>@pw^M<FD><FD>^R4<FD>Gv<FD>~<FD>[<FD><FD><FD><FD><FD>

Anyone has an idea of what could have gone wrong?

I have tried so many things (encoding tools, encoding settings, dumping from the chrome console to get pure base64, etc.)

I don't understand why it is related to latin-1 or utf8 encoding, since a website asks me to select which encoding to use. Tried both.

Answer 1

casper.download() happily accepts a serialized form instead of an object, so you can still use it. You just have to serialize the form in the page context beforehand:

var formData = casper.evaluate(function(){
  return $('form#theirForm').serialize();
});

var url;
casper.download(url, targetFile, 'POST', params);

The only problem might be, that another mimeType is used: "text/plain; charset=x-user-defined".

In that case, you will have to recreate the whole cascade of functions that go into casper.download():

var url;
var response = casper.evaluate(function(url){
    var params = $('form#theirForm').serialize();
    var data = __utils__.sendAJAX(url, 'POST', params, false);
    return __utils__.encode(data);
}, url);

var cu = require('clientutils');

fs.write("test.zip", cu.decode(response), 'wb');

"application/x-www-form-urlencoded" is used by default for __utils__.sendAJAX().