trying to generate a decrypted/javascript process

[bruce]

Hi.

A target site/page uses javascript to generate decrypted page/content.

The page requires javascript, so there needs to be a headless solution to invoke the underlying javascript, and generate the content/html.

Without javascript disabled, the page is encoded. With javascript enabled in the browser, the url displays correctly.

The following test was created to basically, set up the headers, wait for the "selector" that is clearly apparent in the browser, and to then generate the page.. Should be pretty simple.

However, (not sure why) the test times out.. There's not way the page should take 20+ secs to generate. In the FF browser, the process takes a 1-2 secs.. So something is screwing up.

The casperjs version is 1.1/running on fedora.

To really keep things simple, the url, and cookie file name have been hardcoded.

Pointers to what might be happening would be great!


-------------------------------

/*
*
*    oak_bn.js:        test to take input url
*
*    casper.js oak_bn.js  --cfile="aa.lwp" --url="foo" 
*
*
*/
var casper = require('casper').create({
  verbose: true,
  logLevel: 'error',
  waitTimeout: 25000, // new maximum waitTimeout
  pageSettings: {
  loadImages: false,
  javascriptEnabled: true,
  userAgent: 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36'
  }
});

phantom.timeout=10000;
phantom.stepTimeout=10000;

var url = casper.cli.get("url");
var cfile = casper.cli.get("cfile");
//var pdata = casper.cli.get("post");
//var ref = casper.cli.get("ref");

url="https://securex.bncollege.com/webapp/wcs/stores/servlet/TBListView?catalogId=10001&storeId=13551&langId=-1&level=1&termMapping=N&courseXml=%3C%3Fxml%20version%3D%221.0%22%20encoding%3D%22UTF-8%22%3F%3E%3Ctextbookorder%3E%3Ccourses%3E%3Ccourse%20dept%3D%22ACC%22%20num%3D%22200%22%20sect%3D%2210136%22%20%20term%3D%22W16%22%2F%3E%3C%2Fcourses%3E%3C%2Ftextbookorder%3E"

var cookie_file = cfile;
var fs = require('fs')
//var data = fs.read(cookie_file);
var data = fs.read("aa.lwp");
phantom.cookies = JSON.parse(data);

var processPage = function() {
        this.echo(this.page.content);
};

casper.echo(url)
//casper.start(url);
casper.start().then(function() {
  this.open(url, {
    method : 'get',
    headers : {
      'Content-Type': 'application/json',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'en-US,en;q=0.5',
        'Connection': 'keep-alive',
        'Host': 'securex.bncollege.com'
    }
//    data: pdata
  });
});


 
//casper.waitForSelector('input[id="conversionId"]', processPage);
casper.waitForSelector("li[class='childSubEvntFooter']", processPage);

casper.run();