Newbie question

[Peter van Gils]

Hi everyone,

I am trying to scrape a Dutch genealogy website. The end result should be that I start with one person and then keep on scraping search results until there is nothing left to find. Ok, that's Nirvana..

For now I have managed to get to the advanced search page, enter the lastname and firstname of my trial person (my grandfather, who is convenient because there are just 7 direct hits) and navigate to the results page. Not too bad so far.

The problem is that the results are in a table and I would like to copy the contents of the table to a file, then download the scans (link in the first table column) and follow the link in the second column) and I cannot even find a way to populate an array with the stupid table.

Here is my code so far, any help is greatly appreciated!

Thanks in advance, Cheers,

Peter

var x = require('casper').selectXPath;

var utils = require('utils');

var casper = require('casper').create({

    verbose: true,

    logLevel: "debug",

    viewPortsize: {width: 1024, height: 768}

});

casper.userAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X)');

var lastname = 'Walters';

var firstname ='Hermanus Adrianus';

//var count = 0;

//var extract = new array();

//var fields = 7;

casper.start('https://www.wiewaswie.nl/personen-zoeken/uitgebreid-zoeken/', function() {

this.capture('www1.png');

this.echo('screenshot taken before input: ' + this.getCurrentUrl());

// this.test.assertExists('form[action="/personen-zoeken/uitgebreid-zoeken/"]', 'main form is found');

// this.test.assertExists('#lastname', 'lastname is found');

// this.test.assertExists('#firstname', 'firstname is found');

// this.test.assertExists('#cphBody_cphBody_wucSearchAdvanced_btnSearch', 'button is found');

this.fill('form#formAdvancedSearch', {

        'ctl00$ctl00$ctl00$cphBody$cphBody$wucSearchAdvanced$lastname':    lastname,

        'ctl00$ctl00$ctl00$cphBody$cphBody$wucSearchAdvanced$firstname':   firstname,

        }, true);

    

    this.capture('www2.png');

    this.echo('screenshot taken after input: ' + this.getCurrentUrl());

casper.then(function() {

this.test.assertExists({

        type: 'xpath',

        path: '//*[@id="cphBody_cphBody_wucSearchAdvanced_btnSearch"]'

    }, 'Zoeken button exists');

});

casper.then(function() {

    this.click('input#cphBody_cphBody_wucSearchAdvanced_btnSearch.button');

    }); 

    

    this.wait(2000, function() {

        this.capture('www3.png');

        this.echo('screenshot after mouseclick: ' + this.getCurrentUrl());

       

    this.echo(this.fetchText('h1'));

    count = this.getHTML('span#spanCount');

    this.echo(count);

    });

    

//    });

casper.then(function() {

this.test.assertExists({

        type: 'xpath',

        path: '//*[@href="/personen-zoeken/zoeken/document/a2apersonid/182086986/srcid/7892706/oid/28"]'

    }, 'Zoeken a exists');

});

/html/body/div[2]/div[2]/div/form/div[3]/div/div[2]/table/tbody/tr[2]/td[3]                             

//casper.then(function() {

// for (var iter = 0 ; iter < count; iter++) {

// for (var inner = 0; inner < fields; inner++) {

// /html/body/div[2]/div[2]/div/form/div[3]/div/div[2]/table/tbody/tr[2]/td[2]/a

// extract[iter],[inner] 

// }

// }

//});

});    

       

casper.run();