Why don't request() and cheerio work in a for loop?

[Shelby Matlock]

Hello,

I am extremely new to Javascript and I am trying to make a simple web scraper that will get vendor titles from a website (www.zinc.docking.org). The problem is that I cannot get request() to work inside a for loop (I still don't understand why I can't do this). One solution someone suggested was for me to use callback functions. I tried this but now I cannot get the bodies array to be returned from getZincBody(). I would like some help or insight as to why I cannot get this array returned.  

This is my code:

var request = require('request'),

cheerio = require('cheerio'),

http = require('http');

var numZinc = ["17", "200", "193"], // arbitrary; can be any number/zinc id

bodies = []; 

getZincUrls(numZinc);

function getZincUrls(arr){

   for(var x=0; x<arr.length; x++){

    getZincBody(x);

    // get information from the bodies (purchasable catalogs)

   }

}

function getZincBody(x) {

var url = 'http://zinc.docking.org/substance/'+numZinc[x];

return request(url, function(err, resp, body){

if (!err && resp.statusCode==200) {

         // return body for each url so body can be manipulated in getZincUrls()

         bodies.push(body) // does not work 

         console.log(body) // displays body properly

      }

      else{

      console.error("Something went wrong!");

      }

     });

return bodies;

}

Any help is very much appreciated. Thank you.

[Ryan Schmidt]

The someone you spoke with is correct: you need to use callbacks, and think asynchronously. This is a common problem for node and javascript newcomers to wrap their heads around.

In your getZincBody() function you are currently returning "bodies" when the function ends. This cannot work the way you want it to, because within getZincBody(), you are calling request(), which is asynchronous: it accepts a callback function which it will call when it is done. In your anonymous callback function you are pushing the body onto the bodies array, but your "return bodies" statement executes before the anonymous callback function has had a chance to run.