Web Scraping with Node.js

var mongoose = require('mongoose');
var request = require("request");
var cheerio = require("cheerio");
var http = require('http');

//these arrays will store the scraped information from webpage
var prodList = [];
var priceList = [];

//this is the array that will be used to organize and display the scraped info
var products = [];

//store scraped data as an object 
function Prod(prodName, price) {
    this.prodName = prodName;
    this.price = price;
};



var populateArray = function() {

    //urls to scrape
    var nyxLinks = [
        "http://www.nyxcosmetics.ca/en_CA/face?sz=999&viewall=1",
        "http://www.nyxcosmetics.ca/en_CA/lips?sz=999&viewall=1",
        "http://www.nyxcosmetics.ca/en_CA/eyes?sz=999&viewall=1"
    ];

    //empty all arrays
    prodList  = [];
    priceList = [];
    products = [];

    for(var i = 0; i < nyxLinks.length; i++) {

        //define url to download 
        var url = nyxLinks[i];
        console.log(url);


        request(url, function(error, response, body) {
            if(!error) {

                //load page into cheerio
                var $ = cheerio.load(body);

                //for each product on the page store in respective arrays
                $(".product_tile_wrapper").each(function(i, elem) {
                    prodList.push($(this).find($(".product_name")).attr("title"));
                    priceList.push($(this).find($(".product_price")).attr("data-pricevalue"));
                });

                for(var i = 0; i < prodList.length; i++) {
                    //store product info as an object

                    products.push(new Prod(prodList[i], priceList[i]));
                }
            } else {
                console.log("We've encountered an error!")
            }
        }).on("end", function(err, data) {          
            if(!err) {
                console.log("products length " + products.length);
            } else {
                console.log(err);
            }

        });
    }
    console.log("products length " + products.length);

}



mongoose.connect('mongodb://127.0.0.1:27017/makeupdb');

var db = mongoose.connection;
db.on('error', console.error.bind(console, 'Connection Error:'));
db.once('open', function() {
    // we're connected

    populateArray();
    console.log("number of products in products array " + products.length);

    //clear the current collection - db.remove({})

    //insert data in mongodb - db.insert(products)

});

var hostname = '127.0.0.01';
var port = process.env.PORT || 1337;

http.createServer(function(request, response) {
    response.writeHead(200, {'Content-Type': 'text/plain'});
    response.end("Hello World");
}).listen(port, hostname, function() {
    console.log("Server running at http://" + hostname + ":" + port + "/");
});