How to parse JSON on multiple pages?

Question

I’m trying to parse information about products in json format from an online store: at http://raredevice.net/products.json. The problem, however, is that the above URL is actually http://raredevice.net/products.json?page=1 i.e. only page 1 and the products information is laid out on multiple pages! Coincidentally, there are exactly 13 pages and if you enter http://raredevice.net/products.json?page=14 you will find that the products list is empty. Even if you enter any other number outside the range (1=< page <= 13) such as http://raredevice.net/products.json?page=100000 it would still be a valid URL except that the products list would still be zero.

I need a way to parse the JSON on these multiple URLs without having to know in advance how many pages the product lists are laid out on. As explained above, the only way to do that is to check the length of the products list on every page and exit if the length is equal to 0. I’m having problems with the exiting part. I’m not sure how to return from the function in

response.on('end', function() {} )

so that I return from

var request = http.get("http://raredevice.net/products.json?page=" + page, function(response) {} )

And

function getProducts(page) {}

So far, I’m setting a variable endOfPages to false in the beginning and change it to true once on a given page the list of products is 0. I call the getProduts(page) in a while loop and try to break out of it if endOfPages === true. However, the loop doesn’t seem to stop or possibly does but too late. This is probably because of the asynchronous, non-blocking feature of node.js since the loop keeps executing while the callback functions are still executing.

I’m sure there’s a better way to do all this and I’d appreciate some help :)

// import node.js's http module
var http = require("http");

// global variable
var endOfPage = false;


function printMessage(page) {
    var message = "on page " + page;
    console.log(message);
}
// Print out error messages
function printError(error) {
    console.error(error.message);
}


function getProducts(page) {

    // 1. Connect to the API URL page= 1 to n where n is not known in advance
    var request = http.get("http://raredevice.net/products.json?page=" + page, function(response) {

        var stringBody = "";

        // 2. Read the data
        response.on('data', function(chunk) {
            stringBody = stringBody + chunk;
        });

        response.on('end', function() { 

            if(response.statusCode === 200) {
                try {   
                    // 3. Parse the Json data
                    var jsonBody = JSON.parse(stringBody); 

                    var productsArray = jsonBody.products;

                    /* ******************************* */
                    // EXIT if reached end of pages
                    if (productsArray.length === 0) {
                        // HOUSTON, WE HAVE A PROBLEM !!!
                        // ABORT!
                        // BUT, BUT HOW??
                        // return!
                        // Can't return fast enough for caller to shop calling!
                        endOfPage = true;
                    }
                    /* ******************************* */

                    // 5. Print the data
                    printMessage(page);
                } 
                catch(error) {
                    // Parse Error
                    printError(error);
                }
            }
            else {
                // status Code Error 
                printError({message: "There was an error getting the jsonBody for page" + page + ": " + response.statusCode + " " + response.statusMessage});
            }
        });
        });

       // Connection Error
        request.on("error", printError); 
}



var numPage = 1;

while (true) {

    getProducts(numPage);

    // Never breaks out of loop !
    if (endOfPage === true) {
        break;
    }

    numPage++;
}

Welcome to the Treehouse Community

Looking to learn something new?

sh10

sh10

How to parse JSON on multiple pages?