Node.JS – Working with Request Library A Simple HTTP Client

NodeJS has a very usefully HTTP client library called request. It like an alternate of cURL in PHP. In this blog post we will see how to use this library and different case scenarios

The library is available at https://github.com/request/request and can be easily installed by “npm install request”.

Let start looking in various cases where this library can be used

Parsing a URL using NodeJS Request

In this example we will simple see how to parse a url or getting its source though the request library.

var request = require('request');
var zlib = require('zlib');

function getHTML(url, callback) {

    var headers = {};
    headers['user-agent'] = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7';

    var options = {
        url: url,
        headers: headers
    };
    var req = request.get(options);

    req.on('response', function (res) {
        var chunks = [];
        res.on('data', function (chunk) {
            chunks.push(chunk);
        });
        res.on('end', function () {
            var buffer = Buffer.concat(chunks);
            var encoding = res.headers['content-encoding'];
            if (encoding == 'gzip') {
                zlib.gunzip(buffer, function (err, decoded) {
                    callback(err, decoded && decoded.toString());
                });
            } else if (encoding == 'deflate') {
                zlib.inflate(buffer, function (err, decoded) {
                    callback(err, decoded && decoded.toString());
                })
            } else {
                callback(null, buffer.toString());
            }
        });
    });
    req.on('error', function (err) {
        callback(err);
    });
}

getHTML('http://flipkart.com', function (err, html) {
    if (err) {
        console.log(err);
        process.exit();
    }
    console.log(html);
});

In the above code, I have use zlib library in case the html recieved is compressed using gzip or deflate as in many cases. This is most simple use case of request library to parse a url.

POST or Form Submit Using Request

In the below code will see how to post a form using nodejs request library

var request = require('request');
var zlib = require('zlib');

function postForm(url, param, callback) {

    var headers = {};
    headers['user-agent'] = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7';

    var options = {
        url: url,
        headers: headers
    };
    headers['X-Requested-With'] = 'XMLHttpRequest'; //if ajax request
    //headers['Referer'] = ''; //if you need to set referer header
    headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8';
    var j = request.jar()
    var options = {
        url: url,
        headers: this.headers,
        body: param,
        json: true,
        jar : j //for cookies
    };
    var req = request.post(options);

    req.on('response', function (res) {
        var chunks = [];
        res.on('data', function (chunk) {
            chunks.push(chunk);
        });
        res.on('end', function () {
            var buffer = Buffer.concat(chunks);
            var encoding = res.headers['content-encoding'];
            if (encoding == 'gzip') {
                zlib.gunzip(buffer, function (err, decoded) {
                    callback(err, decoded && decoded.toString());
                });
            } else if (encoding == 'deflate') {
                zlib.inflate(buffer, function (err, decoded) {
                    callback(err, decoded && decoded.toString());
                })
            } else {
                callback(null, buffer.toString());
            }
        });
    });
    req.on('error', function (err) {
        callback(err);
    });
}

postForm('https://www.flipkart.com/account/loginIframe', {
    'email': 'test@test.com',
    'password': 'test1234',
    'password_hash' : '======',
}, function (err, html) {
    if (err) {
        console.log(err);
        process.exit();
    }
    console.log(html);
});

Send Request Payload

Many websites these days use Request Payload technique with their ajax request. Here is how it looks in network tab of chrome dev tools.

To automate such a request, here is the code.

var request = require('request');
var zlib = require('zlib');

function postForm(url, param, callback) {

    var headers = {};
    var options = {
        url: url,
        headers: headers
    };
    headers['X-Requested-With'] = 'XMLHttpRequest'; //if ajax request
    headers['Content-Type'] = 'application/json; charset=utf-8';
    var options = {
        url: url,
        headers: headers,
        body: JSON.stringify(param)
    };
    var req = request.post(options);

    req.on('response', function (res) {
        var chunks = [];
        res.on('data', function (chunk) {
            chunks.push(chunk);
        });
        res.on('end', function () {
            var buffer = Buffer.concat(chunks);
            var encoding = res.headers['content-encoding'];
            if (encoding == 'gzip') {
                zlib.gunzip(buffer, function (err, decoded) {
                    callback(err, decoded && decoded.toString());
                });
            } else if (encoding == 'deflate') {
                zlib.inflate(buffer, function (err, decoded) {
                    callback(err, decoded && decoded.toString());
                })
            } else {
                callback(null, buffer.toString());
            }
        });
    });
    req.on('error', function (err) {
        callback(err);
    });
}

Basic HTTP Authorization with Request

Here is the code automated http authorization

var request = require('request');
var zlib = require('zlib');

function postForm(url, param, callback) {

    var headers = {};
    headers['Cache-Control'] = 'no-cache';
    headers['Authorization'] = 'Basic ' + new Buffer('user:password').toString('base64')
    var options = {
        url: url,
        headers: headers
    };
    var req = request.get(options);

    req.on('response', function (res) {
        var chunks = [];
        res.on('data', function (chunk) {
            chunks.push(chunk);
        });
        res.on('end', function () {
            var buffer = Buffer.concat(chunks);
            var encoding = res.headers['content-encoding'];
            if (encoding == 'gzip') {
                zlib.gunzip(buffer, function (err, decoded) {
                    callback(err, decoded && decoded.toString());
                });
            } else if (encoding == 'deflate') {
                zlib.inflate(buffer, function (err, decoded) {
                    callback(err, decoded && decoded.toString());
                })
            } else {
                callback(null, buffer.toString());
            }
        });
    });
    req.on('error', function (err) {
        callback(err);
    });
}