Reading content from URL with Node.js

Question

I\'m trying to read the content from a URL with Node.js but all I seem to get are a bunch of bytes. I\'m obviously doing something wrong but I\'m not sure what. This is the

Answer 1

HTTP and HTTPS:

const getScript = (url) => {
    return new Promise((resolve, reject) => {
        const http      = require('http'),
              https     = require('https');

        let client = http;

        if (url.toString().indexOf("https") === 0) {
            client = https;
        }

        client.get(url, (resp) => {
            let data = '';

            // A chunk of data has been recieved.
            resp.on('data', (chunk) => {
                data += chunk;
            });

            // The whole response has been received. Print out the result.
            resp.on('end', () => {
                resolve(data);
            });

        }).on("error", (err) => {
            reject(err);
        });
    });
};

(async (url) => {
    console.log(await getScript(url));
})('https://sidanmor.com/');

Answer 2

the data object is a buffer of bytes. Simply call .toString() to get human-readable code:

console.log( data.toString() );

reference: Node.js buffers

Answer 3

A slightly modified version of @sidanmor 's code. The main point is, not every webpage is purely ASCII, user should be able to handle the decoding manually (even encode into base64)

function httpGet(url) {
  return new Promise((resolve, reject) => {
    const http = require('http'),
      https = require('https');

    let client = http;

    if (url.toString().indexOf("https") === 0) {
      client = https;
    }

    client.get(url, (resp) => {
      let chunks = [];

      // A chunk of data has been recieved.
      resp.on('data', (chunk) => {
        chunks.push(chunk);
      });

      // The whole response has been received. Print out the result.
      resp.on('end', () => {
        resolve(Buffer.concat(chunks));
      });

    }).on("error", (err) => {
      reject(err);
    });
  });
}

(async(url) => {
  var buf = await httpGet(url);
  console.log(buf.toString('utf-8'));
})('https://httpbin.org/headers');

Answer 4

try using the on error event of the client to find the issue.

var http = require('http');

var options = {
    host: 'google.com',
    path: '/'
}
var request = http.request(options, function (res) {
    var data = '';
    res.on('data', function (chunk) {
        data += chunk;
    });
    res.on('end', function () {
        console.log(data);

    });
});
request.on('error', function (e) {
    console.log(e.message);
});
request.end();