request / request-promise

The simplified HTTP request client 'request' with Promise support. Powered by Bluebird.
ISC License
4.77k stars 297 forks source link

Trying to scrape data from web page with content-encoding: br, doesn't work with options like "JSON=true" or "gzip=true" #347

Open Sathyajith95 opened 4 years ago

Sathyajith95 commented 4 years ago

const request = require("request-promise") const cheerio = require("cheerio") const fs = require("fs") const json2csv = require("json2csv").Parser;

const coronaURL = "https://www.worldometers.info/coronavirus/country/india/";

(async() => { const coronaData = [] const response = await request({ uri: coronaURL, headers: { accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.9", "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en-IN;q=0.9,en;q=0.8,kn;q=0.7", }, gzip: true, });

let $ = cheerio.load(response)
let totalCases = $('div[class="maincounter-number"] > span[style="color:#aaa"]').text().trim();
//let totalCases = $('div[class="title_wrapper"]> h1').text().trim();

console.log(totalCases);
coronaData.push({
    totalCases
});

console.log(coronaData);
const json2csvParser = new json2csv();
const csv = json2csvParser.parse(coronaData);

fs.writeFileSync("./coronaCount.csv", csv, "utf-8")

}

)();

This is the node.js code. I'm new to web-scrapping and node.js. It returns an empty string. Can anyone please help me?