import robots from 'robots';
const robotsParser = new robots.RobotsParser();
const url = 'https://google.com/robots.txt';
return new Promise((resolve, reject) => {
robotsParser.setUrl(url, (parser, success) => {
if (success) {
resolve(parser.parse());
} else {
reject();
}
});
});
This returns
'TypeError: Cannot read property \'length\' of undefined',
' at RobotsParser.parse (/home/debian/project/node_modules/robots/lib/parser.js:250:28)',
' at robotsParser.setUrl (/home/debian/project/src/lib/site-mapper.js:24:32)',
' at IncomingMessage.<anonymous> (/home/debian/project/node_modules/robots/lib/parser.js:191:9)',
' at emitNone (events.js:91:20)',
' at IncomingMessage.emit (events.js:188:7)',
' at endReadableNT (_stream_readable.js:975:12)',
' at _combinedTickCallback (internal/process/next_tick.js:80:11)',
' at process._tickCallback (internal/process/next_tick.js:104:9)'
My code
This returns
Using: node v7.8.0 npm 5.0.0 robots 0.10.1