search

codemanki / cloudscraper

--DEPRECATED -- 🛑 🛑 Node.js library to bypass cloudflare's anti-ddos page
MIT License
600 stars 139 forks source link

CaptchaError: captcha v4.0.1 (SSL negotiation) #211

Closed karnadii closed 5 years ago

karnadii commented 5 years ago 
var uri = "https://nanime.in";
  var cloudscraper = require("cloudscraper").defaults({
    headers: {
      Connection: "keep-alive",
      "User-Agent":
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36",
      Accept:       "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
      "Accept-Language": "en-US,en;q=0.9"
    }
  });

  cloudscraper.get(uri, function(error, response, body) {
    if (error) {
      throw error;
      res.send(error);
      next();
    } else {
      console.log(body);

      res.send(body);
      next();
    }
  });

/media/karnadi/Boice/Proyek/scraper-api/node_modules/request-promise-core/lib/plumbing.js:130
            throw thrownException;
            ^
CaptchaError: captcha
karnadii commented 5 years ago

@pro-src every example i run all of it almost get me error CaptchaError: captcha, I am using node v10.15.3. the url is https://nanime.in. I am clueless of what was the problem.

ghost commented 5 years ago

@karnadii read #208 and let me know if that doesn't make sense to you. I can try to simplify it.

karnadii commented 5 years ago

@pro-src u say it is the header, i am using header or not still get the same error, the rest I dont understand, also ArnaudPiroelle upgrade node to v12, i have upgrade it, and still the same error.

ghost commented 5 years ago

@karnadii I'm unable to reproduce the problem using the URL that you provided. Those headers do trigger a reCAPTCHA whereas the default ones do not. Do you have another URL?

codemanki commented 5 years ago

@karnadii could you please try rm -rf node_modules, make sure that you have at least 4.0.1 version in your package.json and do npm/yarn install?

ghost commented 5 years ago

@karnadii Also if you enable debugging, you can see which headers are being sent.

const cloudscraper = require('cloudscraper');
cloudscraper.debug = true

Note that upgrading your nodejs version is not necessary. It should work just fine on nodejs v6+ without any accommodations. Although, using the latest stable version of nodejs is always recommended and using brotli is future proofing.

karnadii commented 5 years ago

@codemanki @pro-src still error

Stacktrace ``` REQUEST { requester: { [Function: request] get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: { [Function: Request] super_: [Function], debug: true, defaultProxyHeaderWhiteList: [Array], defaultProxyHeaderExclusiveList: [Array] }, initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: CookieJar { enableLooseMode: true, store: { idx: {} } } }, headers: { Host: Symbol(host), Connection: 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 3, decodeEmails: false, gzip: true, uri: 'https://nanime.in', callback: [Function: RP$callback], method: 'GET', realEncoding: 'utf8', encoding: null, transform: undefined, simple: true, resolveWithFullResponse: false, transform2xxOnly: false } REQUEST make request https://nanime.in/ REQUEST onRequestResponse https://nanime.in/ 503 { date: 'Tue, 30 Apr 2019 11:39:20 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'set-cookie': [ '__cfduid=d139a5944da08d29483056940b4813b221556624360; expires=Wed, 29-Apr-20 11:39:20 GMT; path=/; domain=.nanime.in; HttpOnly' ], 'x-frame-options': 'SAMEORIGIN', 'cache-control': 'no-cache', 'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"', vary: 'Accept-Encoding', server: 'cloudflare', 'cf-ray': '4cf93c101c2aaeff-KIX' } REQUEST reading response's body REQUEST finish init function https://nanime.in/ REQUEST response end https://nanime.in/ 503 { date: 'Tue, 30 Apr 2019 11:39:20 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'set-cookie': [ '__cfduid=d139a5944da08d29483056940b4813b221556624360; expires=Wed, 29-Apr-20 11:39:20 GMT; path=/; domain=.nanime.in; HttpOnly' ], 'x-frame-options': 'SAMEORIGIN', 'cache-control': 'no-cache', 'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"', vary: 'Accept-Encoding', server: 'cloudflare', 'cf-ray': '4cf93c101c2aaeff-KIX' } REQUEST end event https://nanime.in/ REQUEST has body https://nanime.in/ 14614 REQUEST emitting complete https://nanime.in/ REQUEST { requester: { [Function: request] get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: { [Function: Request] super_: [Function], debug: true, defaultProxyHeaderWhiteList: [Array], defaultProxyHeaderExclusiveList: [Array] }, initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: CookieJar { enableLooseMode: true, store: { idx: { 'nanime.in': { '/': { __cfduid: Cookie="__cfduid=d139a5944da08d29483056940b4813b221556624360; Expires=Wed, 29 Apr 2020 11:39:20 GMT; Domain=nanime.in; Path=/; HttpOnly; hostOnly=false; aAge=4052ms; cAge=4052ms" } } } } } }, headers: { Host: Symbol(host), Connection: 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate', Referer: 'https://nanime.in/' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 2, decodeEmails: false, gzip: true, uri: 'https://nanime.in/cdn-cgi/l/chk_jschl', callback: [Function: RP$callback], method: 'GET', realEncoding: 'utf8', encoding: null, qs: { s: 'a5b5556630242528d4a4e13559020d376298925d-1556624360-1800-AVyPA6TqRaV6xPUWsd0vk8cRI5QMpxUT3gkk43PgotYw6X3PDTL0Q2BdE4OdTpDu0/G7zez10NW10cr5ZOBzGdKP78VtepTtNWQLZjBMB3lb9/eQMvgk5WfUEZR9hkQA4w==', jschl_vc: '3538eb7f4ae4c1bcc92e45466b354462', pass: '1556624364.98-dw0RDAVoZq', jschl_answer: '30.0161640138' }, transform: undefined, simple: true, resolveWithFullResponse: false, transform2xxOnly: false } REQUEST make request https://nanime.in/cdn-cgi/l/chk_jschl?s=a5b5556630242528d4a4e13559020d376298925d-1556624360-1800-AVyPA6TqRaV6xPUWsd0vk8cRI5QMpxUT3gkk43PgotYw6X3PDTL0Q2BdE4OdTpDu0%2FG7zez10NW10cr5ZOBzGdKP78VtepTtNWQLZjBMB3lb9%2FeQMvgk5WfUEZR9hkQA4w%3D%3D&jschl_vc=3538eb7f4ae4c1bcc92e45466b354462&pass=1556624364.98-dw0RDAVoZq&jschl_answer=30.0161640138 REQUEST onRequestResponse https://nanime.in/cdn-cgi/l/chk_jschl?s=a5b5556630242528d4a4e13559020d376298925d-1556624360-1800-AVyPA6TqRaV6xPUWsd0vk8cRI5QMpxUT3gkk43PgotYw6X3PDTL0Q2BdE4OdTpDu0%2FG7zez10NW10cr5ZOBzGdKP78VtepTtNWQLZjBMB3lb9%2FeQMvgk5WfUEZR9hkQA4w%3D%3D&jschl_vc=3538eb7f4ae4c1bcc92e45466b354462&pass=1556624364.98-dw0RDAVoZq&jschl_answer=30.0161640138 403 { date: 'Tue, 30 Apr 2019 11:39:25 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'cf-chl-bypass': '1', 'cache-control': 'max-age=2', expires: 'Tue, 30 Apr 2019 11:39:27 GMT', 'x-frame-options': 'SAMEORIGIN', server: 'cloudflare', 'cf-ray': '4cf93c2d2a5caecf-KIX', 'content-encoding': 'gzip' } REQUEST reading response's body REQUEST finish init function https://nanime.in/cdn-cgi/l/chk_jschl?s=a5b5556630242528d4a4e13559020d376298925d-1556624360-1800-AVyPA6TqRaV6xPUWsd0vk8cRI5QMpxUT3gkk43PgotYw6X3PDTL0Q2BdE4OdTpDu0%2FG7zez10NW10cr5ZOBzGdKP78VtepTtNWQLZjBMB3lb9%2FeQMvgk5WfUEZR9hkQA4w%3D%3D&jschl_vc=3538eb7f4ae4c1bcc92e45466b354462&pass=1556624364.98-dw0RDAVoZq&jschl_answer=30.0161640138 REQUEST response end https://nanime.in/cdn-cgi/l/chk_jschl?s=a5b5556630242528d4a4e13559020d376298925d-1556624360-1800-AVyPA6TqRaV6xPUWsd0vk8cRI5QMpxUT3gkk43PgotYw6X3PDTL0Q2BdE4OdTpDu0%2FG7zez10NW10cr5ZOBzGdKP78VtepTtNWQLZjBMB3lb9%2FeQMvgk5WfUEZR9hkQA4w%3D%3D&jschl_vc=3538eb7f4ae4c1bcc92e45466b354462&pass=1556624364.98-dw0RDAVoZq&jschl_answer=30.0161640138 403 { date: 'Tue, 30 Apr 2019 11:39:25 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'cf-chl-bypass': '1', 'cache-control': 'max-age=2', expires: 'Tue, 30 Apr 2019 11:39:27 GMT', 'x-frame-options': 'SAMEORIGIN', server: 'cloudflare', 'cf-ray': '4cf93c2d2a5caecf-KIX', 'content-encoding': 'gzip' } REQUEST end event https://nanime.in/cdn-cgi/l/chk_jschl?s=a5b5556630242528d4a4e13559020d376298925d-1556624360-1800-AVyPA6TqRaV6xPUWsd0vk8cRI5QMpxUT3gkk43PgotYw6X3PDTL0Q2BdE4OdTpDu0%2FG7zez10NW10cr5ZOBzGdKP78VtepTtNWQLZjBMB3lb9%2FeQMvgk5WfUEZR9hkQA4w%3D%3D&jschl_vc=3538eb7f4ae4c1bcc92e45466b354462&pass=1556624364.98-dw0RDAVoZq&jschl_answer=30.0161640138 REQUEST has body https://nanime.in/cdn-cgi/l/chk_jschl?s=a5b5556630242528d4a4e13559020d376298925d-1556624360-1800-AVyPA6TqRaV6xPUWsd0vk8cRI5QMpxUT3gkk43PgotYw6X3PDTL0Q2BdE4OdTpDu0%2FG7zez10NW10cr5ZOBzGdKP78VtepTtNWQLZjBMB3lb9%2FeQMvgk5WfUEZR9hkQA4w%3D%3D&jschl_vc=3538eb7f4ae4c1bcc92e45466b354462&pass=1556624364.98-dw0RDAVoZq&jschl_answer=30.0161640138 8183 REQUEST emitting complete https://nanime.in/cdn-cgi/l/chk_jschl?s=a5b5556630242528d4a4e13559020d376298925d-1556624360-1800-AVyPA6TqRaV6xPUWsd0vk8cRI5QMpxUT3gkk43PgotYw6X3PDTL0Q2BdE4OdTpDu0%2FG7zez10NW10cr5ZOBzGdKP78VtepTtNWQLZjBMB3lb9%2FeQMvgk5WfUEZR9hkQA4w%3D%3D&jschl_vc=3538eb7f4ae4c1bcc92e45466b354462&pass=1556624364.98-dw0RDAVoZq&jschl_answer=30.0161640138 /media/karnadi/Boice/Proyek/scraper-api/node_modules/request-promise-core/lib/plumbing.js:130 throw thrownException; ^ CaptchaError: captcha at validate (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:231:11) at onCloudflareResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:192:5) at onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:171:5) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:132:7) at Object.onceWrapper (events.js:273:13) at Request.emit (events.js:182:13) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1161:10) at Request.emit (events.js:182:13) at Gunzip. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1083:12) at Object.onceWrapper (events.js:273:13) at Gunzip.emit (events.js:187:15) at endReadableNT (_stream_readable.js:1081:12) at process._tickCallback (internal/process/next_tick.js:63:19) zsh: exit 1 node a.js ```
karnadii commented 5 years ago

the one that I want to scrape is nanime.in, so this is the only url i can provide.

karnadii commented 5 years ago

using example of rechaptav2

Stacktrace ``` The url is "https://nanime.in/cdn-cgi/l/chk_jschl?s=7bde9c1432c63b8f8e2e19e073d2a9faa5a22aee-1556623321-1800-AXFAykJcuHr6bPu3HT%2By0vCOKNc6VQyKZUt8eIA6u1fspb9%2FwNxTJf5o1yxWzjTdHVdFpZBLMlTxovJU8sQTxx6kr%2Bgp8tCtfVBlroWrQXFHlJSUkcMDAgxodRwUZGb2xQ%3D%3D&jschl_vc=a33b4ee53309095b05f27be7fdbec1aa&pass=1556623325.471-EH1SSD%2Bzm4&jschl_answer=108.2667425162" The site key is "6LfBixYUAAAAABhdHynFUIMA_sa4s-XsJvnjtgB0" CaptchaError: Error: This is a dummy function at submit (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:409:23) at /media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:428:9 at processTicksAndRejections (internal/process/task_queues.js:89:5) { name: 'CaptchaError', message: 'Error: This is a dummy function', cause: Error: This is a dummy function at alternative (/media/karnadi/Boice/Proyek/scraper-api/a.js:23:25) at onCaptcha (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:419:20) at onCloudflareResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:196:14) at onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:171:5) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:132:7) at Object.onceWrapper (events.js:284:20) at Request.emit (events.js:196:13) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1161:10) at Request.emit (events.js:196:13) at Gunzip. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1083:12) at Object.onceWrapper (events.js:284:20) at Gunzip.emit (events.js:201:15) at endReadableNT (_stream_readable.js:1130:12) at processTicksAndRejections (internal/process/task_queues.js:84:17), error: Error: This is a dummy function at alternative (/media/karnadi/Boice/Proyek/scraper-api/a.js:23:25) at onCaptcha (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:419:20) at onCloudflareResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:196:14) at onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:171:5) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:132:7) at Object.onceWrapper (events.js:284:20) at Request.emit (events.js:196:13) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1161:10) at Request.emit (events.js:196:13) at Gunzip. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1083:12) at Object.onceWrapper (events.js:284:20) at Gunzip.emit (events.js:201:15) at endReadableNT (_stream_readable.js:1130:12) at processTicksAndRejections (internal/process/task_queues.js:84:17), options: { requester: [Function: request] { get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: [Function], initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: [CookieJar] }, headers: { Referer: 'https://nanime.in/' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 1, decodeEmails: false, gzip: true, onCaptcha: [Function: alternative], uri: 'https://nanime.in/cdn-cgi/l/chk_jschl', pool: undefined, method: 'GET', realEncoding: 'utf8', encoding: null, callback: [Function], qs: { s: '7bde9c1432c63b8f8e2e19e073d2a9faa5a22aee-1556623321-1800-AXFAykJcuHr6bPu3HT+y0vCOKNc6VQyKZUt8eIA6u1fspb9/wNxTJf5o1yxWzjTdHVdFpZBLMlTxovJU8sQTxx6kr+gp8tCtfVBlroWrQXFHlJSUkcMDAgxodRwUZGb2xQ==', jschl_vc: 'a33b4ee53309095b05f27be7fdbec1aa', pass: '1556623325.471-EH1SSD+zm4', jschl_answer: '108.2667425162' } }, response: IncomingMessage { _readableState: ReadableState { objectMode: false, highWaterMark: 16384, buffer: BufferList { head: null, tail: null, length: 0 }, length: 0, pipes: null, pipesCount: 0, flowing: false, ended: true, endEmitted: true, reading: false, sync: false, needReadable: false, emittedReadable: false, readableListening: false, resumeScheduled: false, paused: false, emitClose: true, autoDestroy: false, destroyed: false, defaultEncoding: 'utf8', awaitDrain: 0, readingMore: false, decoder: null, encoding: null }, readable: false, _events: [Object: null prototype] { end: [Array], close: [Function] }, _eventsCount: 2, _maxListeners: undefined, socket: TLSSocket { _tlsOptions: [Object], _secureEstablished: true, _securePending: false, _newSessionPending: false, _controlReleased: true, _SNICallback: null, servername: false, alpnProtocol: false, authorized: true, authorizationError: null, encrypted: true, _events: [Object], _eventsCount: 8, connecting: false, _hadError: false, _parent: null, _host: 'nanime.in', _readableState: [ReadableState], readable: false, _maxListeners: undefined, _writableState: [WritableState], writable: false, allowHalfOpen: false, _sockname: null, _pendingData: null, _pendingEncoding: '', server: undefined, _server: null, ssl: null, _requestCert: true, _rejectUnauthorized: true, parser: null, _httpMessage: [ClientRequest], [Symbol(res)]: null, [Symbol(asyncId)]: 94, [Symbol(kHandle)]: null, [Symbol(lastWriteQueueSize)]: 0, [Symbol(timeout)]: null, [Symbol(kBytesRead)]: 4540, [Symbol(kBytesWritten)]: 501, [Symbol(connect-options)]: [Object] }, connection: TLSSocket { _tlsOptions: [Object], _secureEstablished: true, _securePending: false, _newSessionPending: false, _controlReleased: true, _SNICallback: null, servername: false, alpnProtocol: false, authorized: true, authorizationError: null, encrypted: true, _events: [Object], _eventsCount: 8, connecting: false, _hadError: false, _parent: null, _host: 'nanime.in', _readableState: [ReadableState], readable: false, _maxListeners: undefined, _writableState: [WritableState], writable: false, allowHalfOpen: false, _sockname: null, _pendingData: null, _pendingEncoding: '', server: undefined, _server: null, ssl: null, _requestCert: true, _rejectUnauthorized: true, parser: null, _httpMessage: [ClientRequest], [Symbol(res)]: null, [Symbol(asyncId)]: 94, [Symbol(kHandle)]: null, [Symbol(lastWriteQueueSize)]: 0, [Symbol(timeout)]: null, [Symbol(kBytesRead)]: 4540, [Symbol(kBytesWritten)]: 501, [Symbol(connect-options)]: [Object] }, httpVersionMajor: 1, httpVersionMinor: 1, httpVersion: '1.1', complete: true, headers: { date: 'Tue, 30 Apr 2019 11:22:06 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'cf-chl-bypass': '1', 'cache-control': 'max-age=2', expires: 'Tue, 30 Apr 2019 11:22:08 GMT', 'x-frame-options': 'SAMEORIGIN', server: 'cloudflare', 'cf-ray': '4cf922ce0d66af1d-KIX', 'content-encoding': 'gzip' }, rawHeaders: [ 'Date', 'Tue, 30 Apr 2019 11:22:06 GMT', 'Content-Type', 'text/html; charset=UTF-8', 'Transfer-Encoding', 'chunked', 'Connection', 'close', 'CF-Chl-Bypass', '1', 'Cache-Control', 'max-age=2', 'Expires', 'Tue, 30 Apr 2019 11:22:08 GMT', 'X-Frame-Options', 'SAMEORIGIN', 'Server', 'cloudflare', 'CF-RAY', '4cf922ce0d66af1d-KIX', 'Content-Encoding', 'gzip' ], trailers: {}, rawTrailers: [], aborted: false, upgrade: false, url: '', method: null, statusCode: 403, statusMessage: 'Forbidden', client: TLSSocket { _tlsOptions: [Object], _secureEstablished: true, _securePending: false, _newSessionPending: false, _controlReleased: true, _SNICallback: null, servername: false, alpnProtocol: false, authorized: true, authorizationError: null, encrypted: true, _events: [Object], _eventsCount: 8, connecting: false, _hadError: false, _parent: null, _host: 'nanime.in', _readableState: [ReadableState], readable: false, _maxListeners: undefined, _writableState: [WritableState], writable: false, allowHalfOpen: false, _sockname: null, _pendingData: null, _pendingEncoding: '', server: undefined, _server: null, ssl: null, _requestCert: true, _rejectUnauthorized: true, parser: null, _httpMessage: [ClientRequest], [Symbol(res)]: null, [Symbol(asyncId)]: 94, [Symbol(kHandle)]: null, [Symbol(lastWriteQueueSize)]: 0, [Symbol(timeout)]: null, [Symbol(kBytesRead)]: 4540, [Symbol(kBytesWritten)]: 501, [Symbol(connect-options)]: [Object] }, _consuming: true, _dumped: false, req: ClientRequest { _events: [Object], _eventsCount: 5, _maxListeners: undefined, outputData: [], outputSize: 0, writable: true, _last: true, chunkedEncoding: false, shouldKeepAlive: false, useChunkedEncodingByDefault: false, sendDate: false, _removedConnection: false, _removedContLen: false, _removedTE: false, _contentLength: 0, _hasBody: true, _trailer: '', finished: true, _headerSent: true, socket: [TLSSocket], connection: [TLSSocket], _header: 'GET ' + '/cdn-cgi/l/chk_jschl?s=7bde9c1432c63b8f8e2e19e073d2a9faa5a22aee-1556623321-1800-AXFAykJcuHr6bPu3HT%2By0vCOKNc6VQyKZUt8eIA6u1fspb9%2FwNxTJf5o1yxWzjTdHVdFpZBLMlTxovJU8sQTxx6kr%2Bgp8tCtfVBlroWrQXFHlJSUkcMDAgxodRwUZGb2xQ%3D%3D&jschl_vc=a33b4ee53309095b05f27be7fdbec1aa&pass=1556623325.471-EH1SSD%2Bzm4&jschl_answer=108.2667425162 ' + 'HTTP/1.1\r' + '\nReferer: https://nanime.in/\r' + '\nhost: nanime.in\r' + '\ncookie: __cfduid=d15ffbaf793e3067e612956793e1242e91556623315\r' + '\naccept-encoding: gzip, deflate\r' + '\nConnection: close\r' + '\n\r' + '\n', _onPendingData: [Function: noopPendingOutput], agent: [Agent], socketPath: undefined, method: 'GET', path: '/cdn-cgi/l/chk_jschl?s=7bde9c1432c63b8f8e2e19e073d2a9faa5a22aee-1556623321-1800-AXFAykJcuHr6bPu3HT%2By0vCOKNc6VQyKZUt8eIA6u1fspb9%2FwNxTJf5o1yxWzjTdHVdFpZBLMlTxovJU8sQTxx6kr%2Bgp8tCtfVBlroWrQXFHlJSUkcMDAgxodRwUZGb2xQ%3D%3D&jschl_vc=a33b4ee53309095b05f27be7fdbec1aa&pass=1556623325.471-EH1SSD%2Bzm4&jschl_answer=108.2667425162', _ended: true, res: [Circular], aborted: false, timeoutCb: null, upgradeOrConnect: false, parser: null, maxHeadersCount: null, [Symbol(isCorked)]: false, [Symbol(outHeadersKey)]: [Object] }, request: Request { _events: [Object], _eventsCount: 4, _maxListeners: undefined, requester: [Function], headers: [Object], cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 1, decodeEmails: false, gzip: true, onCaptcha: [Function: alternative], uri: [Url], method: 'GET', realEncoding: 'utf8', encoding: null, callback: [Function], readable: true, writable: true, explicitMethod: true, _qs: [Querystring], _auth: [Auth], _oauth: [OAuth], _multipart: [Multipart], _redirect: [Redirect], _tunnel: [Tunnel], _rp_resolve: [Function], _rp_reject: [Function], _rp_promise: [Promise], _rp_callbackOrig: [Function], _rp_options: [Object], setHeader: [Function], hasHeader: [Function], getHeader: [Function], removeHeader: [Function], localAddress: undefined, pool: {}, dests: [], __isRequestRequest: true, _callback: [Function: RP$callback], proxy: null, tunnel: true, setHost: true, originalCookieHeader: undefined, _jar: [RequestJar], port: 443, host: 'nanime.in', url: [Url], path: '/cdn-cgi/l/chk_jschl?s=7bde9c1432c63b8f8e2e19e073d2a9faa5a22aee-1556623321-1800-AXFAykJcuHr6bPu3HT%2By0vCOKNc6VQyKZUt8eIA6u1fspb9%2FwNxTJf5o1yxWzjTdHVdFpZBLMlTxovJU8sQTxx6kr%2Bgp8tCtfVBlroWrQXFHlJSUkcMDAgxodRwUZGb2xQ%3D%3D&jschl_vc=a33b4ee53309095b05f27be7fdbec1aa&pass=1556623325.471-EH1SSD%2Bzm4&jschl_answer=108.2667425162', httpModule: [Object], agentClass: [Function: Agent], agent: [Agent], cloudscraper: true, _started: true, href: 'https://nanime.in/cdn-cgi/l/chk_jschl?s=7bde9c1432c63b8f8e2e19e073d2a9faa5a22aee-1556623321-1800-AXFAykJcuHr6bPu3HT%2By0vCOKNc6VQyKZUt8eIA6u1fspb9%2FwNxTJf5o1yxWzjTdHVdFpZBLMlTxovJU8sQTxx6kr%2Bgp8tCtfVBlroWrQXFHlJSUkcMDAgxodRwUZGb2xQ%3D%3D&jschl_vc=a33b4ee53309095b05f27be7fdbec1aa&pass=1556623325.471-EH1SSD%2Bzm4&jschl_answer=108.2667425162', req: [ClientRequest], ntick: true, response: [Circular], originalHost: 'nanime.in', originalHostHeaderName: 'host', responseContent: [Gunzip], _ended: true, _destdata: true }, toJSON: [Function: responseToJSON], caseless: Caseless { dict: [Object] }, body: , responseStartTime: 1556623326609, isCloudflare: true, isHTML: true, isCaptcha: true, challengeForm: '\n \n' + ' \n ' + '
\n \n', captcha: { url: 'https://nanime.in/cdn-cgi/l/chk_jschl?s=7bde9c1432c63b8f8e2e19e073d2a9faa5a22aee-1556623321-1800-AXFAykJcuHr6bPu3HT%2By0vCOKNc6VQyKZUt8eIA6u1fspb9%2FwNxTJf5o1yxWzjTdHVdFpZBLMlTxovJU8sQTxx6kr%2Bgp8tCtfVBlroWrQXFHlJSUkcMDAgxodRwUZGb2xQ%3D%3D&jschl_vc=a33b4ee53309095b05f27be7fdbec1aa&pass=1556623325.471-EH1SSD%2Bzm4&jschl_answer=108.2667425162', siteKey: '6LfBixYUAAAAABhdHynFUIMA_sa4s-XsJvnjtgB0', form: [Object], submit: [Function: submit] } } } ```
ghost commented 5 years ago

@karnadii The recaptcha example is a dummy function and is not meant to work. It's something that you have to code yourself. I think the reason that you're getting a CAPTCHA in this case would because of an IP related issue. Perhaps the site owner has banned (set a CAPTCHA for) your ASN/ISP or the entire IP range of your country. I can't say for sure. I'd like to have you perform an experiment for me to be on the safe side though. If you don't mind, run this code and paste the results:

var cloudscraper = require('cloudscraper');
var fs = require('fs');

cloudscraper.get('https://www.ssllabs.com/ssltest/viewMyClient.html').then(body => fs.writeFileSync('./results.html', body, 'utf-8'), console.error);

Upload the contents of results.html, pretty please.

karnadii commented 5 years ago

@pro-src

results.html ```html Qualys SSL Labs - Projects / SSL Client Test
SSL/TLS Capabilities of Your Browser
User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36

Other User Agents »

Protocol Support
Please wait, checking protocol support...
Logjam Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the Logjam attack, please go to weakdh.org.
To test manually, click here. Your user agent is not vulnerable if it fails to connect to the site.
FREAK Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the FREAK attack, please go to www.freakattack.com.
To test manually, click here. Your user agent is not vulnerable if it fails to connect to the site.
POODLE Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the POODLE attack, please read this blog post.
SSL 2 Protocol Support
Your user agent supports SSL 2. You should upgrade.
SSL 2 is a very old, obsolete, and insecure version of the SSL protocol. You can usually disable this protocol version in configuration, but modern clients don't support it at all. This really means that you should upgrade your software to a better version.
iOS and OS X TLS Authentication Vulnerability
Please wait, checking if your user agent is vulnerable...
To test manually, click here. If your user agent refuses to connect, you are not vulnerable. This test requires a connection to the SSL Labs server on port 10443. A strict outbound firewall might interfere. You should test Safari running on iOS or OS X. Chrome and Firefox are not vulnerable, even when running on a vulnerable operating system. MORE »
Protocol Features
Protocols
TLS 1.3 No
TLS 1.2 Yes*
TLS 1.1 Yes*
TLS 1.0 Yes*
SSL 3 Yes*
SSL 2 No
(*) Without JavaScript, this test reliably detects only the highest supported protocol.


Cipher Suites (in order of preference)
TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 (0xc02f)   Forward Secrecy 128
TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 (0xc02b)   Forward Secrecy 128
TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (0xc030)   Forward Secrecy 256
TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 (0xc02c)   Forward Secrecy 256
TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 (0x9e)   Forward Secrecy 128
TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 (0xc027)   Forward Secrecy 128
TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 (0x67)   Forward Secrecy 128
TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 (0xc028)   Forward Secrecy 256
TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 (0x6b)   Forward Secrecy 256
TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 (0xa3)   Forward Secrecy2 256
TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 (0x9f)   Forward Secrecy 256
TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 (0xcca9)   Forward Secrecy 256
TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 (0xcca8)   Forward Secrecy 256
TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 (0xccaa)   Forward Secrecy 256
TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 (0xc0af)   Forward Secrecy 256
TLS_ECDHE_ECDSA_WITH_AES_256_CCM (0xc0ad)   Forward Secrecy 256
TLS_DHE_RSA_WITH_AES_256_CCM_8 (0xc0a3)   Forward Secrecy 256
TLS_DHE_RSA_WITH_AES_256_CCM (0xc09f)   Forward Secrecy 256
TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 (0xa2)   Forward Secrecy2 128
TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 (0xc0ae)   Forward Secrecy 128
TLS_ECDHE_ECDSA_WITH_AES_128_CCM (0xc0ac)   Forward Secrecy 128
TLS_DHE_RSA_WITH_AES_128_CCM_8 (0xc0a2)   Forward Secrecy 128
TLS_DHE_RSA_WITH_AES_128_CCM (0xc09e)   Forward Secrecy 128
TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 (0xc024)   Forward Secrecy 256
TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 (0x6a)   Forward Secrecy2 256
TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 (0xc023)   Forward Secrecy 128
TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 (0x40)   Forward Secrecy2 128
TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA (0xc00a)   Forward Secrecy 256
TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA (0xc014)   Forward Secrecy 256
TLS_DHE_RSA_WITH_AES_256_CBC_SHA (0x39)   Forward Secrecy 256
TLS_DHE_DSS_WITH_AES_256_CBC_SHA (0x38)   Forward Secrecy2 256
TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA (0xc009)   Forward Secrecy 128
TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA (0xc013)   Forward Secrecy 128
TLS_DHE_RSA_WITH_AES_128_CBC_SHA (0x33)   Forward Secrecy 128
TLS_DHE_DSS_WITH_AES_128_CBC_SHA (0x32)   Forward Secrecy2 128
TLS_RSA_WITH_AES_256_GCM_SHA384 (0x9d)   WEAK 256
TLS_RSA_WITH_AES_256_CCM_8 (0xc0a1)   WEAK 256
TLS_RSA_WITH_AES_256_CCM (0xc09d)   WEAK 256
TLS_RSA_WITH_AES_128_GCM_SHA256 (0x9c)   WEAK 128
TLS_RSA_WITH_AES_128_CCM_8 (0xc0a0)   WEAK 128
TLS_RSA_WITH_AES_128_CCM (0xc09c)   WEAK 128
TLS_RSA_WITH_AES_256_CBC_SHA256 (0x3d)   WEAK 256
TLS_RSA_WITH_AES_128_CBC_SHA256 (0x3c)   WEAK 128
TLS_RSA_WITH_AES_256_CBC_SHA (0x35)   WEAK 256
TLS_RSA_WITH_AES_128_CBC_SHA (0x2f)   WEAK 128
TLS_EMPTY_RENEGOTIATION_INFO_SCSV (0xff) -
(1) When a browser supports SSL 2, its SSL 2-only suites are shown only on the very first connection to this site. To see the suites, close all browser windows, then open this exact page directly. Don't refresh.
(2) Cannot be used for Forward Secrecy because they require DSA keys, which are effectively limited to 1024 bits.


Protocol Details
Server Name Indication (SNI) Yes
Secure Renegotiation Yes
TLS compression No
Session tickets Yes
OCSP stapling No
Signature algorithms SHA512/RSA, SHA512/DSA, SHA512/ECDSA, SHA384/RSA, SHA384/DSA, SHA384/ECDSA, SHA256/RSA, SHA256/DSA, SHA256/ECDSA, SHA224/RSA, SHA224/DSA, SHA224/ECDSA, SHA1/RSA, SHA1/DSA, SHA1/ECDSA
Named Groups x25519, secp256r1, secp521r1, secp384r1
Next Protocol Negotiation No
Application Layer Protocol Negotiation No
SSL 2 handshake compatibility No

```
ghost commented 5 years ago

@karnadii This could very well be an issue with Cloudflare checking the ciphers used during SSL negotiation as your cipher suite list is different from mine. I'd like to rule that out but I'll need a minute to work out a test.

ghost commented 5 years ago

@karnadii Please use this command exactly to run your test:

node --tls-cipher-list='GREASE_3A:GREASE_6A:AES128-GCM-SHA256:AES256-GCM-SHA256:AES256-GCM-SHA384:CHACHA20-POLY1305-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305-SHA256:ECDHE-RSA-CHACHA20-POLY1305-SHA256:ECDHE-RSA-AES128-CBC-SHA:ECDHE-RSA-AES256-CBC-SHA:RSA-AES128-GCM-SHA256:RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:RSA-AES256-SHA:3DES-EDE-CBC' a.js
karnadii commented 5 years ago

@pro-src

Stacktrace ``` ➜ node --tls-cipher-list='GREASE_3A:GREASE_6A:AES128-GCM-SHA256:AES256-GCM-SHA256:AES256-GCM-SHA384:CHACHA20-POLY1305-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305-SHA256:ECDHE-RSA-CHACHA20-POLY1305-SHA256:ECDHE-RSA-AES128-CBC-SHA:ECDHE-RSA-AES256-CBC-SHA:RSA-AES128-GCM-SHA256:RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:RSA-AES256-SHA:3DES-EDE-CBC' a.js REQUEST { requester: { [Function: request] get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: { [Function: Request] super_: [Function], debug: true, defaultProxyHeaderWhiteList: [Array], defaultProxyHeaderExclusiveList: [Array] }, initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: CookieJar { enableLooseMode: true, store: { idx: {} } } }, headers: { Host: Symbol(host), Connection: 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.40 Safari/537.36', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 3, decodeEmails: false, gzip: true, uri: 'https://nanime.in', callback: [Function: RP$callback], method: 'GET', realEncoding: 'utf8', encoding: null, transform: undefined, simple: true, resolveWithFullResponse: false, transform2xxOnly: false } REQUEST make request https://nanime.in/ REQUEST onRequestResponse https://nanime.in/ 503 { date: 'Tue, 30 Apr 2019 12:26:15 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'set-cookie': [ '__cfduid=d04a469c10f49205b586473b252f707c71556627175; expires=Wed, 29-Apr-20 12:26:15 GMT; path=/; domain=.nanime.in; HttpOnly' ], 'x-frame-options': 'SAMEORIGIN', 'cache-control': 'no-cache', 'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"', vary: 'Accept-Encoding', server: 'cloudflare', 'cf-ray': '4cf980c6ac68af0b-KIX' } REQUEST reading response's body REQUEST finish init function https://nanime.in/ REQUEST response end https://nanime.in/ 503 { date: 'Tue, 30 Apr 2019 12:26:15 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'set-cookie': [ '__cfduid=d04a469c10f49205b586473b252f707c71556627175; expires=Wed, 29-Apr-20 12:26:15 GMT; path=/; domain=.nanime.in; HttpOnly' ], 'x-frame-options': 'SAMEORIGIN', 'cache-control': 'no-cache', 'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"', vary: 'Accept-Encoding', server: 'cloudflare', 'cf-ray': '4cf980c6ac68af0b-KIX' } REQUEST end event https://nanime.in/ REQUEST has body https://nanime.in/ 12836 REQUEST emitting complete https://nanime.in/ REQUEST { requester: { [Function: request] get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: { [Function: Request] super_: [Function], debug: true, defaultProxyHeaderWhiteList: [Array], defaultProxyHeaderExclusiveList: [Array] }, initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: CookieJar { enableLooseMode: true, store: { idx: { 'nanime.in': { '/': { __cfduid: Cookie="__cfduid=d04a469c10f49205b586473b252f707c71556627175; Expires=Wed, 29 Apr 2020 12:26:15 GMT; Domain=nanime.in; Path=/; HttpOnly; hostOnly=false; aAge=4178ms; cAge=4178ms" } } } } } }, headers: { Host: Symbol(host), Connection: 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.40 Safari/537.36', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate', Referer: 'https://nanime.in/' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 2, decodeEmails: false, gzip: true, uri: 'https://nanime.in/cdn-cgi/l/chk_jschl', callback: [Function: RP$callback], method: 'GET', realEncoding: 'utf8', encoding: null, qs: { s: 'ca0af859db59cb1557cd3e84ba12eb8d657127d2-1556627175-1800-AR8ab6j07+3JCpY9af2CEznaHgoicg02sUdlTwAe4xb6DMA9QtcJXlPDCXZfGbqSZ5DiCbeOpWwqKrnYNNE7MoMBW+LokhHik6l1YLZjei3NNt0ZmGj9xcOhpg04ICbzWQ==', jschl_vc: '7edf5fd1b7404c22bbfec341d92ade08', pass: '1556627179.463-ATcKwX5ANF', jschl_answer: '18.9631045985' }, transform: undefined, simple: true, resolveWithFullResponse: false, transform2xxOnly: false } REQUEST make request https://nanime.in/cdn-cgi/l/chk_jschl?s=ca0af859db59cb1557cd3e84ba12eb8d657127d2-1556627175-1800-AR8ab6j07%2B3JCpY9af2CEznaHgoicg02sUdlTwAe4xb6DMA9QtcJXlPDCXZfGbqSZ5DiCbeOpWwqKrnYNNE7MoMBW%2BLokhHik6l1YLZjei3NNt0ZmGj9xcOhpg04ICbzWQ%3D%3D&jschl_vc=7edf5fd1b7404c22bbfec341d92ade08&pass=1556627179.463-ATcKwX5ANF&jschl_answer=18.9631045985 REQUEST onRequestResponse https://nanime.in/cdn-cgi/l/chk_jschl?s=ca0af859db59cb1557cd3e84ba12eb8d657127d2-1556627175-1800-AR8ab6j07%2B3JCpY9af2CEznaHgoicg02sUdlTwAe4xb6DMA9QtcJXlPDCXZfGbqSZ5DiCbeOpWwqKrnYNNE7MoMBW%2BLokhHik6l1YLZjei3NNt0ZmGj9xcOhpg04ICbzWQ%3D%3D&jschl_vc=7edf5fd1b7404c22bbfec341d92ade08&pass=1556627179.463-ATcKwX5ANF&jschl_answer=18.9631045985 403 { date: 'Tue, 30 Apr 2019 12:26:20 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'cf-chl-bypass': '1', 'cache-control': 'max-age=2', expires: 'Tue, 30 Apr 2019 12:26:22 GMT', 'x-frame-options': 'SAMEORIGIN', server: 'cloudflare', 'cf-ray': '4cf980e4a9c9af03-KIX', 'content-encoding': 'gzip' } REQUEST reading response's body REQUEST finish init function https://nanime.in/cdn-cgi/l/chk_jschl?s=ca0af859db59cb1557cd3e84ba12eb8d657127d2-1556627175-1800-AR8ab6j07%2B3JCpY9af2CEznaHgoicg02sUdlTwAe4xb6DMA9QtcJXlPDCXZfGbqSZ5DiCbeOpWwqKrnYNNE7MoMBW%2BLokhHik6l1YLZjei3NNt0ZmGj9xcOhpg04ICbzWQ%3D%3D&jschl_vc=7edf5fd1b7404c22bbfec341d92ade08&pass=1556627179.463-ATcKwX5ANF&jschl_answer=18.9631045985 REQUEST response end https://nanime.in/cdn-cgi/l/chk_jschl?s=ca0af859db59cb1557cd3e84ba12eb8d657127d2-1556627175-1800-AR8ab6j07%2B3JCpY9af2CEznaHgoicg02sUdlTwAe4xb6DMA9QtcJXlPDCXZfGbqSZ5DiCbeOpWwqKrnYNNE7MoMBW%2BLokhHik6l1YLZjei3NNt0ZmGj9xcOhpg04ICbzWQ%3D%3D&jschl_vc=7edf5fd1b7404c22bbfec341d92ade08&pass=1556627179.463-ATcKwX5ANF&jschl_answer=18.9631045985 403 { date: 'Tue, 30 Apr 2019 12:26:20 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'cf-chl-bypass': '1', 'cache-control': 'max-age=2', expires: 'Tue, 30 Apr 2019 12:26:22 GMT', 'x-frame-options': 'SAMEORIGIN', server: 'cloudflare', 'cf-ray': '4cf980e4a9c9af03-KIX', 'content-encoding': 'gzip' } REQUEST end event https://nanime.in/cdn-cgi/l/chk_jschl?s=ca0af859db59cb1557cd3e84ba12eb8d657127d2-1556627175-1800-AR8ab6j07%2B3JCpY9af2CEznaHgoicg02sUdlTwAe4xb6DMA9QtcJXlPDCXZfGbqSZ5DiCbeOpWwqKrnYNNE7MoMBW%2BLokhHik6l1YLZjei3NNt0ZmGj9xcOhpg04ICbzWQ%3D%3D&jschl_vc=7edf5fd1b7404c22bbfec341d92ade08&pass=1556627179.463-ATcKwX5ANF&jschl_answer=18.9631045985 REQUEST has body https://nanime.in/cdn-cgi/l/chk_jschl?s=ca0af859db59cb1557cd3e84ba12eb8d657127d2-1556627175-1800-AR8ab6j07%2B3JCpY9af2CEznaHgoicg02sUdlTwAe4xb6DMA9QtcJXlPDCXZfGbqSZ5DiCbeOpWwqKrnYNNE7MoMBW%2BLokhHik6l1YLZjei3NNt0ZmGj9xcOhpg04ICbzWQ%3D%3D&jschl_vc=7edf5fd1b7404c22bbfec341d92ade08&pass=1556627179.463-ATcKwX5ANF&jschl_answer=18.9631045985 9314 REQUEST emitting complete https://nanime.in/cdn-cgi/l/chk_jschl?s=ca0af859db59cb1557cd3e84ba12eb8d657127d2-1556627175-1800-AR8ab6j07%2B3JCpY9af2CEznaHgoicg02sUdlTwAe4xb6DMA9QtcJXlPDCXZfGbqSZ5DiCbeOpWwqKrnYNNE7MoMBW%2BLokhHik6l1YLZjei3NNt0ZmGj9xcOhpg04ICbzWQ%3D%3D&jschl_vc=7edf5fd1b7404c22bbfec341d92ade08&pass=1556627179.463-ATcKwX5ANF&jschl_answer=18.9631045985 /media/karnadi/Boice/Proyek/scraper-api/node_modules/request-promise-core/lib/plumbing.js:130 throw thrownException; ^ CaptchaError: captcha at validate (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:231:11) at onCloudflareResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:192:5) at onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:171:5) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:132:7) at Object.onceWrapper (events.js:273:13) at Request.emit (events.js:182:13) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1161:10) at Request.emit (events.js:182:13) at Gunzip. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1083:12) at Object.onceWrapper (events.js:273:13) at Gunzip.emit (events.js:187:15) at endReadableNT (_stream_readable.js:1081:12) at process._tickCallback (internal/process/next_tick.js:63:19) zsh: exit 1 node a.js ```
ghost commented 5 years ago

@karnadii :thinking: Please run this code again but with the CLI flag that I provided:

var cloudscraper = require('cloudscraper');
var fs = require('fs');

cloudscraper.get('https://www.ssllabs.com/ssltest/viewMyClient.html').then(body => fs.writeFileSync('./results.html', body, 'utf-8'), console.error);

And please paste the results.

karnadii commented 5 years ago

@pro-src

results.html ``` ➜ cat results.html Qualys SSL Labs - Projects / SSL Client Test
SSL/TLS Capabilities of Your Browser
User Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36

Other User Agents »

Protocol Support
Please wait, checking protocol support...
Logjam Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the Logjam attack, please go to weakdh.org.
To test manually, click here. Your user agent is not vulnerable if it fails to connect to the site.
FREAK Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the FREAK attack, please go to www.freakattack.com.
To test manually, click here. Your user agent is not vulnerable if it fails to connect to the site.
POODLE Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the POODLE attack, please read this blog post.
SSL 2 Protocol Support
Your user agent supports SSL 2. You should upgrade.
SSL 2 is a very old, obsolete, and insecure version of the SSL protocol. You can usually disable this protocol version in configuration, but modern clients don't support it at all. This really means that you should upgrade your software to a better version.
iOS and OS X TLS Authentication Vulnerability
Please wait, checking if your user agent is vulnerable...
To test manually, click here. If your user agent refuses to connect, you are not vulnerable. This test requires a connection to the SSL Labs server on port 10443. A strict outbound firewall might interfere. You should test Safari running on iOS or OS X. Chrome and Firefox are not vulnerable, even when running on a vulnerable operating system. MORE »
Protocol Features
Protocols
TLS 1.3 No
TLS 1.2 Yes*
TLS 1.1 Yes*
TLS 1.0 Yes*
SSL 3 Yes*
SSL 2 No
(*) Without JavaScript, this test reliably detects only the highest supported protocol.


Cipher Suites (in order of preference)
TLS_RSA_WITH_AES_128_GCM_SHA256 (0x9c)   WEAK 128
TLS_RSA_WITH_AES_256_GCM_SHA384 (0x9d)   WEAK 256
TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 (0xc02b)   Forward Secrecy 128
TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 (0xc02f)   Forward Secrecy 128
TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 (0xc02c)   Forward Secrecy 256
TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (0xc030)   Forward Secrecy 256
TLS_EMPTY_RENEGOTIATION_INFO_SCSV (0xff) -
(1) When a browser supports SSL 2, its SSL 2-only suites are shown only on the very first connection to this site. To see the suites, close all browser windows, then open this exact page directly. Don't refresh.


Protocol Details
Server Name Indication (SNI) Yes
Secure Renegotiation Yes
TLS compression No
Session tickets Yes
OCSP stapling No
Signature algorithms SHA512/RSA, SHA512/DSA, SHA512/ECDSA, SHA384/RSA, SHA384/DSA, SHA384/ECDSA, SHA256/RSA, SHA256/DSA, SHA256/ECDSA, SHA224/RSA, SHA224/DSA, SHA224/ECDSA, SHA1/RSA, SHA1/DSA, SHA1/ECDSA
Named Groups x25519, secp256r1, secp521r1, secp384r1
Next Protocol Negotiation No
Application Layer Protocol Negotiation No
SSL 2 handshake compatibility No

```
ghost commented 5 years ago

@karnadii I'd like to think that we ruled it out and that it is indeed just an IP problem but I can't say for sure due to some differences that persisted even when specifying the same ciphers. :confused:

@VeNoMouS any thoughts?

Mine ![localhost_8000_results html](https://user-images.githubusercontent.com/34285059/56961996-f679a600-6b1a-11e9-977f-cc828ea778aa.png)
Yours ![localhost_8000_that html](https://user-images.githubusercontent.com/34285059/56962001-f8dc0000-6b1a-11e9-8759-1c5ad36e8071.png)
ghost commented 5 years ago

The only thing I can think of, if it's not an IP issue, is that they're checking the signature algorithms:

The TLSv1.2 protocol made the signature algorithm and the hash algorithm that are used for digital signatures an independent attribute. Previously the negotiated cipher suite determined these algorithms.

I'm not sure how to tell node to change that right at the moment. @karnadii Can you try from another system on the same network to rule out an IP issue? That would be awesome if you could.

ghost commented 5 years ago

@karnadii Actually, just try your web browser... If it works there than it's not an IP issue. :laughing: Meaning, if you don't get a CAPTCHA, it works.

karnadii commented 5 years ago

@pro-src it work from my browser, as usual cloudflare 5 second chalenge(no CAPTCHA) then redirect to the https://nanime.in/. I dont have another device.

ghost commented 5 years ago

@karnadii I've been aware of this issue but I didn't think nodejs was affected until now. It's clear that Cloudflare is putting the SSL negotiation under a microscope. Thanks for helping identify this problem!

VeNoMouS commented 5 years ago

works fine with my python cloudscraper

ghost commented 5 years ago

It works fine for me too with either of them and specifying the SSL ciphers didn't help him at all.

karnadii commented 5 years ago

now after I open the url from my browser, the error is invalid URI

/media/karnadi/Boice/Proyek/scraper-api/node_modules/request-promise-core/lib/plumbing.js:130
            throw thrownException;
            ^
RequestError: Error: Invalid URI "https:/nanime.in/". This can be caused by a crappy redirection.
    at onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:148:21)
    at Request.<anonymous> (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:127:7)
    at Object.onceWrapper (events.js:273:13)
    at Request.emit (events.js:182:13)
    at Request.init (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:273:17)
    at Request.RP$initInterceptor [as init] (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request-promise-core/configure/request2.js:45:29)
    at Redirect.onResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/lib/redirect.js:149:11)
    at Request.onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:993:22)
    at ClientRequest.emit (events.js:182:13)
    at HTTPParser.parserOnIncomingClient [as onIncoming] (_http_client.js:546:21)
    at HTTPParser.parserOnHeadersComplete (_http_common.js:109:17)
    at TLSSocket.socketOnData (_http_client.js:432:20)
    at TLSSocket.emit (events.js:182:13)
    at addChunk (_stream_readable.js:283:12)
    at readableAddChunk (_stream_readable.js:264:11)
    at TLSSocket.Readable.push (_stream_readable.js:219:10)
    at TLSWrap.onread (net.js:639:20)
codemanki commented 5 years ago

@karnadii because the URI is incorrect https:/nanime.in/ -> https://nanime.in/

karnadii commented 5 years ago

@codemanki my code is correct 

var uri = "https://nanime.in";
const cloudscraper = require("cloudscraper");
cloudscraper.debug = true;

cloudscraper.get(uri, function(error, response, body) {
  if (error) {
    throw error;
  } else {
    console.log(body);
  }
});

i didnt make a change on the code

VeNoMouS commented 5 years ago

wouldn't the url actually end in /?

var uri = "https://nanime.in/";

As a browser would append the / and cf may be picking up on that..

VeNoMouS commented 5 years ago 

> HTTP/1.1 302 Found
> Date: Tue, 30 Apr 2019 13:11:35 GMT
> Content-Type: text/html; charset=UTF-8
> Transfer-Encoding: chunked
> Connection: keep-alive
> Location: https://n.nanime.in/
> Vary: Accept
> X-Powered-By: EasyEngine 3.7.5
> Expect-CT: max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"
> Server: cloudflare
> CF-RAY: 4cf9c328bf5c1902-AKL
>
<p>Found. Redirecting to <a href="https://n.nanime.in/">https://n.nanime.in/</a></p>< GET / HTTP/1.1
< Host: n.nanime.in
ghost commented 5 years ago

Works either way for me, with or without the ending /

VeNoMouS commented 5 years ago

@karnadii whats your node version? and what is the libssl version its using?

karnadii commented 5 years ago

it is the same, I have tried both with "/" and without. running the same code with different time, and different error

Stacktrace ```    karnadi-PC   …/scraper-api    master  ▓▒░ ░▒▓   10.4   20:04:16  ➜ node a.js REQUEST { requester: { [Function: request] get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: { [Function: Request] super_: [Function], debug: true, defaultProxyHeaderWhiteList: [Array], defaultProxyHeaderExclusiveList: [Array] }, initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: CookieJar { enableLooseMode: true, store: { idx: {} } } }, headers: { Host: Symbol(host), Connection: 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.8', 'Accept-Encoding': 'gzip, deflate' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 3, decodeEmails: false, gzip: true, uri: 'https://nanime.in', callback: [Function: RP$callback], method: 'GET', realEncoding: 'utf8', encoding: null, transform: undefined, simple: true, resolveWithFullResponse: false, transform2xxOnly: false } REQUEST make request https://nanime.in/ REQUEST onRequestResponse https://nanime.in/ 503 { date: 'Tue, 30 Apr 2019 13:05:06 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'set-cookie': [ '__cfduid=da4df305fc9f93e92e89c1b1af5cc96931556629506; expires=Wed, 29-Apr-20 13:05:06 GMT; path=/; domain=.nanime.in; HttpOnly' ], 'x-frame-options': 'SAMEORIGIN', 'cache-control': 'no-cache', 'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"', vary: 'Accept-Encoding', server: 'cloudflare', 'cf-ray': '4cf9b9acb912aec3-KIX' } REQUEST reading response's body REQUEST finish init function https://nanime.in/ REQUEST response end https://nanime.in/ 503 { date: 'Tue, 30 Apr 2019 13:05:06 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'set-cookie': [ '__cfduid=da4df305fc9f93e92e89c1b1af5cc96931556629506; expires=Wed, 29-Apr-20 13:05:06 GMT; path=/; domain=.nanime.in; HttpOnly' ], 'x-frame-options': 'SAMEORIGIN', 'cache-control': 'no-cache', 'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"', vary: 'Accept-Encoding', server: 'cloudflare', 'cf-ray': '4cf9b9acb912aec3-KIX' } REQUEST end event https://nanime.in/ REQUEST has body https://nanime.in/ 14145 REQUEST emitting complete https://nanime.in/ REQUEST { requester: { [Function: request] get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: { [Function: Request] super_: [Function], debug: true, defaultProxyHeaderWhiteList: [Array], defaultProxyHeaderExclusiveList: [Array] }, initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: CookieJar { enableLooseMode: true, store: { idx: { 'nanime.in': { '/': { __cfduid: Cookie="__cfduid=da4df305fc9f93e92e89c1b1af5cc96931556629506; Expires=Wed, 29 Apr 2020 13:05:06 GMT; Domain=nanime.in; Path=/; HttpOnly; hostOnly=false; aAge=4078ms; cAge=4078ms" } } } } } }, headers: { Host: Symbol(host), Connection: 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.8', 'Accept-Encoding': 'gzip, deflate', Referer: 'https://nanime.in/' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 2, decodeEmails: false, gzip: true, uri: 'https://nanime.in/cdn-cgi/l/chk_jschl', callback: [Function: RP$callback], method: 'GET', realEncoding: 'utf8', encoding: null, qs: { s: '6bd6d6db4ca7ea33aa630c51d179a398ee24bad1-1556629506-1800-AXw67/v1fyB9RCrJR+cfXOno19WwBOz0cAznjwTdqzOtw7H79km0Fsk7BL0qvtMy+b0ZGgFeYfxAQAOgzqUW4qSFku1L6W2858HicdmdFFYe/S34hTiDuQkPaAWOU5J0Vw==', jschl_vc: '7b37bbc48fbd1495c2272fd3f2d79ecd', pass: '1556629510.038-1LmAjcSW4d', jschl_answer: '221.2451390938' }, transform: undefined, simple: true, resolveWithFullResponse: false, transform2xxOnly: false } REQUEST make request https://nanime.in/cdn-cgi/l/chk_jschl?s=6bd6d6db4ca7ea33aa630c51d179a398ee24bad1-1556629506-1800-AXw67%2Fv1fyB9RCrJR%2BcfXOno19WwBOz0cAznjwTdqzOtw7H79km0Fsk7BL0qvtMy%2Bb0ZGgFeYfxAQAOgzqUW4qSFku1L6W2858HicdmdFFYe%2FS34hTiDuQkPaAWOU5J0Vw%3D%3D&jschl_vc=7b37bbc48fbd1495c2272fd3f2d79ecd&pass=1556629510.038-1LmAjcSW4d&jschl_answer=221.2451390938 REQUEST onRequestResponse https://nanime.in/cdn-cgi/l/chk_jschl?s=6bd6d6db4ca7ea33aa630c51d179a398ee24bad1-1556629506-1800-AXw67%2Fv1fyB9RCrJR%2BcfXOno19WwBOz0cAznjwTdqzOtw7H79km0Fsk7BL0qvtMy%2Bb0ZGgFeYfxAQAOgzqUW4qSFku1L6W2858HicdmdFFYe%2FS34hTiDuQkPaAWOU5J0Vw%3D%3D&jschl_vc=7b37bbc48fbd1495c2272fd3f2d79ecd&pass=1556629510.038-1LmAjcSW4d&jschl_answer=221.2451390938 302 { date: 'Tue, 30 Apr 2019 13:05:10 GMT', 'content-type': 'text/html', 'content-length': '159', connection: 'keep-alive', location: 'https:/nanime.in/', server: 'cloudflare', 'cf-ray': '4cf9b9ca6de0af17-KIX', 'x-frame-options': 'SAMEORIGIN' } REQUEST redirect https:/nanime.in/ REQUEST redirect to https:/nanime.in/ REQUEST {} /media/karnadi/Boice/Proyek/scraper-api/node_modules/request-promise-core/lib/plumbing.js:130 throw thrownException; ^ RequestError: Error: Invalid URI "https:/nanime.in/". This can be caused by a crappy redirection. at onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:148:21) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:127:7) at Object.onceWrapper (events.js:273:13) at Request.emit (events.js:182:13) at Request.init (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:273:17) at Request.RP$initInterceptor [as init] (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request-promise-core/configure/request2.js:45:29) at Redirect.onResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/lib/redirect.js:149:11) at Request.onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:993:22) at ClientRequest.emit (events.js:182:13) at HTTPParser.parserOnIncomingClient [as onIncoming] (_http_client.js:546:21) at HTTPParser.parserOnHeadersComplete (_http_common.js:109:17) at TLSSocket.socketOnData (_http_client.js:432:20) at TLSSocket.emit (events.js:182:13) at addChunk (_stream_readable.js:283:12) at readableAddChunk (_stream_readable.js:264:11) at TLSSocket.Readable.push (_stream_readable.js:219:10) at TLSWrap.onread (net.js:639:20) zsh: exit 1 node a.js    karnadi-PC   …/scraper-api    master  ▓▒░ ░▒▓   14.0   20:05:12  ➜ node a.js REQUEST { requester: { [Function: request] get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: { [Function: Request] super_: [Function], debug: true, defaultProxyHeaderWhiteList: [Array], defaultProxyHeaderExclusiveList: [Array] }, initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: CookieJar { enableLooseMode: true, store: { idx: {} } } }, headers: { Host: Symbol(host), Connection: 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 3, decodeEmails: false, gzip: true, uri: 'https://nanime.in', callback: [Function: RP$callback], method: 'GET', realEncoding: 'utf8', encoding: null, transform: undefined, simple: true, resolveWithFullResponse: false, transform2xxOnly: false } REQUEST make request https://nanime.in/ REQUEST onRequestResponse https://nanime.in/ 503 { date: 'Tue, 30 Apr 2019 13:10:17 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'set-cookie': [ '__cfduid=d626ca5cedf2364598147ffacb31bf9551556629817; expires=Wed, 29-Apr-20 13:10:17 GMT; path=/; domain=.nanime.in; HttpOnly' ], 'x-frame-options': 'SAMEORIGIN', 'cache-control': 'no-cache', 'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"', vary: 'Accept-Encoding', server: 'cloudflare', 'cf-ray': '4cf9c146cf3faec7-KIX' } REQUEST reading response's body REQUEST finish init function https://nanime.in/ REQUEST response end https://nanime.in/ 503 { date: 'Tue, 30 Apr 2019 13:10:17 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'set-cookie': [ '__cfduid=d626ca5cedf2364598147ffacb31bf9551556629817; expires=Wed, 29-Apr-20 13:10:17 GMT; path=/; domain=.nanime.in; HttpOnly' ], 'x-frame-options': 'SAMEORIGIN', 'cache-control': 'no-cache', 'expect-ct': 'max-age=604800, report-uri="https://report-uri.cloudflare.com/cdn-cgi/beacon/expect-ct"', vary: 'Accept-Encoding', server: 'cloudflare', 'cf-ray': '4cf9c146cf3faec7-KIX' } REQUEST end event https://nanime.in/ REQUEST has body https://nanime.in/ 8725 REQUEST emitting complete https://nanime.in/ REQUEST { requester: { [Function: request] get: [Function], head: [Function], options: [Function], post: [Function], put: [Function], patch: [Function], del: [Function], delete: [Function], jar: [Function], cookie: [Function], defaults: [Function], forever: [Function], Request: { [Function: Request] super_: [Function], debug: true, defaultProxyHeaderWhiteList: [Array], defaultProxyHeaderExclusiveList: [Array] }, initParams: [Function: initParams], debug: [Getter/Setter], bindCLS: [Function: RP$bindCLS] }, jar: RequestJar { _jar: CookieJar { enableLooseMode: true, store: { idx: { 'nanime.in': { '/': { __cfduid: Cookie="__cfduid=d626ca5cedf2364598147ffacb31bf9551556629817; Expires=Wed, 29 Apr 2020 13:10:17 GMT; Domain=nanime.in; Path=/; HttpOnly; hostOnly=false; aAge=4091ms; cAge=4091ms" } } } } } }, headers: { Host: Symbol(host), Connection: 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36', Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate', Referer: 'https://nanime.in/' }, cloudflareMaxTimeout: 30000, followAllRedirects: true, challengesToSolve: 2, decodeEmails: false, gzip: true, uri: 'https://nanime.in/cdn-cgi/l/chk_jschl', callback: [Function: RP$callback], method: 'GET', realEncoding: 'utf8', encoding: null, qs: { s: '4a73c5532dc60acefd2a7a8851355bbbf09114ed-1556629817-1800-AWQeqv99sckrHPGP8K3BA9ktJxy5mVyYBfGAgpIskGK3Wxw5bhOlvXFe5FK9DoRlVuwnv28ASp+qobZpetlzDZqK1SD5cHRO2hP3zcigVOmYVTYRBmhB6rXU+lRi6ED+5Q==', jschl_vc: '99043223a6c023364faaca49579656fd', pass: '1556629821.411-aq2NQAOiBE', jschl_answer: '228.7918033774' }, transform: undefined, simple: true, resolveWithFullResponse: false, transform2xxOnly: false } REQUEST make request https://nanime.in/cdn-cgi/l/chk_jschl?s=4a73c5532dc60acefd2a7a8851355bbbf09114ed-1556629817-1800-AWQeqv99sckrHPGP8K3BA9ktJxy5mVyYBfGAgpIskGK3Wxw5bhOlvXFe5FK9DoRlVuwnv28ASp%2BqobZpetlzDZqK1SD5cHRO2hP3zcigVOmYVTYRBmhB6rXU%2BlRi6ED%2B5Q%3D%3D&jschl_vc=99043223a6c023364faaca49579656fd&pass=1556629821.411-aq2NQAOiBE&jschl_answer=228.7918033774 REQUEST onRequestResponse https://nanime.in/cdn-cgi/l/chk_jschl?s=4a73c5532dc60acefd2a7a8851355bbbf09114ed-1556629817-1800-AWQeqv99sckrHPGP8K3BA9ktJxy5mVyYBfGAgpIskGK3Wxw5bhOlvXFe5FK9DoRlVuwnv28ASp%2BqobZpetlzDZqK1SD5cHRO2hP3zcigVOmYVTYRBmhB6rXU%2BlRi6ED%2B5Q%3D%3D&jschl_vc=99043223a6c023364faaca49579656fd&pass=1556629821.411-aq2NQAOiBE&jschl_answer=228.7918033774 403 { date: 'Tue, 30 Apr 2019 13:10:22 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'cf-chl-bypass': '1', 'cache-control': 'max-age=2', expires: 'Tue, 30 Apr 2019 13:10:24 GMT', 'x-frame-options': 'SAMEORIGIN', server: 'cloudflare', 'cf-ray': '4cf9c1645953aec3-KIX', 'content-encoding': 'gzip' } REQUEST reading response's body REQUEST finish init function https://nanime.in/cdn-cgi/l/chk_jschl?s=4a73c5532dc60acefd2a7a8851355bbbf09114ed-1556629817-1800-AWQeqv99sckrHPGP8K3BA9ktJxy5mVyYBfGAgpIskGK3Wxw5bhOlvXFe5FK9DoRlVuwnv28ASp%2BqobZpetlzDZqK1SD5cHRO2hP3zcigVOmYVTYRBmhB6rXU%2BlRi6ED%2B5Q%3D%3D&jschl_vc=99043223a6c023364faaca49579656fd&pass=1556629821.411-aq2NQAOiBE&jschl_answer=228.7918033774 REQUEST response end https://nanime.in/cdn-cgi/l/chk_jschl?s=4a73c5532dc60acefd2a7a8851355bbbf09114ed-1556629817-1800-AWQeqv99sckrHPGP8K3BA9ktJxy5mVyYBfGAgpIskGK3Wxw5bhOlvXFe5FK9DoRlVuwnv28ASp%2BqobZpetlzDZqK1SD5cHRO2hP3zcigVOmYVTYRBmhB6rXU%2BlRi6ED%2B5Q%3D%3D&jschl_vc=99043223a6c023364faaca49579656fd&pass=1556629821.411-aq2NQAOiBE&jschl_answer=228.7918033774 403 { date: 'Tue, 30 Apr 2019 13:10:22 GMT', 'content-type': 'text/html; charset=UTF-8', 'transfer-encoding': 'chunked', connection: 'close', 'cf-chl-bypass': '1', 'cache-control': 'max-age=2', expires: 'Tue, 30 Apr 2019 13:10:24 GMT', 'x-frame-options': 'SAMEORIGIN', server: 'cloudflare', 'cf-ray': '4cf9c1645953aec3-KIX', 'content-encoding': 'gzip' } REQUEST end event https://nanime.in/cdn-cgi/l/chk_jschl?s=4a73c5532dc60acefd2a7a8851355bbbf09114ed-1556629817-1800-AWQeqv99sckrHPGP8K3BA9ktJxy5mVyYBfGAgpIskGK3Wxw5bhOlvXFe5FK9DoRlVuwnv28ASp%2BqobZpetlzDZqK1SD5cHRO2hP3zcigVOmYVTYRBmhB6rXU%2BlRi6ED%2B5Q%3D%3D&jschl_vc=99043223a6c023364faaca49579656fd&pass=1556629821.411-aq2NQAOiBE&jschl_answer=228.7918033774 REQUEST has body https://nanime.in/cdn-cgi/l/chk_jschl?s=4a73c5532dc60acefd2a7a8851355bbbf09114ed-1556629817-1800-AWQeqv99sckrHPGP8K3BA9ktJxy5mVyYBfGAgpIskGK3Wxw5bhOlvXFe5FK9DoRlVuwnv28ASp%2BqobZpetlzDZqK1SD5cHRO2hP3zcigVOmYVTYRBmhB6rXU%2BlRi6ED%2B5Q%3D%3D&jschl_vc=99043223a6c023364faaca49579656fd&pass=1556629821.411-aq2NQAOiBE&jschl_answer=228.7918033774 9852 REQUEST emitting complete https://nanime.in/cdn-cgi/l/chk_jschl?s=4a73c5532dc60acefd2a7a8851355bbbf09114ed-1556629817-1800-AWQeqv99sckrHPGP8K3BA9ktJxy5mVyYBfGAgpIskGK3Wxw5bhOlvXFe5FK9DoRlVuwnv28ASp%2BqobZpetlzDZqK1SD5cHRO2hP3zcigVOmYVTYRBmhB6rXU%2BlRi6ED%2B5Q%3D%3D&jschl_vc=99043223a6c023364faaca49579656fd&pass=1556629821.411-aq2NQAOiBE&jschl_answer=228.7918033774 /media/karnadi/Boice/Proyek/scraper-api/node_modules/request-promise-core/lib/plumbing.js:130 throw thrownException; ^ CaptchaError: captcha at validate (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:231:11) at onCloudflareResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:192:5) at onRequestResponse (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:171:5) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/cloudscraper/index.js:132:7) at Object.onceWrapper (events.js:273:13) at Request.emit (events.js:182:13) at Request. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1161:10) at Request.emit (events.js:182:13) at Gunzip. (/media/karnadi/Boice/Proyek/scraper-api/node_modules/request/request.js:1083:12) at Object.onceWrapper (events.js:273:13) at Gunzip.emit (events.js:187:15) at endReadableNT (_stream_readable.js:1081:12) at process._tickCallback (internal/process/next_tick.js:63:19) zsh: exit 1 node a.js    karnadi-PC   …/scraper-api    master  ▓▒░ ░▒▓   8.4   20:10:22  ➜ ```
ghost commented 5 years ago

Probably really was just a bogus redirect: https://github.com/request/request/blob/master/request.js#L269

karnadii commented 5 years ago

node v10.8.0, how to find libss version?

ghost commented 5 years ago

@karnadii node -e 'console.log(process.versions)'

karnadii commented 5 years ago

oh god, i switch to node v12.1.0 and now it work...

ghost commented 5 years ago

This is my process info:

{ http_parser: '2.8.0',
  node: '10.15.2',
  v8: '6.8.275.32-node.12',
  uv: '1.24.1',
  zlib: '1.2.11',
  ares: '1.14.0',
  modules: '64',
  nghttp2: '1.37.0',
  napi: '3',
  openssl: '1.1.1b',
  icu: '63.1',
  unicode: '11.0',
  cldr: '34.0',
  tz: '2018e' }

I don't have brotli installed.

ghost commented 5 years ago

@karnadii Please run this one last time so we can do a comparison.

CLI flag from earlier https://github.com/codemanki/cloudscraper/issues/211#issuecomment-487930595 And code from earlier https://github.com/codemanki/cloudscraper/issues/211#issuecomment-487932489

I'm really glad that you got it working :)

karnadii commented 5 years ago

not working v10{ http_parser: '2.8.0', node: '10.8.0', v8: '6.7.288.49-node.19', uv: '1.22.0', zlib: '1.2.11', ares: '1.14.0', modules: '64', nghttp2: '1.32.0', napi: '3', openssl: '1.1.0h', icu: '62.1', unicode: '11.0', cldr: '33.1', tz: '2018e' }

working v12 { node: '12.1.0', v8: '7.4.288.21-node.16', uv: '1.28.0', zlib: '1.2.11', brotli: '1.0.7', ares: '1.15.0', modules: '72', nghttp2: '1.38.0', napi: '4', llhttp: '1.1.1', http_parser: '2.8.0', openssl: '1.1.1b', cldr: '35.1', icu: '64.2', tz: '2019a', unicode: '12.1' }

karnadii commented 5 years ago

@pro-src

results.html ```html Qualys SSL Labs - Projects / SSL Client Test
SSL/TLS Capabilities of Your Browser
User Agent: Mozilla/5.0 (Linux; Android 9; Pixel 3 XL Build/PD1A.180720.030) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.85 Mobile Safari/537.36

Other User Agents »

Protocol Support
Please wait, checking protocol support...
Logjam Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the Logjam attack, please go to weakdh.org.
To test manually, click here. Your user agent is not vulnerable if it fails to connect to the site.
FREAK Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the FREAK attack, please go to www.freakattack.com.
To test manually, click here. Your user agent is not vulnerable if it fails to connect to the site.
POODLE Vulnerability
Please wait, checking if your user agent is vulnerable...
For more information about the POODLE attack, please read this blog post.
SSL 2 Protocol Support
Your user agent supports SSL 2. You should upgrade.
SSL 2 is a very old, obsolete, and insecure version of the SSL protocol. You can usually disable this protocol version in configuration, but modern clients don't support it at all. This really means that you should upgrade your software to a better version.
iOS and OS X TLS Authentication Vulnerability
Please wait, checking if your user agent is vulnerable...
To test manually, click here. If your user agent refuses to connect, you are not vulnerable. This test requires a connection to the SSL Labs server on port 10443. A strict outbound firewall might interfere. You should test Safari running on iOS or OS X. Chrome and Firefox are not vulnerable, even when running on a vulnerable operating system. MORE »
Protocol Features
Protocols
TLS 1.3 No
TLS 1.2 Yes*
TLS 1.1 Yes*
TLS 1.0 Yes*
SSL 3 Yes*
SSL 2 No
(*) Without JavaScript, this test reliably detects only the highest supported protocol.


Cipher Suites (in order of preference)
TLS_RSA_WITH_AES_128_GCM_SHA256 (0x9c)   WEAK 128
TLS_RSA_WITH_AES_256_GCM_SHA384 (0x9d)   WEAK 256
TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 (0xc02b)   Forward Secrecy 128
TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 (0xc02f)   Forward Secrecy 128
TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 (0xc02c)   Forward Secrecy 256
TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 (0xc030)   Forward Secrecy 256
TLS_EMPTY_RENEGOTIATION_INFO_SCSV (0xff) -
(1) When a browser supports SSL 2, its SSL 2-only suites are shown only on the very first connection to this site. To see the suites, close all browser windows, then open this exact page directly. Don't refresh.


Protocol Details
Server Name Indication (SNI) Yes
Secure Renegotiation Yes
TLS compression No
Session tickets Yes
OCSP stapling No
Signature algorithms SHA256/ECDSA, SHA384/ECDSA, SHA512/ECDSA, Ed25519, Ed448, Unknown (0x8)/Unknown (0x9), Unknown (0x8)/Unknown (0xa), Unknown (0x8)/Unknown (0xb), RSA_PSS_SHA256, RSA_PSS_SHA384, RSA_PSS_SHA512, SHA256/RSA, SHA384/RSA, SHA512/RSA, SHA224/ECDSA, SHA1/ECDSA, SHA224/RSA, SHA1/RSA, SHA224/DSA, SHA1/DSA, SHA256/DSA, SHA384/DSA, SHA512/DSA
Named Groups x25519, secp256r1, x448, secp521r1, secp384r1
Next Protocol Negotiation No
Application Layer Protocol Negotiation No
SSL 2 handshake compatibility No

```
karnadii commented 5 years ago

thanks guys :D

ghost commented 5 years ago

Here is a screenshot of the html you just pasted, it matches mine exactly now: localhost_8000_test html

I think it's pretty conclusive that the problem was indeed the signature algorithms and that upgrading your nodejs (and it's build of openssl) solved the problem.

So it'd be nice if we could find a way to override that so other people won't face similar problems.

ghost commented 5 years ago

Further proof and a very bad work around:

var cloudscraper = require('cloudscraper');
cloudscraper.get({ agentOptions: { secureProtocol: 'TLSv1_1_method' }, uri: 'https://nanime.in' }).then(console.log)

Works when running the same process:

{ http_parser: '2.8.0',
  node: '10.8.0',
  v8: '6.7.288.49-node.19',
  uv: '1.22.0',
  zlib: '1.2.11',
  ares: '1.14.0',
  modules: '64',
  nghttp2: '1.32.0',
  napi: '3',
  openssl: '1.1.0h',
  icu: '62.1',
  unicode: '11.0',
  cldr: '33.1',
  tz: '2018e' }

The TLSv1.2 protocol made the signature algorithm and the hash algorithm that are used for digital signatures an independent attribute. Previously the negotiated cipher suite determined these algorithms.

https://tools.ietf.org/html/rfc5246#section-7.4.1.4.1

ghost commented 5 years ago

@karnadii Can I get you to try this out on your system to confirm a fix?

Code snippet ```js if (process.version !== 'v10.8.0') { throw Error('Please run this test with the affected node version.'); } const ciphers = [ 'AES128-GCM-SHA256', 'AES128-SHA', 'AES128-SHA256', 'AES256-GCM-SHA384', 'AES256-SHA', 'AES256-SHA256', 'DHE-PSK-AES128-CBC-SHA', 'DHE-PSK-AES128-CBC-SHA256', 'DHE-PSK-AES128-GCM-SHA256', 'DHE-PSK-AES256-CBC-SHA', 'DHE-PSK-AES256-CBC-SHA384', 'DHE-PSK-AES256-GCM-SHA384', 'DHE-PSK-CHACHA20-POLY1305', 'DHE-RSA-AES128-GCM-SHA256', 'DHE-RSA-AES128-SHA', 'DHE-RSA-AES128-SHA256', 'DHE-RSA-AES256-GCM-SHA384', 'DHE-RSA-AES256-SHA', 'DHE-RSA-AES256-SHA256', 'DHE-RSA-CHACHA20-POLY1305', 'ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-AES128-SHA', 'ECDHE-ECDSA-AES128-SHA256', 'ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES256-SHA', 'ECDHE-ECDSA-AES256-SHA384', 'ECDHE-ECDSA-CHACHA20-POLY1305', 'ECDHE-PSK-AES128-CBC-SHA', 'ECDHE-PSK-AES128-CBC-SHA256', 'ECDHE-PSK-AES256-CBC-SHA', 'ECDHE-PSK-AES256-CBC-SHA384', 'ECDHE-PSK-CHACHA20-POLY1305', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-SHA', 'ECDHE-RSA-AES128-SHA256', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-SHA', 'ECDHE-RSA-AES256-SHA384', 'ECDHE-RSA-CHACHA20-POLY1305', 'PSK-AES128-CBC-SHA', 'PSK-AES128-CBC-SHA256', 'PSK-AES128-GCM-SHA256', 'PSK-AES256-CBC-SHA', 'PSK-AES256-CBC-SHA384', 'PSK-AES256-GCM-SHA384', 'PSK-CHACHA20-POLY1305', 'RSA-PSK-AES128-CBC-SHA', 'RSA-PSK-AES128-CBC-SHA256', 'RSA-PSK-AES128-GCM-SHA256', 'RSA-PSK-AES256-CBC-SHA', 'RSA-PSK-AES256-CBC-SHA384', 'RSA-PSK-AES256-GCM-SHA384', 'RSA-PSK-CHACHA20-POLY1305', 'SRP-AES-128-CBC-SHA', 'SRP-AES-256-CBC-SHA', 'SRP-RSA-AES-128-CBC-SHA', 'SRP-RSA-AES-256-CBC-SHA', 'TLS_AES_128_GCM_SHA256', 'TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256' ].join(':'); const cloudscraper = require('cloudscraper').defaults({ agentOptions: { ciphers } }); cloudscraper.get('https://nanime.in').then(console.log, console.error); ```
karnadii commented 5 years ago

@pro-src it works

ghost commented 5 years ago

Thank you, I'll send a PR soon so we can get this fix released.

karnadii commented 5 years ago

@pro-src i have try using node v10.8.0, v10.15.3 and v12.1.0, only v12.1.0 give me succes, v10.15.3 also give me error CaptchaError: captcha except using ur work around using agenoptions

ghost commented 5 years ago

Thanks for letting me know, I'll be testing as many node versions as possible before sending an official PR for peer review.

ghost commented 5 years ago

Upon reexamination, the TLSv1.3 ciphers don't appear to be solving the problem after all. Rather the ciphers are changed in a non-intuitive way. The list of ciphers obtained from require('tls').getCiphers() is not equivalent to the list that is generated internally from require('tls').DEFAULT_CIPHERS.

The difference doesn't appear to disagree with what's been observed in a similar issue for python: https://github.com/Anorov/cloudflare-scrape/issues/235

While the current code is solving the problem for users using old openssl versions, it's possible that some users might still encounter this problem even on the most recent version of openssl.

Thanks to @lukastribus for causing me to question the effect of the TLSv1.3 ciphers.

ghost commented 5 years ago

I do believe that this article explains why Cloudflare made these changes recently: https://community.cloudflare.com/t/tls-1-0-to-1-2-why/83643

karnadii commented 5 years ago

hello, I don't know why but I still get error when using cloudscraper with nodejs version lower than 12. I am using the newest version 4.1.4. using nodejs v10.16.0. I get this error. and switching to the latest stable v12.9.1 fix this error.

I believe I am not the only one facing this error, the error message is not helpful either, so at least inform user to update to newest version of node if they having the same error. maybe #251 also related to this error, and the user is clueless of what is the error.

stacktrace 

/home/karnadi/Projects/nodejs/scraper/node_modules/request-promise-core/lib/plumbing.js:130
            throw thrownException;
            ^
CaptchaError: captcha
    at validateResponse (/home/karnadi/Projects/nodejs/scraper/node_modules/cloudscraper/index.js:232:11)
    at onCloudflareResponse (/home/karnadi/Projects/nodejs/scraper/node_modules/cloudscraper/index.js:195:5)
    at onRequestResponse (/home/karnadi/Projects/nodejs/scraper/node_modules/cloudscraper/index.js:178:5)
    at Request.<anonymous> (/home/karnadi/Projects/nodejs/scraper/node_modules/cloudscraper/index.js:137:7)
    at Object.onceWrapper (events.js:286:20)
    at Request.emit (events.js:198:13)
    at Request.<anonymous> (/home/karnadi/Projects/nodejs/scraper/node_modules/request/request.js:1161:10)
    at Request.emit (events.js:198:13)
    at Gunzip.<anonymous> (/home/karnadi/Projects/nodejs/scraper/node_modules/request/request.js:1083:12)
    at Object.onceWrapper (events.js:286:20)
    at Gunzip.emit (events.js:203:15)
    at endReadableNT (_stream_readable.js:1129:12)
    at process._tickCallback (internal/process/next_tick.js:63:19)
Emitted 'error' event at:
    at /home/karnadi/Projects/nodejs/scraper/node_modules/x-ray/lib/stream.js:55:23
    at next (/home/karnadi/Projects/nodejs/scraper/node_modules/x-ray/index.js:95:25)
    at /home/karnadi/Projects/nodejs/scraper/node_modules/x-ray/index.js:66:27
    at /home/karnadi/Projects/nodejs/scraper/node_modules/x-ray/index.js:207:23
    at _done (/home/karnadi/Projects/nodejs/scraper/node_modules/enqueue/index.js:78:20)
    at _once (/home/karnadi/Projects/nodejs/scraper/node_modules/enqueue/index.js:93:15)
    at result (/home/karnadi/Projects/nodejs/scraper/node_modules/x-ray-crawler/lib/index.js:99:23)
    at /home/karnadi/Projects/nodejs/scraper/node_modules/wrap-fn/index.js:121:18
    at Request._rp_callbackOrig (/home/karnadi/Projects/nodejs/scraper/driver.js:16:11)
    at Request.plumbing.callback (/home/karnadi/Projects/nodejs/scraper/node_modules/request-promise-core/lib/plumbing.js:76:39)
    at Request.RP$callback [as _callback] (/home/karnadi/Projects/nodejs/scraper/node_modules/request-promise-core/lib/plumbing.js:46:31)
    at self.callback (/home/karnadi/Projects/nodejs/scraper/node_modules/request/request.js:185:22)
    at onCloudflareResponse (/home/karnadi/Projects/nodejs/scraper/node_modules/cloudscraper/index.js:202:12)
    at onRequestResponse (/home/karnadi/Projects/nodejs/scraper/node_modules/cloudscraper/index.js:178:5)
    [... lines matching original stack trace ...]
    at Gunzip.emit (events.js:203:15)
zsh: exit 1     node novels.js