Error: Failed to set ECDH curve

[hannesschaletzky]

Hi together 😊

I am not sure whether or not this is an issue with got-scraping or rather one with the server I am scraping from (Apache/2.4.38; Debian) ... Every ~15th request to the server fails with the error Failed to set ECDH curve with code ERR_CRYPTO_OPERATION_FAILED. The SSL report states that the handshake Java 6u45 is failing. Does this correlate?

I looked up ECDH curve but it does not give me any hint and I don't have a lot of knowledge when it comes to topics like http request forging, cryptography, keys, etc. Any help in any directions, or resources to read into are highly appreciated.

Since I need at least like 200k requests to the server, I would really love to fix this any way I can. Thanks in advance 🙌🏻

Request:

import { gotScraping } from 'got-scraping'

export function scrape(url: string): Promise<string> {
  return new Promise((resolve) => {
    gotScraping
      .get(url)
      .then(({ body }) => resolve(body as string))
      .catch((err) => {
        console.error(err)
      })
  })
}

Error:

RequestError: Failed to set ECDH curve
    at Request._beforeError (<mypath>/node_modules/got-cjs/dist/source/core/index.js:324:21)
    at Request.flush (<mypath>/node_modules/got-cjs/dist/source/core/index.js:313:18)
    at processTicksAndRejections (node:internal/process/task_queues:95:5)
    at configSecureContext (node:internal/tls/secure-context:239:11)
    at Object.createSecureContext (node:_tls_common:117:3)
    at Object.connect (node:_tls_wrap:1629:48)
    at Agent.createConnection (node:https:150:22)
    at Agent.createSocket (node:_http_agent:350:26)
    at Agent.addRequest (node:_http_agent:297:10)
    at TransformHeadersAgent.addRequest (<mypath>/node_modules/got-scraping/src/agent/wrapped-agent.ts:19:20)
    at TransformHeadersAgent.addRequest (<mypath>/node_modules/got-scraping/src/agent/transform-headers-agent.ts:95:22)
    at new ClientRequest (node:_http_client:335:16)
    at Object.request (node:http:97:10) {
  input: undefined,
  code: 'ERR_CRYPTO_OPERATION_FAILED',
  timings: undefined,
  options: {
    request: [Function (anonymous)],
    agent: {
      http: TransformHeadersAgent { agent: [Agent] },
      https: TransformHeadersAgent { agent: [Agent] },
      http2: undefined
    },
    h2session: undefined,
    decompress: true,
    timeout: {
      connect: undefined,
      lookup: undefined,
      read: undefined,
      request: 60000,
      response: undefined,
      secureConnect: undefined,
      send: undefined,
      socket: undefined
    },
    prefixUrl: '',
    body: undefined,
    form: undefined,
    json: undefined,
    cookieJar: undefined,
    ignoreInvalidCookies: false,
    searchParams: undefined,
    dnsLookup: undefined,
    dnsCache: undefined,
    context: {
      headerGenerator: HeaderGenerator {
        globalOptions: [Object],
        browserListQuery: undefined,
        inputGeneratorNetwork: [BayesianNetwork],
        headerGeneratorNetwork: [BayesianNetwork],
        uniqueBrowsers: [Array],
        headersOrder: [Object]
      },
      useHeaderGenerator: true,
      insecureHTTPParser: true,
      sessionData: undefined
    },
    hooks: {
      init: [
        [Function: optionsValidationHandler],
        [Function: customOptionsHook]
      ],
      beforeRequest: [
        [Function: insecureParserHook],
        [Function: sessionDataHook],
        [Function: http2Hook],
        [AsyncFunction: proxyHook],
        [AsyncFunction: browserHeadersHook],
        [Function: tlsHook]
      ],
      beforeError: [],
      beforeRedirect: [ [Function: refererHook] ],
      beforeRetry: [],
      afterResponse: []
    },
    followRedirect: true,
    maxRedirects: 10,
    cache: undefined,
    throwHttpErrors: false,
    username: '',
    password: '',
    http2: true,
    allowGetBody: false,
    headers: {
      'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0',
      accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
      'accept-language': 'en-US',
      'accept-encoding': 'gzip, deflate, br',
      dnt: '1',
      connection: 'keep-alive',
      'upgrade-insecure-requests': '1',
      'sec-fetch-mode': 'navigate',
      'sec-fetch-dest': 'document',
      'sec-fetch-site': 'same-site',
      'sec-fetch-user': '?1'
    },
    methodRewriting: false,
    dnsLookupIpVersion: undefined,
    parseJson: [Function: parse],
    stringifyJson: [Function: stringify],
    retry: {
      limit: 0,
      methods: [ 'GET', 'PUT', 'HEAD', 'DELETE', 'OPTIONS', 'TRACE' ],
      statusCodes: [
        408, 413, 429, 500,
        502, 503, 504, 521,
        522, 524
      ],
      errorCodes: [
        'ETIMEDOUT',
        'ECONNRESET',
        'EADDRINUSE',
        'ECONNREFUSED',
        'EPIPE',
        'ENOTFOUND',
        'ENETUNREACH',
        'EAI_AGAIN'
      ],
      maxRetryAfter: undefined,
      calculateDelay: [Function: calculateDelay],
      backoffLimit: Infinity,
      noise: 100
    },
    localAddress: undefined,
    method: 'GET',
    createConnection: undefined,
    cacheOptions: {
      shared: undefined,
      cacheHeuristic: undefined,
      immutableMinTimeToLive: undefined,
      ignoreCargoCult: undefined
    },
    https: {
      alpnProtocols: undefined,
      rejectUnauthorized: false,
      checkServerIdentity: undefined,
      certificateAuthority: undefined,
      key: undefined,
      certificate: undefined,
      passphrase: undefined,
      pfx: undefined,
      ciphers: 'TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA:DES-CBC3-SHA',
      honorCipherOrder: undefined,
      minVersion: 'TLSv1.2',
      maxVersion: 'TLSv1.3',
      signatureAlgorithms: 'ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:ecdsa_secp521r1_sha512:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384:rsa_pss_rsae_sha512:rsa_pkcs1_sha256:rsa_pkcs1_sha384:rsa_pkcs1_sha512:ECDSA+SHA1:rsa_pkcs1_sha1',
      tlsSessionLifetime: undefined,
      dhparam: undefined,
      ecdhCurve: 'X25519:prime256v1:secp384r1:secp521r1:ffdhe2048:ffdhe3072',
      certificateRevocationLists: undefined
    },
    encoding: undefined,
    resolveBodyOnly: false,
    isStream: false,
    responseType: 'text',
    url: URL {
      href: "theURLiAmScraping"
    },
    pagination: {
      transform: [Function: transform],
      paginate: [Function: paginate],
      filter: [Function: filter],
      shouldContinue: [Function: shouldContinue],
      countLimit: Infinity,
      backoff: 0,
      requestLimit: 10000,
      stackAllItems: false
    },
    setHost: true,
    maxHeaderSize: undefined,
    signal: undefined,
    enableUnixSockets: true
  }
}
<mypath>/node_modules/got-cjs/dist/source/core/index.js:324
            error = new errors_js_1.RequestError(error.message, error, this);
                    ^
RequestError: Failed to set ECDH curve
    at Request._beforeError (<mypath>/node_modules/got-cjs/dist/source/core/index.js:324:21)
    at Request.flush (<mypath>/node_modules/got-cjs/dist/source/core/index.js:313:18)
    at processTicksAndRejections (node:internal/process/task_queues:95:5)
    at configSecureContext (node:internal/tls/secure-context:239:11)
    at Object.createSecureContext (node:_tls_common:117:3)
    at Object.connect (node:_tls_wrap:1629:48)
    at Agent.createConnection (node:https:150:22)
    at Agent.createSocket (node:_http_agent:350:26)
    at Agent.addRequest (node:_http_agent:297:10)
    at TransformHeadersAgent.addRequest (<mypath>/node_modules/got-scraping/src/agent/wrapped-agent.ts:19:20)
    at TransformHeadersAgent.addRequest (<mypath>/node_modules/got-scraping/src/agent/transform-headers-agent.ts:95:22)
    at new ClientRequest (node:_http_client:335:16)
    at Object.request (node:http:97:10) {
  input: undefined,
  code: 'ERR_CRYPTO_OPERATION_FAILED',
  timings: undefined,
  options: Options {
    _unixOptions: {
      insecureHTTPParser: true,
      secureOptions: 524304,
      requestOCSP: true
    },
    _internals: {
      request: [Function (anonymous)],
      agent: [Object],
      h2session: undefined,
      decompress: true,
      timeout: [Object],
      prefixUrl: '',
      body: undefined,
      form: undefined,
      json: undefined,
      cookieJar: undefined,
      ignoreInvalidCookies: false,
      searchParams: undefined,
      dnsLookup: undefined,
      dnsCache: undefined,
      context: [Object],
      hooks: [Object],
      followRedirect: true,
      maxRedirects: 10,
      cache: undefined,
      throwHttpErrors: false,
      username: '',
      password: '',
      http2: true,
      allowGetBody: false,
      headers: [Object],
      methodRewriting: false,
      dnsLookupIpVersion: undefined,
      parseJson: [Function: parse],
      stringifyJson: [Function: stringify],
      retry: [Object],
      localAddress: undefined,
      method: 'GET',
      createConnection: undefined,
      cacheOptions: [Object],
      https: [Object],
      encoding: undefined,
      resolveBodyOnly: false,
      isStream: false,
      responseType: 'text',
      url: [URL],
      pagination: [Object],
      setHost: true,
      maxHeaderSize: undefined,
      signal: undefined,
      enableUnixSockets: true
    },
    _merging: false,
    _init: [ [Object], [Object] ]
  }
}

[dbalders]

I was having this running on Node v18. Switched down to Node v16 and it went away.

[petrcherni]

I had the same issue and in my case it didn't work with node 19.8.1. Going down to node v.18.12.1 helped.