sindresorhus / got

🌐 Human-friendly and powerful HTTP request library for Node.js
MIT License
14.27k stars 935 forks source link

RequestError: Invalid character in header content ["referer"] #2159

Closed chahakshahcs5 closed 2 years ago

chahakshahcs5 commented 2 years ago

Describe the bug

RequestError: Invalid character in header content ["referer"]

When try to fetch website28.nicepage.io got below error

RequestError: Invalid character in header content ["referer"]
    at Request._beforeError (file:///D:/Personal/Projects/nicepage-template/node_modules/got/dist/source/core/index.js:295:21)
    at Request.flush (file:///D:/Personal/Projects/nicepage-template/node_modules/got/dist/source/core/index.js:284:18)
    at runMicrotasks (<anonymous>)
    at ClientRequest.setHeader (node:_http_outgoing:579:3)
    at new ClientRequest (node:_http_client:256:14)
    at request (node:https:353:10)
    at Request._makeRequest (file:///D:/Personal/Projects/nicepage-template/node_modules/got/dist/source/core/index.js:947:37)
    at Request.flush (file:///D:/Personal/Projects/nicepage-template/node_modules/got/dist/source/core/index.js:270:24)
    at runMicrotasks (<anonymous>)
    at processTicksAndRejections (node:internal/process/task_queues:96:5) {
  input: undefined,
  code: 'ERR_INVALID_CHAR',
  timings: undefined,
  options: {
    request: undefined,
    agent: { http: undefined, https: undefined, http2: undefined },
    h2session: undefined,
    decompress: true,
    timeout: {
      connect: undefined,
      lookup: undefined,
      read: undefined,
      request: undefined,
      response: undefined,
      secureConnect: undefined,
      send: undefined,
      socket: undefined
    },
    prefixUrl: '',
    body: undefined,
    form: undefined,
    json: undefined,
    cookieJar: undefined,
    ignoreInvalidCookies: false,
    searchParams: undefined,
    dnsLookup: undefined,
    dnsCache: undefined,
    context: {},
    hooks: {
      init: [],
      beforeRequest: [],
      beforeError: [],
      beforeRedirect: [],
      beforeRetry: [],
      afterResponse: []
    },
    followRedirect: true,
    maxRedirects: 10,
    cache: undefined,
    throwHttpErrors: false,
    username: '',
    password: '',
    http2: false,
    allowGetBody: false,
    headers: {
      'user-agent': 'nicepage-template/1.0.0 (https://github.com/website-scraper/node-website-scraper)',
      referer: 'https://website27.nicepage.io/Главная.html',
      'accept-encoding': 'gzip, deflate, br'
    },
    methodRewriting: false,
    dnsLookupIpVersion: undefined,
    parseJson: [Function: parse],
    stringifyJson: [Function: stringify],
    retry: {
      limit: 2,
      methods: [ 'GET', 'PUT', 'HEAD', 'DELETE', 'OPTIONS', 'TRACE' ],
      statusCodes: [
        408, 413, 429, 500,
        502, 503, 504, 521,
        522, 524
      ],
      errorCodes: [
        'ETIMEDOUT',
        'ECONNRESET',
        'EADDRINUSE',
        'ECONNREFUSED',
        'EPIPE',
        'ENOTFOUND',
        'ENETUNREACH',
        'EAI_AGAIN'
      ],
      maxRetryAfter: undefined,
      calculateDelay: [Function: calculateDelay],
      backoffLimit: Infinity,
      noise: 100
    },
    localAddress: undefined,
    method: 'GET',
    createConnection: undefined,
    cacheOptions: {
      shared: undefined,
      cacheHeuristic: undefined,
      immutableMinTimeToLive: undefined,
      ignoreCargoCult: undefined
    },
    https: {
      alpnProtocols: undefined,
      rejectUnauthorized: false,
      checkServerIdentity: undefined,
      certificateAuthority: undefined,
      key: undefined,
      certificate: undefined,
      passphrase: undefined,
      pfx: undefined,
      ciphers: undefined,
      honorCipherOrder: undefined,
      minVersion: undefined,
      maxVersion: undefined,
      signatureAlgorithms: undefined,
      tlsSessionLifetime: undefined,
      dhparam: undefined,
      ecdhCurve: undefined,
      certificateRevocationLists: undefined
    },
    encoding: undefined,
    resolveBodyOnly: false,
    isStream: false,
    responseType: 'buffer',
    url: URL {
      href: 'https://website28.nicepage.io/',
      origin: 'https://website28.nicepage.io',
      protocol: 'https:',
      username: '',
      password: '',
      host: 'website28.nicepage.io',
      hostname: 'website28.nicepage.io',
      port: '',
      pathname: '/',
      search: '',
      searchParams: URLSearchParams {},
      hash: ''
    },
    pagination: {
      transform: [Function: transform],
      paginate: [Function: paginate],
      filter: [Function: filter],
      shouldContinue: [Function: shouldContinue],
      countLimit: Infinity,
      backoff: 0,
      requestLimit: 10000,
      stackAllItems: false
    },
    setHost: true,
    maxHeaderSize: undefined,
    signal: undefined,
    enableUnixSockets: true
  }
}

Actual behavior

...

Expected behavior

...

Code to reproduce

...

Checklist

szmarczak commented 2 years ago

You get the same with the native client: http.get({ headers: { referer: 'https://website27.nicepage.io/Главная.html' } })

townechow commented 1 year ago

you should use encodeURI() encode your url and any others request headers value~ ༼ つ ◕_◕ ༽つ