apify / got-scraping

HTTP client made for scraping based on got.
422 stars 32 forks source link

Random ERROR read ECONNRESET #121

Closed Vance-ng-vn closed 6 months ago

Vance-ng-vn commented 6 months ago

image

The server's network is good, I checked it continuously:

const isOnline = async(...args) => (await import('is-online'));
setInterval(async ()=> await isOnline().then(online => {
    //if(online) console.log(`Server is Online!`);
    if(!online) console.warn(`Server is offline!`);
}).catch(err => console.error(err)), 1000);

And here is my code:

let gotConfig = {
    headerGeneratorOptions:{
        browsers: [
            {
                name: 'firefox',
                minVersion: 102,
                maxVersion: 120
            }
        ],
        devices: ['desktop'],
        operatingSystems: ['linux']
    },
    timeout: {
        request: 15000
    },
    retry: {
        limit: 2,
        calculateDelay: ({attemptCount, retryOptions, error}) => {
            if(attemptCount >= retryOptions.limit) return 0;
            if(error.code == 'ECONNRESET') return 1000; //unknow error ...

            let response = err.response;
            if(response) {
                if(response.statusCode == 429) return 1000; //too many request
                if([400,401,403,404,408,413,500,502,503,504,521,522,524].includes(response.statusCode)) return 0;
            }

            //default
            return 500;
        }
    },
    http2: true
}

let gotScraping;

const import_got = async () => (await import('got-scraping')).gotScraping.extend(gotConfig);

import_got().then(async got => {
    gotScraping = got
    //console.log(gotScraping)
    //console.log(await got.get('https://duckduckgo.com').then(res => res))
});

const got = () => gotScraping;

module.exports = got;
const res = await got().get(url).catch(err => console.error('ERROR', {
      url: url,
      statusCode: err.response?.statusCode,
      stacktrace: err.stack
}));

docker: alpine:3.17.5 nodejs: 18.18.2

got-scraping: 4.0.0/4.0.1/4.0.2

Vance-ng-vn commented 6 months ago

It seems like the error disappeared when I replaced the configs like this: (and I assume that's http2: false )

let gotConfig = {
    headerGeneratorOptions:{
        browsers: [
            {
                name: 'firefox',
                minVersion: 102,
                maxVersion: 120
            }
        ],
        devices: ['desktop'],
        operatingSystems: ['linux']
    },
    timeout: {
        request: 15000
    },
    retry: {
        limit: 2,
        statusCodes: [409, 429],
        calculateDelay: ({attemptCount, retryOptions, error}) => {
            if(attemptCount > retryOptions.limit) return 0;
            if(error.code == 'ECONNRESET') return 1000; //unknow error ...
            //default
            return 1500; //too many request
        }
    },
    http2: false
}