search

Danny-Dasilva / CycleTLS

Spoof TLS/JA3 fingerprints in GO and Javascript
GNU General Public License v3.0
873 stars 167 forks source link

address in use #280

Open lhh1771090703 opened 9 months ago

lhh1771090703 commented 9 months ago

Description

When I use it for high concurrency, the address in use situation occurs. How to solve this problem?

Issue Type

Bug, Others

Operating System

Linux

Node Version

Node 14.x

Golang Version

None

Relevant Log Output

Error Processing Request (Please open an issue https://github.com/Danny-Dasilva/CycleTLS/issues/new/choose)-> address already in use
Danny-Dasilva commented 9 months ago

Can you give me a code example so I can reproduce this

lhh1771090703 commented 9 months ago

你能给我一个代码示例，以便我可以重现这个

var searchPrice = function (_child, _ipObj, _cb) { var _jar1 = request.jar(); var _param = {} searchPrice_1(_child, _ipObj, _param, _jar1, _cb); }

const CookieListToArray = function (cookiesList) { let cArray = {} for (let _cookiestr of cookiesList) { let _cookie = _cookiestr.split(";")[0] let _key = _cookie.split("=")[0] let _val = _cookie.split("=")[1] cArray[_key] = _val } return cArray }

var searchPrice_1 = async function (_child, _ipObj, _param, _jar1, _cb) { const cycleTLS = await initCycleTLS(); if (_ipObj) { _param.proxy = http://${_ipObj.ip}:${_ipObj.port};

}
const response = cycleTLS.get('https://www.flyporter.com/en-ca/', {
    ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
    headers: {
        "authority": "www.flyporter.com",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "content-type": "application/json; charset=UTF-8",
        "pragma": "no-cache",
        "sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\"",
        "sec-fetch-mode": "navigate",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
    },
    proxy: _param.proxy,
})
try {
    response.then(out => {
        if (out.status !== 200) {
            logger.error('Failed to get home page!')
            cycleTLS.exit()
            return
        }
        let code = out.body;
        let idx = code.lastIndexOf("bookaflight.CONST.FLIGHT_SUBMIT_VERFICATION_TOKEN =")
        let codeTemp = code.substr(idx, 500);
        let ldx = codeTemp.indexOf('value="') + 7
        let rdx = codeTemp.indexOf(`" />';`) - 115
        let res = codeTemp.substr(ldx, rdx)
        _param.res = res;
        let cookiesList = out.headers['Set-Cookie'];
        let cookiesArray = CookieListToArray(cookiesList)
        _param.cookie = cookiesArray;
        searchPrice_2(_child, _ipObj, _param, _jar1, _cb, cycleTLS)
    })
} catch (err) {
    cycleTLS.exit();
    logger.error(err);
}

} var searchPrice_2 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) { const url = "https://www.flyporter.com/ircan-thence-thate-he-was-yell-A-lights-come-all" if (_ipObj) { _param.proxy = http://${_ipObj.ip}:${_ipObj.port};

}
const response = cycleTLS.get(url, {
    ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
    headers: {
        "authority": "www.flyporter.com",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "content-type": "application/json; charset=UTF-8",
        "pragma": "no-cache",
        "sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\"",
        "sec-fetch-mode": "navigate",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
    },
    cookies: _param.cookie,
    proxy: _param.proxy,
})
try {
    response.then(out => {
        let code = out.body;
        let idx = code.lastIndexOf("=window.btoa");
        let codeTemp = code.substr(idx, 500);
        idx += (codeTemp.indexOf(";") + 1);
        let rdx = codeTemp.indexOf("[")
        let codex = codeTemp.indexOf(";") + 1
        let _name = codeTemp.substr(0, rdx);
        let rname = _name.substr(codex)
        let _injectJs = `window.__data__=${rname};`
        let fjs = code.slice(0, idx);
        let hjs = code.slice(idx);
        _param.alljs = fjs + _injectJs + hjs + "window.__data__;";
        searchPrice_3(_child, _ipObj, _param, _jar1, _cb, cycleTLS);
    })
} catch (err) {
    cycleTLS.exit();
    console.log(err);
}

}

var searchPrice_3 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) { var _url = http://127.0.0.1:8124; var _options = { 'method': 'POST', 'url': _url, 'jar': _jar1, 'headers': { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.9', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', 'Cache-Control': 'max-age=0', 'Connection': 'keep-alive', 'sec-ch-ua': '".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"', 'sec-ch-ua-mobile': '?1', 'sec-ch-ua-platform': '"Android"', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'same-origin', 'Sec-Fetch-User': '?1', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Mobile Safari/537.36', }, 'body': _param.alljs, 'gzip': true, 'timeout': 30000 } request(_options, (error, response, body) => { if (error) { _cb(error, _child); return; }

    if (!error && response.statusCode != 200 && response.statusCode != 302) {
        _cb(new Error(`${_url} - response status [${response.statusCode}]`), _child);
        logger.info(body)
        return;
    }

    try {
        _param._p = JSON.parse(body);
        searchPrice_4(_child, _ipObj, _param, _jar1, _cb, cycleTLS)
    } catch (error) {
        cycleTLS.exit();
        _cb(error, _child);
    }
});

}

var searchPrice_4 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) { let url = 'https://www.flyporter.com/ircan-thence-thate-he-was-yell-A-lights-come-all?d=www.flyporter.com'; let _data = { "solution": { "interrogation": _param._p, "version": "beta" }, "old_token": null, "error": null, "performance": { "interrogation": randInt(200, 500) } } if (_ipObj) { _param.proxy = http://${_ipObj.ip}:${_ipObj.port};

}
let response = cycleTLS.post(url, {
    body: JSON.stringify(_data),
    ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
    disableRedirect: true,
    headers: {
        "authority": "www.flyporter.com",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "content-type": "application/json; charset=UTF-8",
        "pragma": "no-cache",
        "sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\"",
        "sec-fetch-mode": "navigate",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
    },
    cookies: _param.cookie,
    proxy: _param.proxy,
})
response.then(out => {
    let _reese84 = out.body.token;
    _param.reese84 = _reese84;
    searchPrice_5(_child, _ipObj, _param, _jar1, _cb, cycleTLS);
})

}

//302 var searchPrice_5 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) { let _depAirport = _child.depAirport; let _arrAirport = _child.arrAirport; let _fareDate = moment(_child.fareDate).format('YYYY/MM/DD'); let _adultnum = _child.adultNum; let url = 'https://www.flyporter.com/en-ca/flight/select'; let _data = __RequestVerificationToken=${_param.res}&CorporateTravelBookingSelected=False&tabId=&DepartStations=${_depAirport}&ArrivalStations=${_arrAirport}&TripType=OneWay&DepartDate=${_fareDate}&ArrivalDate=&selADT=${_adultnum}&selCHD=0&selINF=0&txtPromoCode=&txtAirMilesNumber=&ClassOfServices=&FareClasses=R&LoyaltyFilter=0 _param.cookie["reese84"] = _param.reese84; if (_ipObj) { _param.proxy = http://${_ipObj.ip}:${_ipObj.port};

}
let response = cycleTLS.post(url, {
    body: _data,
    ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
    disableRedirect: true,
    headers: {
        "authority": "www.flyporter.com",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "pragma": "no-cache",
        "Content-Type": "application/x-www-form-urlencoded",
        "sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\"",
        "sec-fetch-mode": "navigate",
        "Referer": "https://www.flyporter.com/en-ca/",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
    },
    cookies: _param.cookie,
    proxy: _param.proxy,
})
response.then(out => {
    // _param.redirectUrl = 'https://www.flyporter.com' + out.headers.Location;
    let _resCookies = out.headers['Set-Cookie'];
    let _cookieObj = CookieListToArray(_resCookies);
    for (let _key in _cookieObj) {
        _param.cookie[_key] = _cookieObj[_key];
    }
    searchPrice_6(_child, _ipObj, _param, _jar1, _cb, cycleTLS);
})

}

var searchPrice_6 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) { let _depAirport = _child.depAirport; let _arrAirport = _child.arrAirport; let _fareDate = _child.fareDate; let _adultnum = _child.adultNum; let url = 'https://www.flyporter.com/en-ca/flight/GetFlightAvailability'; _param.postData = [ { "DepartureStation": _depAirport, "ArrivalStation": _arrAirport, "BeginDate": _fareDate, "EndDate": _fareDate, "CarrierCode": null, "FlightNumber": null, "FlightType": 5, "PaxCount": _adultnum, "Dow": 1024, "CurrencyCode": "CAD", "DiscountCode": null, "PromotionCode": "", "AvailabilityType": 0, "SourceOrganization": null, "MaximumConnectingFlights": 20, "AvailabilityFilter": 0, "FareClassControl": 1, "InboundOutbound": 2, "TimeWindowsTotalMinutes": 0, "FareTypes": [ "R" ], "ProductClasses": [ "SS", "SI", "SL", "SR", "SV", "US", "BS", "FI", "FL", "FR", "UL", "NV", "SS", "SI", "SL", "SR", "SV", "US" ], "FareClasses": null, "PaxPriceTypes": [ { "PaxDiscountCode": null, "PaxType": "ADT", "Id": 0 } ], "JourneySortKeys": [ 4 ], "IncludeTaxesAndFees": true, "FareRuleFilter": 0, "LoyaltyFilter": 0, "PaxResidentCountry": "CA", "selectedJourneySellKey": null, "selectedFareSellKey": null, "RemoveFareTypesAfterSubmit": null, "NavitaireIncludeTaxesAndFees": true, "SameDaySearch": false, "FromLowFareSelection": false, "DepartureStationName": null, "ArrivalStationName": null, "FlightSearchType": 0, "Id": 0 } ] if (_ipObj) { _param.proxy = http://${_ipObj.ip}:${_ipObj.port};

}

let response = cycleTLS.post(url, {
    body: JSON.stringify(_param.postData),
    ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
    headers: {
        "authority": "www.flyporter.com",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "content-type": "application/json; charset=UTF-8",
        "pragma": "no-cache",
        "sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\"",
        "sec-fetch-mode": "navigate",
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
        "X-Requested-With": "XMLHttpRequest",
    },
    cookies: _param.cookie,
    proxy: _param.proxy,
})
response.then(out => {
    let _resJson = out.body;
    _cb(null, _child, _resJson);
    cycleTLS.exit()
})

}

lhh1771090703 commented 9 months ago

This is just a JS file. When I deploy it on the server for concurrency, this file will be read, but the _child passed in is different so that different tasks can be generated.

Danny-Dasilva commented 9 months ago

@lhh1771090703

This isn't the best example of concurrency since you're making the api calls sequentially, I'm going to assume some missing context here is that you're calling this function multiple times. But if that's not the case feel free to provide a simplified example that better reproduces the issue.

A simple example that reproduces the issue is preferred since I can add it in as a test to the repo.

https://github.com/Danny-Dasilva/CycleTLS#multiple-requests-example-for-typescript-and-javascript

In this particular case I would try moving this outside your function const cycleTLS = await initCycleTLS();

something like

 const cycleTLS = await initCycleTLS();
 var searchPrice = function (_child, _ipObj, _cb) {
// ...remaining code

would be preferred

On every initCycleTLS call we are spawning the golang instance, currently there is some code that checks for an existing instance and tries to connect to it. The address already in use issue means that a CycleTLS instance has already been started and therefore it can't be recreated on that port.

lhh1771090703 commented 9 months ago

Thank you very much for your reply, it seems I now know how to optimize it