Open lhh1771090703 opened 9 months ago
Can you give me a code example so I can reproduce this
你能给我一个代码示例,以便我可以重现这个
var searchPrice = function (_child, _ipObj, _cb) { var _jar1 = request.jar(); var _param = {} searchPrice_1(_child, _ipObj, _param, _jar1, _cb); }
const CookieListToArray = function (cookiesList) { let cArray = {} for (let _cookiestr of cookiesList) { let _cookie = _cookiestr.split(";")[0] let _key = _cookie.split("=")[0] let _val = _cookie.split("=")[1] cArray[_key] = _val } return cArray }
var searchPrice_1 = async function (_child, _ipObj, _param, _jar1, _cb) {
const cycleTLS = await initCycleTLS();
if (_ipObj) {
_param.proxy = http://${_ipObj.ip}:${_ipObj.port}
;
}
const response = cycleTLS.get('https://www.flyporter.com/en-ca/', {
ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
headers: {
"authority": "www.flyporter.com",
"accept-language": "zh-CN,zh;q=0.9",
"cache-control": "no-cache",
"content-type": "application/json; charset=UTF-8",
"pragma": "no-cache",
"sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"Windows\"",
"sec-fetch-mode": "navigate",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
},
proxy: _param.proxy,
})
try {
response.then(out => {
if (out.status !== 200) {
logger.error('Failed to get home page!')
cycleTLS.exit()
return
}
let code = out.body;
let idx = code.lastIndexOf("bookaflight.CONST.FLIGHT_SUBMIT_VERFICATION_TOKEN =")
let codeTemp = code.substr(idx, 500);
let ldx = codeTemp.indexOf('value="') + 7
let rdx = codeTemp.indexOf(`" />';`) - 115
let res = codeTemp.substr(ldx, rdx)
_param.res = res;
let cookiesList = out.headers['Set-Cookie'];
let cookiesArray = CookieListToArray(cookiesList)
_param.cookie = cookiesArray;
searchPrice_2(_child, _ipObj, _param, _jar1, _cb, cycleTLS)
})
} catch (err) {
cycleTLS.exit();
logger.error(err);
}
}
var searchPrice_2 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) {
const url = "https://www.flyporter.com/ircan-thence-thate-he-was-yell-A-lights-come-all"
if (_ipObj) {
_param.proxy = http://${_ipObj.ip}:${_ipObj.port}
;
}
const response = cycleTLS.get(url, {
ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
headers: {
"authority": "www.flyporter.com",
"accept-language": "zh-CN,zh;q=0.9",
"cache-control": "no-cache",
"content-type": "application/json; charset=UTF-8",
"pragma": "no-cache",
"sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"Windows\"",
"sec-fetch-mode": "navigate",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
},
cookies: _param.cookie,
proxy: _param.proxy,
})
try {
response.then(out => {
let code = out.body;
let idx = code.lastIndexOf("=window.btoa");
let codeTemp = code.substr(idx, 500);
idx += (codeTemp.indexOf(";") + 1);
let rdx = codeTemp.indexOf("[")
let codex = codeTemp.indexOf(";") + 1
let _name = codeTemp.substr(0, rdx);
let rname = _name.substr(codex)
let _injectJs = `window.__data__=${rname};`
let fjs = code.slice(0, idx);
let hjs = code.slice(idx);
_param.alljs = fjs + _injectJs + hjs + "window.__data__;";
searchPrice_3(_child, _ipObj, _param, _jar1, _cb, cycleTLS);
})
} catch (err) {
cycleTLS.exit();
console.log(err);
}
}
var searchPrice_3 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) {
var _url = http://127.0.0.1:8124
;
var _options = {
'method': 'POST',
'url': _url,
'jar': _jar1,
'headers':
{
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'sec-ch-ua': '".Not/A)Brand";v="99", "Google Chrome";v="103", "Chromium";v="103"',
'sec-ch-ua-mobile': '?1',
'sec-ch-ua-platform': '"Android"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Mobile Safari/537.36',
},
'body': _param.alljs,
'gzip': true,
'timeout': 30000
}
request(_options, (error, response, body) => {
if (error) {
_cb(error, _child);
return;
}
if (!error && response.statusCode != 200 && response.statusCode != 302) {
_cb(new Error(`${_url} - response status [${response.statusCode}]`), _child);
logger.info(body)
return;
}
try {
_param._p = JSON.parse(body);
searchPrice_4(_child, _ipObj, _param, _jar1, _cb, cycleTLS)
} catch (error) {
cycleTLS.exit();
_cb(error, _child);
}
});
}
var searchPrice_4 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) {
let url = 'https://www.flyporter.com/ircan-thence-thate-he-was-yell-A-lights-come-all?d=www.flyporter.com';
let _data = {
"solution": {
"interrogation": _param._p,
"version": "beta"
},
"old_token": null,
"error": null,
"performance": {
"interrogation": randInt(200, 500)
}
}
if (_ipObj) {
_param.proxy = http://${_ipObj.ip}:${_ipObj.port}
;
}
let response = cycleTLS.post(url, {
body: JSON.stringify(_data),
ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
disableRedirect: true,
headers: {
"authority": "www.flyporter.com",
"accept-language": "zh-CN,zh;q=0.9",
"cache-control": "no-cache",
"content-type": "application/json; charset=UTF-8",
"pragma": "no-cache",
"sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"Windows\"",
"sec-fetch-mode": "navigate",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
},
cookies: _param.cookie,
proxy: _param.proxy,
})
response.then(out => {
let _reese84 = out.body.token;
_param.reese84 = _reese84;
searchPrice_5(_child, _ipObj, _param, _jar1, _cb, cycleTLS);
})
}
//302
var searchPrice_5 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) {
let _depAirport = _child.depAirport;
let _arrAirport = _child.arrAirport;
let _fareDate = moment(_child.fareDate).format('YYYY/MM/DD');
let _adultnum = _child.adultNum;
let url = 'https://www.flyporter.com/en-ca/flight/select';
let _data = __RequestVerificationToken=${_param.res}&CorporateTravelBookingSelected=False&tabId=&DepartStations=${_depAirport}&ArrivalStations=${_arrAirport}&TripType=OneWay&DepartDate=${_fareDate}&ArrivalDate=&selADT=${_adultnum}&selCHD=0&selINF=0&txtPromoCode=&txtAirMilesNumber=&ClassOfServices=&FareClasses=R&LoyaltyFilter=0
_param.cookie["reese84"] = _param.reese84;
if (_ipObj) {
_param.proxy = http://${_ipObj.ip}:${_ipObj.port}
;
}
let response = cycleTLS.post(url, {
body: _data,
ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
disableRedirect: true,
headers: {
"authority": "www.flyporter.com",
"accept-language": "zh-CN,zh;q=0.9",
"cache-control": "no-cache",
"pragma": "no-cache",
"Content-Type": "application/x-www-form-urlencoded",
"sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"Windows\"",
"sec-fetch-mode": "navigate",
"Referer": "https://www.flyporter.com/en-ca/",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
},
cookies: _param.cookie,
proxy: _param.proxy,
})
response.then(out => {
// _param.redirectUrl = 'https://www.flyporter.com' + out.headers.Location;
let _resCookies = out.headers['Set-Cookie'];
let _cookieObj = CookieListToArray(_resCookies);
for (let _key in _cookieObj) {
_param.cookie[_key] = _cookieObj[_key];
}
searchPrice_6(_child, _ipObj, _param, _jar1, _cb, cycleTLS);
})
}
var searchPrice_6 = async function (_child, _ipObj, _param, _jar1, _cb, cycleTLS) {
let _depAirport = _child.depAirport;
let _arrAirport = _child.arrAirport;
let _fareDate = _child.fareDate;
let _adultnum = _child.adultNum;
let url = 'https://www.flyporter.com/en-ca/flight/GetFlightAvailability';
_param.postData = [
{
"DepartureStation": _depAirport,
"ArrivalStation": _arrAirport,
"BeginDate": _fareDate,
"EndDate": _fareDate,
"CarrierCode": null,
"FlightNumber": null,
"FlightType": 5,
"PaxCount": _adultnum,
"Dow": 1024,
"CurrencyCode": "CAD",
"DiscountCode": null,
"PromotionCode": "",
"AvailabilityType": 0,
"SourceOrganization": null,
"MaximumConnectingFlights": 20,
"AvailabilityFilter": 0,
"FareClassControl": 1,
"InboundOutbound": 2,
"TimeWindowsTotalMinutes": 0,
"FareTypes": [
"R"
],
"ProductClasses": [
"SS",
"SI",
"SL",
"SR",
"SV",
"US",
"BS",
"FI",
"FL",
"FR",
"UL",
"NV",
"SS",
"SI",
"SL",
"SR",
"SV",
"US"
],
"FareClasses": null,
"PaxPriceTypes": [
{
"PaxDiscountCode": null,
"PaxType": "ADT",
"Id": 0
}
],
"JourneySortKeys": [
4
],
"IncludeTaxesAndFees": true,
"FareRuleFilter": 0,
"LoyaltyFilter": 0,
"PaxResidentCountry": "CA",
"selectedJourneySellKey": null,
"selectedFareSellKey": null,
"RemoveFareTypesAfterSubmit": null,
"NavitaireIncludeTaxesAndFees": true,
"SameDaySearch": false,
"FromLowFareSelection": false,
"DepartureStationName": null,
"ArrivalStationName": null,
"FlightSearchType": 0,
"Id": 0
}
]
if (_ipObj) {
_param.proxy = http://${_ipObj.ip}:${_ipObj.port}
;
}
let response = cycleTLS.post(url, {
body: JSON.stringify(_param.postData),
ja3: '771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0',
headers: {
"authority": "www.flyporter.com",
"accept-language": "zh-CN,zh;q=0.9",
"cache-control": "no-cache",
"content-type": "application/json; charset=UTF-8",
"pragma": "no-cache",
"sec-ch-ua": "\"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"",
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": "\"Windows\"",
"sec-fetch-mode": "navigate",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
"X-Requested-With": "XMLHttpRequest",
},
cookies: _param.cookie,
proxy: _param.proxy,
})
response.then(out => {
let _resJson = out.body;
_cb(null, _child, _resJson);
cycleTLS.exit()
})
}
This is just a JS file. When I deploy it on the server for concurrency, this file will be read, but the _child passed in is different so that different tasks can be generated.
@lhh1771090703
This isn't the best example of concurrency since you're making the api calls sequentially, I'm going to assume some missing context here is that you're calling this function multiple times. But if that's not the case feel free to provide a simplified example that better reproduces the issue.
A simple example that reproduces the issue is preferred since I can add it in as a test to the repo.
https://github.com/Danny-Dasilva/CycleTLS#multiple-requests-example-for-typescript-and-javascript
In this particular case I would try moving this outside your function const cycleTLS = await initCycleTLS();
something like
const cycleTLS = await initCycleTLS();
var searchPrice = function (_child, _ipObj, _cb) {
// ...remaining code
would be preferred
On every initCycleTLS
call we are spawning the golang instance, currently there is some code that checks for an existing instance and tries to connect to it. The address already in use issue means that a CycleTLS instance has already been started and therefore it can't be recreated on that port.
Thank you very much for your reply, it seems I now know how to optimize it
Description
When I use it for high concurrency, the address in use situation occurs. How to solve this problem?
Issue Type
Bug, Others
Operating System
Linux
Node Version
Node 14.x
Golang Version
None
Relevant Log Output