JuliaWeb / HTTP.jl

HTTP for Julia
https://juliaweb.github.io/HTTP.jl/stable/
Other
626 stars 177 forks source link

Error 400 when executing the second GET request #1094

Open lucasmsoares96 opened 10 months ago

lucasmsoares96 commented 10 months ago

In the code below I convert a curl command to julia then perform a GET request with the information obtained from curl which responds with status 200. However, when executing the second and third requests I always get a 400 error. The equivalent Python code works fine.

Environment:

using HTTP

function curl2jl(string::String)::NamedTuple{(:url, :headers),Tuple{String,Dict{String,String}}}
    tokens::Vector{String} = split(string)
    headers::Dict{String,String} = Dict()
    url::String = ""
    current::String = popfirst!(tokens)
    while (current != "")
        if current == "curl"
            url = popfirst!(tokens) |> x -> chop(x, head=1, tail=1)
        elseif current == "-H"
            header, tokens = getHeader(tokens)
            title = header[1] |> x -> chop(x, head=1, tail=1)
            body = join(header[2:end]) |> chop
            headers[title] = body
        end
        current = length(tokens) > 0 ? popfirst!(tokens) : ""
    end
    return (; url, headers)
end

function getHeader(tokens::Vector{String})::Tuple{Vector{String},Vector{String}}
    header = []
    while true
        token = popfirst!(tokens)
        push!(header, token)
        if token[end] == '\''
            break
        end
    end
    return header, tokens
end

const conf = (readtimeoiut=10, retry=true)

posix1 = """curl 'https://www.linkedin.com/voyager/api/voyagerHiringDashJobHiringSocialHirersCards?decorationId=com.linkedin.voyager.dash.deco.hiring.FullJobHiringSocialHirersCard-8&jobPosting=urn%3Ali%3Afsd_jobPosting%3A3682904701&q=jobPosting&start=0'       --compressed -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0' -H 'Accept: application/vnd.linkedin.normalized+json+2.1' -H 'Accept-Language: pt-BR,pt;q=0.8,en-US;q=0.5,en;q=0.3' -H 'Accept-Encoding: gzip, deflate, br' -H 'x-li-lang: pt_BR' -H 'x-li-track: {"clientVersion":"1.13.1618","mpVersion":"1.13.1618","osName":"web","timezoneOffset":-3,"timezone":"America/Sao_Paulo","deviceFormFactor":"DESKTOP","mpName":"voyager-web","displayDensity":1.25,"displayWidth":1920,"displayHeight":1080}' -H 'x-li-page-instance: urn:li:page:d_flagship3_search_srp_jobs;bANxBiqeTOOzwZMpVmjl5w==' -H 'csrf-token: ajax:3992064359415469819' -H 'x-restli-protocol-version: 2.0.0' -H 'x-li-deco-include-micro-schema: true' -H 'Connection: keep-alive' -H 'Referer: https://www.linkedin.com/jobs/search/?currentJobId=3682904701&distance=25&geoId=106057199&keywords=Software%20Engineer' -H 'Cookie: bcookie="v=2&791c0bbf-d3ae-429e-8321-c11fdeb5ac1a"; bscookie="v=1&2023081710295125baf7da-693a-436f-85d0-b0bc1125b9d8AQFDqtUq47VUEzuCnUo_nFBErUDndaGr"; AMCV_14215E3D5995C57C0A495C55%40AdobeOrg=-637568504%7CMCIDTS%7C19589%7CMCMID%7C03991584888849507642085648156029746815%7CMCAAMLH-1693107905%7C4%7CMCAAMB-1693107905%7C6G1ynYcLPuiQxYZrsz_pkqfLG9yMXBpb2zX5dvJdYQJzPXImdj0y%7CMCOPTOUT-1692510305s%7CNONE%7CvVersion%7C5.1.1%7CMCCIDH%7C-2045535338; li_rm=AQF2Cz6EGubRHAAAAYoDC2n3mimDPrUcVuWvFou-j_zS6BaAXyF0tvY-6Zl2osMtlT_K2_ZSIJsnLK-SAyoDhmRfetTDplTd6wITkEjcIKPWcZ3_dnlVMC7p; aam_uuid=03797710214898168712035234291816533428; _gcl_au=1.1.2049114774.1692268196; timezone=America/Sao_Paulo; li_theme=light; li_theme_set=app; UserMatchHistory=AQKWnC_wZB7svAAAAYoRDBb-t3BOO47m_Lr_hmUmtqGjKViyxpsoh2_DEsFiRmHH-Lz15cFhXW9ko3MetfJ9MO_17cOztP9qirRMC0Ri2bQl2TVN3GN1ybB4Unm0YZXR3MiCubplamNPKhf9vgaZTMMHp39NJtRMJDkqQJU8w3q8vxthEEWx6ZgcHzdGKpniP8P472wnGiVlS1UZTcETmldhpMf4WFn994jbc4lggzzkBOosTLhcIVstAT2Og-yP-aAPtwsIfCFFnPrEsKfy-HWq4Buu5nqCoKNmg8M1Dv3DlSsOwyXLzQV217D5q4sy67CRFFDZoP09fAVn2vJMEmm_fFDqTkY; li_sugr=df2502ab-f2d1-4edf-92a7-fa64e2a4ea4b; _guid=d9b8d502-5a4b-40d9-ae2f-e220295ff1c1; AnalyticsSyncHistory=AQLkN6OJkGkF6wAAAYoDC9Y8DgaqfOaftyLzEsCjStGVn8_b9EhMSj6hcvZfJ4zLLKWk5S4Sz4_Ul_BCfXXolg; lms_ads=AQEPjc4NdhYKxQAAAYoDC9b4cJDBCmWK6ryMuu8Xvqsb7BnB4K5CFi-sADwLGy3WxYAyyA5Tx5WnnGlZVZ83NNxjWYz_Cx-A; lms_analytics=AQEPjc4NdhYKxQAAAYoDC9b4cJDBCmWK6ryMuu8Xvqsb7BnB4K5CFi-sADwLGy3WxYAyyA5Tx5WnnGlZVZ83NNxjWYz_Cx-A; lidc="b=TB26:s=T:r=T:a=T:p=T:g=4588:u=1:x=1:i=1692503121:t=1692556236:v=2:sig=AQH3lUdP3IA2s5uw0mGAOfAPCiVUUs4T"; visit=v=1&M; JSESSIONID="ajax:3992064359415469819"; liap=true; li_at=AQEDAUXub-4DQo5hAAABig_SnyQAAAGKM98jJE0Aiskq4ytlyBs_bdhOmr4jagOjFzuxDSCKzXnbfp6C-2JKS0c9YicjR8a01ZbFFKhkcXqpnIwewY0YgD-zUYuYHthwKb_ddIRQpHUUY2IX_T3Mx_n1; ln_or=eyI0MzQ2MTM3IjoiZCJ9; lang=v=2&lang=pt-br; sdsc=22%3A1%2C1692503120560%7EJAPP%2C0iV48jL2EDPK6tmi8ogNEO%2B9yUTM%3D; AMCVS_14215E3D5995C57C0A495C55%40AdobeOrg=1' -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-origin' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: trailers'""";
url, headers = curl2jl(posix1);

HTTP.get(url, headers; conf...)   # Works fine
HTTP.get(url, headers; conf...)   # 400 Error

posix2 = """curl 'https://www.linkedin.com/voyager/api/voyagerJobsDashJobCards?decorationId=com.linkedin.voyager.dash.deco.jobs.search.JobSearchCardsCollection-172&count=25&q=jobSearch&query=(origin:JOB_SEARCH_PAGE_OTHER_ENTRY,keywords:Software%20Engineer,locationUnion:(geoId:106057199),selectedFilters:(distance:List(25)),spellCorrectionEnabled:true)&start=0' --compressed -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0' -H 'Accept: application/vnd.linkedin.normalized+json+2.1' -H 'Accept-Language: pt-BR,pt;q=0.8,en-US;q=0.5,en;q=0.3' -H 'Accept-Encoding: gzip, deflate, br' -H 'x-li-lang: pt_BR' -H 'x-li-track: {"clientVersion":"1.13.1618","mpVersion":"1.13.1618","osName":"web","timezoneOffset":-3,"timezone":"America/Sao_Paulo","deviceFormFactor":"DESKTOP","mpName":"voyager-web","displayDensity":1.25,"displayWidth":1920,"displayHeight":1080}' -H 'x-li-page-instance: urn:li:page:d_flagship3_search_srp_jobs;bANxBiqeTOOzwZMpVmjl5w==' -H 'csrf-token: ajax:3992064359415469819' -H 'x-restli-protocol-version: 2.0.0' -H 'x-li-pem-metadata: Voyager - Careers=jobs-search-results' -H 'x-li-deco-include-micro-schema: true' -H 'Connection: keep-alive' -H 'Referer: https://www.linkedin.com/jobs/search/?currentJobId=3682904701&distance=25&geoId=106057199&keywords=Software%20Engineer' -H 'Cookie: bcookie="v=2&791c0bbf-d3ae-429e-8321-c11fdeb5ac1a"; bscookie="v=1&2023081710295125baf7da-693a-436f-85d0-b0bc1125b9d8AQFDqtUq47VUEzuCnUo_nFBErUDndaGr"; AMCV_14215E3D5995C57C0A495C55%40AdobeOrg=-637568504%7CMCIDTS%7C19589%7CMCMID%7C03991584888849507642085648156029746815%7CMCAAMLH-1693107905%7C4%7CMCAAMB-1693107905%7C6G1ynYcLPuiQxYZrsz_pkqfLG9yMXBpb2zX5dvJdYQJzPXImdj0y%7CMCOPTOUT-1692510305s%7CNONE%7CvVersion%7C5.1.1%7CMCCIDH%7C-2045535338; li_rm=AQF2Cz6EGubRHAAAAYoDC2n3mimDPrUcVuWvFou-j_zS6BaAXyF0tvY-6Zl2osMtlT_K2_ZSIJsnLK-SAyoDhmRfetTDplTd6wITkEjcIKPWcZ3_dnlVMC7p; aam_uuid=03797710214898168712035234291816533428; _gcl_au=1.1.2049114774.1692268196; timezone=America/Sao_Paulo; li_theme=light; li_theme_set=app; UserMatchHistory=AQKWnC_wZB7svAAAAYoRDBb-t3BOO47m_Lr_hmUmtqGjKViyxpsoh2_DEsFiRmHH-Lz15cFhXW9ko3MetfJ9MO_17cOztP9qirRMC0Ri2bQl2TVN3GN1ybB4Unm0YZXR3MiCubplamNPKhf9vgaZTMMHp39NJtRMJDkqQJU8w3q8vxthEEWx6ZgcHzdGKpniP8P472wnGiVlS1UZTcETmldhpMf4WFn994jbc4lggzzkBOosTLhcIVstAT2Og-yP-aAPtwsIfCFFnPrEsKfy-HWq4Buu5nqCoKNmg8M1Dv3DlSsOwyXLzQV217D5q4sy67CRFFDZoP09fAVn2vJMEmm_fFDqTkY; li_sugr=df2502ab-f2d1-4edf-92a7-fa64e2a4ea4b; _guid=d9b8d502-5a4b-40d9-ae2f-e220295ff1c1; AnalyticsSyncHistory=AQLkN6OJkGkF6wAAAYoDC9Y8DgaqfOaftyLzEsCjStGVn8_b9EhMSj6hcvZfJ4zLLKWk5S4Sz4_Ul_BCfXXolg; lms_ads=AQEPjc4NdhYKxQAAAYoDC9b4cJDBCmWK6ryMuu8Xvqsb7BnB4K5CFi-sADwLGy3WxYAyyA5Tx5WnnGlZVZ83NNxjWYz_Cx-A; lms_analytics=AQEPjc4NdhYKxQAAAYoDC9b4cJDBCmWK6ryMuu8Xvqsb7BnB4K5CFi-sADwLGy3WxYAyyA5Tx5WnnGlZVZ83NNxjWYz_Cx-A; lidc="b=TB26:s=T:r=T:a=T:p=T:g=4588:u=1:x=1:i=1692503106:t=1692556236:v=2:sig=AQENC52yHzOSq9dz812-Nyx3P6O-D_r9"; visit=v=1&M; JSESSIONID="ajax:3992064359415469819"; liap=true; li_at=AQEDAUXub-4DQo5hAAABig_SnyQAAAGKM98jJE0Aiskq4ytlyBs_bdhOmr4jagOjFzuxDSCKzXnbfp6C-2JKS0c9YicjR8a01ZbFFKhkcXqpnIwewY0YgD-zUYuYHthwKb_ddIRQpHUUY2IX_T3Mx_n1; ln_or=eyI0MzQ2MTM3IjoiZCJ9; lang=v=2&lang=pt-br; sdsc=22%3A1%2C1692503114051%7EJAPP%2C0iRIQ9kYP6M2xt6rW88dOrkyXJuA%3D; AMCVS_14215E3D5995C57C0A495C55%40AdobeOrg=1' -H 'Sec-Fetch-Dest: empty' -H 'Sec-Fetch-Mode: cors' -H 'Sec-Fetch-Site: same-origin' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' -H 'TE: trailers'"""
url, headers = curl2jl(posix2)

HTTP.get(url, headers; conf...)   # 400 Error to