search

JuliaWeb / HTTP.jl

HTTP for Julia
https://juliaweb.github.io/HTTP.jl/stable/
Other
626 stars 177 forks source link

Crash due to Content-Length InexactError on large request #1130

Closed Octogonapus closed 7 months ago

Octogonapus commented 7 months ago

We ran into a request error because of a large (2.28 GB) request body:

│    HTTP.RequestError:
│    HTTP.Request:
│    HTTP.Messages.Request:
│    """
│    POST /xxx HTTP/1.1
│    Authorization: Bearer xxx
│    Host: xxx
│    Accept: */*
│    User-Agent: HTTP.jl/1.9.4
│    Content-Length: 2281871749
│    Content-Type: multipart/form-data; boundary=fb808120ca037c02966686b858c2d6ee
│    Accept-Encoding: gzip
│    
│    [Message Body was streamed]"""Underlying error:
│    InexactError: trunc(Int32, 2281871749)
│    Stacktrace:
│      [1] (::HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, socket_type_tls::Type, readtimeout::Int64, connect_timeout::Int64, logerrors::Bool, logtag::Nothing, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :require_ssl_verification), Tuple{Nothing, Nothing, Int64, Bool}}})
│        @ HTTP.ConnectionRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/ConnectionRequest.jl:143
│      [2] (::Base.var"#90#92"{Base.var"#90#91#93"{ExponentialBackOff, HTTP.RetryRequest.var"#2#5"{Int64, typeof(HTTP.RetryRequest.FALSE), HTTP.Messages.Request, Base.RefValue{Int64}}, HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(args::HTTP.Messages.Request; kwargs::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :require_ssl_verification), Tuple{Nothing, Nothing, Int64, Bool}}})
│        @ Base ./error.jl:296
│      [3] (::HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_delays::ExponentialBackOff, retry_check::Function, retry_non_idempotent::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :require_ssl_verification), Tuple{Nothing, Nothing, Int64, Bool}}})
│        @ HTTP.RetryRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/RetryRequest.jl:75
│      [4] manageretries
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/RetryRequest.jl:30 [inlined]
│      [5] (::HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(req::HTTP.Messages.Request; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{4, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :require_ssl_verification), Tuple{Nothing, Nothing, Int64, Bool}}})
│        @ HTTP.CookieRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/CookieRequest.jl:42
│      [6] managecookies
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/CookieRequest.jl:19 [inlined]
│      [7] (::HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(req::HTTP.Messages.Request; iofunction::Nothing, decompress::Nothing, basicauth::Bool, detect_content_type::Bool, canonicalize_headers::Bool, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :require_ssl_verification), Tuple{Int64, Bool}}})
│        @ HTTP.HeadersRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/HeadersRequest.jl:71
│      [8] defaultheaders
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/HeadersRequest.jl:14 [inlined]
│      [9] (::HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Nothing, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:verbose, :require_ssl_verification), Tuple{Int64, Bool}}})
│        @ HTTP.RedirectRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/RedirectRequest.jl:25
│     [10] redirects
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/RedirectRequest.jl:14 [inlined]
│     [11] (::HTTP.MessageRequest.var"#makerequest#3"{HTTP.MessageRequest.var"#makerequest#1#4"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}})(method::String, url::URIs.URI, headers::Dict{String, String}, body::HTTP.Forms.Form; copyheaders::Bool, response_stream::Nothing, http_version::HTTP.Strings.HTTPVersion, verbose::Int64, kw::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:require_ssl_verification,), Tuple{Bool}}})
│        @ HTTP.MessageRequest /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/MessageRequest.jl:35
│     [12] makerequest
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/clientlayers/MessageRequest.jl:24 [inlined]
│     [13] request(stack::HTTP.MessageRequest.var"#makerequest#3"{HTTP.MessageRequest.var"#makerequest#1#4"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}, method::String, url::String, h::Dict{String, String}, b::HTTP.Forms.Form, q::Nothing; headers::Dict{String, String}, body::HTTP.Forms.Form, query::Nothing, kw::Base.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:require_ssl_verification,), Tuple{Bool}}})
│        @ HTTP /home/runner/.julia/packages/HTTP/SN7VW/src/HTTP.jl:457
│     [14] #request#20
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/HTTP.jl:315 [inlined]
│     [15] request
│        @ /home/runner/.julia/packages/HTTP/SN7VW/src/HTTP.jl:313 [inlined]

Note that the Content-Length 2281871749 is larger than typemax(Int32).

fredrikekre commented 7 months ago

I can not reproduce this. I tried for example

server = HTTP.listen!() do http
    HTTP.setstatus(http, 200)
    bytes_total = Int(typemax(Int32) + 1)
    HTTP.setheader(http, "Content-Length" => string(bytes_total))
    HTTP.startwrite(http)
    io = open("/dev/random", "r")
    bytes_written = 0
    while bytes_written < bytes_total
        toread = min(bytes_total - bytes_written, 1024^2)
        chunk_size = write(http, read(io, toread))
        bytes_written += chunk_size
    end
end

HTTP.get("http://localhost:8081", response_stream = devnull)

Unfortunately HTTP.jl swallows the original stacktrace from the error e here https://github.com/JuliaWeb/HTTP.jl/blob/57d57a513fd9caf689303f12b01e93f6859a2ec5/src/clientlayers/ConnectionRequest.jl#L121: and we only see the trace for https://github.com/JuliaWeb/HTTP.jl/blob/57d57a513fd9caf689303f12b01e93f6859a2ec5/src/clientlayers/ConnectionRequest.jl#L143

If you can still reproduce, perhaps try commenting out the catch block: https://github.com/JuliaWeb/HTTP.jl/blob/57d57a513fd9caf689303f12b01e93f6859a2ec5/src/clientlayers/ConnectionRequest.jl#L121-L144 and see what the original error is

Octogonapus commented 7 months ago

huh I can't reproduce this either. not sure what is going on