search

JuliaServices / CloudStore.jl

A simple, consistent, and performant API for interacting with common cloud storage abstractions
Other
16 stars 8 forks source link

EOFError during put #16

Closed maleadt closed 2 years ago

maleadt commented 2 years ago

I encountered the following error during an upload:

HTTP.Exceptions.RequestError(HTTP.Messages.Request:
"""
POST /reports/2022-09-14T09-54-15-maleadt.tar.zst?uploadId=6VgJU07nLl8m6nce7Ftmbi6IpcdlCezN.xt96iq4NkH5tuJ8kmXgsBOfBDPrFSN_AK70cgTgDdT4K1LkAnF0o6F2pI7sm1KAaHD7rPmoD_VONybKxVzhOSgSRXJtmNmu HTTP/1.1
Host: julialang-dumps.s3.us-east-1.amazonaws.com
Accept: */*
User-Agent: HTTP.jl/1.8.1
Content-Length: 1434
Accept-Encoding: gzip
x-amz-date: 20220914T095447Z
x-amz-security-token: ...
Authorization: ...

<CompleteMultipartUpload><Part><ETag>"271f2398a6c821df2a8ce2a5de79a34b"</ETag><PartNumber>1</PartNumber></Part><Part><ETag>"bd693962a4f6b4c02b91539da524da86"</ETag><PartNumber>2</PartNumber></Part><Part><ETag>"b96730194f1b3db99cd05d2601e07101"</ETag><PartNumber>3</PartNumber></Part><Part><ETag>"54aa7e7466676748400b77fb0cd3bdff"</ETag><PartNumber>4</PartNumber></Part><Part><ETag>"40cfe07d64bc13ab6db253b99e639b68"</ETag><PartNumber>5</PartNumber></Part><Part><ETag>"fa0f19c0fcb3474454eab992e70d864c"</ETag><PartNumber>6</PartNumber></Part><Part><ETag>"8cbd28c36d864fe52308f88f32d3125e"</ETag><PartNumber>7</PartNumber></Part><Part><ETag>"c2359d2050ccfe3c917b949934f08b60"</ETag><PartNumber>8</PartNumber></Part><Part><ETag>"10d36d33560a467a3a54893eed6054e6"</ETag><PartNumber>9</PartNumber></Part><Part><ETag>"6120b0c5ac3fd800ee0b183779710d09"</ETag><PartNumber>10</PartNumber></Part><Part><ETag>"031ae0f50f376c7ced292c72896e0092"</ETag><PartNumber>11</PartNumber></Part><Part><ETag>"11539bc0424b7f
⋮
1434-byte body
""", EOFError())
Stacktrace:
  [1] (::HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, kw::Base.Pairs{Symbol, Any, NTuple{6, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :service, :credentials, :aws), Tuple{Nothing, Bool, Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ HTTP.ConnectionRequest ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/ConnectionRequest.jl:107
  [2] (::HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(req::HTTP.Messages.Request; canonicalize_headers::Bool, kw::Base.Pairs{Symbol, Any, NTuple{6, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :service, :credentials, :aws), Tuple{Nothing, Bool, Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ HTTP.CanonicalizeRequest ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/CanonicalizeRequest.jl:17
  [3] (::Base.var"#88#90"{Base.var"#88#89#91"{ExponentialBackOff, HTTP.RetryRequest.var"#3#6"{Int64, HTTP.Messages.Request, Base.RefValue{Int64}}, HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(args::HTTP.Messages.Request; kwargs::Base.Pairs{Symbol, Any, NTuple{6, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :service, :credentials, :aws), Tuple{Nothing, Bool, Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ Base ./error.jl:296
  [4] (::HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_non_idempotent::Bool, kw::Base.Pairs{Symbol, Any, NTuple{6, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :service, :credentials, :aws), Tuple{Nothing, Bool, Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ HTTP.RetryRequest ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/RetryRequest.jl:60
  [5] (::HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}})(req::HTTP.Messages.Request; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::Base.Pairs{Symbol, Any, NTuple{6, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :service, :credentials, :aws), Tuple{Nothing, Bool, Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ HTTP.CookieRequest ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/CookieRequest.jl:42
  [6] (::HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; detect_content_type::Bool, kw::Base.Pairs{Symbol, Any, NTuple{6, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :service, :credentials, :aws), Tuple{Nothing, Bool, Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ HTTP.ContentTypeDetection ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/ContentTypeRequest.jl:23
  [7] (::HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; basicauth::Bool, kw::Base.Pairs{Symbol, Any, NTuple{6, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :service, :credentials, :aws), Tuple{Nothing, Bool, Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ HTTP.BasicAuthRequest ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/BasicAuthRequest.jl:22
  [8] (::HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; iofunction::Nothing, decompress::Bool, kw::Base.Pairs{Symbol, Any, NTuple{4, Symbol}, NamedTuple{(:verbose, :service, :credentials, :aws), Tuple{Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ HTTP.DefaultHeadersRequest ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/DefaultHeadersRequest.jl:47
  [9] (::HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Nothing, kw::Base.Pairs{Symbol, Any, NTuple{4, Symbol}, NamedTuple{(:verbose, :service, :credentials, :aws), Tuple{Int64, String, CloudBase.AWSCredentials, Bool}}})
    @ HTTP.RedirectRequest ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/RedirectRequest.jl:25
 [10] #1#2
    @ ~/Julia/depot/packages/CloudBase/BqU1k/src/CloudBase.jl:79 [inlined]
 [11] #1#2
    @ ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/DebugRequest.jl:22 [inlined]
 [12] (::HTTP.MessageRequest.var"#1#3"{HTTP.MessageRequest.var"#1#2#4"{HTTP.DebugRequest.var"#1#4"{HTTP.DebugRequest.var"#1#2#5"{CloudBase.AWS.var"#1#3"{CloudBase.AWS.var"#1#2#4"{HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}}}}}}}})(method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::String; response_stream::Nothing, http_version::VersionNumber, kw::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:service, :credentials), Tuple{String, CloudBase.AWSCredentials}}})
    @ HTTP.MessageRequest ~/Julia/depot/packages/HTTP/XJG1J/src/clientlayers/MessageRequest.jl:17
 [13] request(stack::HTTP.MessageRequest.var"#1#3"{HTTP.MessageRequest.var"#1#2#4"{HTTP.DebugRequest.var"#1#4"{HTTP.DebugRequest.var"#1#2#5"{CloudBase.AWS.var"#1#3"{CloudBase.AWS.var"#1#2#4"{HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#4"{HTTP.ConnectionRequest.var"#1#2#5"{HTTP.TimeoutRequest.var"#1#4"{HTTP.TimeoutRequest.var"#1#2#5"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{CloudBase.var"#cloudsign#61"{CloudBase.var"#cloudsign#60#62"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}}}}}}}}, method::String, url::String, h::Vector{Pair{SubString{String}, SubString{String}}}, b::String, q::Dict{String, String}; headers::Vector{Pair{SubString{String}, SubString{String}}}, body::String, query::Dict{String, String}, kw::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:service, :credentials), Tuple{String, CloudBase.AWSCredentials}}})
    @ HTTP ~/Julia/depot/packages/HTTP/XJG1J/src/HTTP.jl:412
 [14] #request#11
    @ ~/Julia/depot/packages/HTTP/XJG1J/src/HTTP.jl:456 [inlined]
 [15] #post#7
    @ ~/Julia/depot/packages/HTTP/XJG1J/src/HTTP.jl:452 [inlined]
 [16] completeMultipartUpload(x::CloudBase.AWS.Bucket, url::String, eTags::Vector{String}, uploadId::String; kw::Base.Pairs{Symbol, CloudBase.AWSCredentials, Tuple{Symbol}, NamedTuple{(:credentials,), Tuple{CloudBase.AWSCredentials}}})
    @ CloudStore.S3 ~/Julia/depot/packages/CloudStore/z4Llt/src/s3.jl:54
 [17] putObjectImpl(x::CloudBase.AWS.Bucket, key::String, in::Base.Process; multipartThreshold::Int64, partSize::Int64, batchSize::Int64, allowMultipart::Bool, compress::Bool, kw::Base.Pairs{Symbol, CloudBase.AWSCredentials, Tuple{Symbol}, NamedTuple{(:credentials,), Tuple{CloudBase.AWSCredentials}}})
    @ CloudStore.API ~/Julia/depot/packages/CloudStore/z4Llt/src/put.jl:95
 [18] #put#10
    @ ~/Julia/depot/packages/CloudStore/z4Llt/src/s3.jl:37 [inlined]
 [19] put(url::String, args::Base.Process; region::String, nowarn::Bool, kw::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:credentials, :multipartThreshold), Tuple{CloudBase.AWSCredentials, Int64}}})
    @ CloudStore.S3 ~/Julia/depot/packages/CloudStore/z4Llt/src/s3.jl:70

This is with reading from a Process (where the bytesavailable might have been an underestimation), while forcing a multipart upload, so it's possible something went wrong there. I'm also seeing this when just uploading a file.

maleadt commented 2 years ago

I keep on seeing this every so often when uploading a large rr trace. @quinnj any thoughts?

quinnj commented 2 years ago

Thanks for the report @maleadt; do you happen to have a more full backtrace? Usually there's a nested exception here that will provide a little more detail on the EOFError

maleadt commented 2 years ago

I don't think there was. I can't reproduce the issue though, but I'll keep an eye when it happens again.

quinnj commented 2 years ago

Sounds good. I've been doing increasingly heavy load testing w/ the package and haven't seen any errors like this yet, but I'm also running with the new OpenSSL.jl for tls instead of MbedTLS.jl which may account for weird differences like this. Please comment again if you see it happen.