search

JuliaServices / CloudStore.jl

A simple, consistent, and performant API for interacting with common cloud storage abstractions
Other
16 stars 8 forks source link

Can't Authenticate with IAM Role in ECS environment #56

Open mrufsvold opened 3 months ago

mrufsvold commented 3 months ago

I have a script that worked using AWS.jl and works locally with CloudStore.jl, but I cannot run in Fargate due to an auth issue. Calling CloudStore.AWS.AWSCredentials() yields a Request is missing Authentication Token error message from AWS.

Full logs:


[ Info: loading   config property `default_region` from   `Figgy.KeyMap{Figgy.EnvironmentVariables}`
[ Info: loading config property `region`   from `Figgy.KeyMap{Figgy.EnvironmentVariables}`
[ Info: loading config property   `container_credentials_relative_uri` from   `Figgy.KeyMap{Figgy.EnvironmentVariables}`
[ Info: loading config property   `aws_secret_access_key` from `CloudBase.ECSCredentialsSource`
[ Info: loading config property   `aws_access_key_id` from `CloudBase.ECSCredentialsSource`
[ Info: loading config property   `expiration` from `CloudBase.ECSCredentialsSource`
[ Info: loading config property   `role_arn` from `CloudBase.ECSCredentialsSource`
[ Info: loading config property   `aws_session_token` from `CloudBase.ECSCredentialsSource`
ERROR: LoadError:   HTTP.Exceptions.StatusError(403, "POST", "/",   HTTP.Messages.Response:
"""
HTTP/1.1 403 Forbidden
x-amzn-RequestId:   3b41cd5c-8c20-48e8-9092-3dcfed024966
Content-Type: text/xml
Content-Length: 297
Date: Mon, 05 Aug 2024 20:03:08 GMT
 
<ErrorResponse   xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<Error>
<Type>Sender</Type>
<Code>MissingAuthenticationToken</Code>
<Message>Request is missing Authentication Token</Message>
</Error>
<RequestId>3b41cd5c-8c20-48e8-9092-3dcfed024966</RequestId>
</ErrorResponse>
""")
Stacktrace:
[1]   (::HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(req::HTTP.Messages.Request;   proxy::Nothing, socket_type::Type, socket_type_tls::Type, readtimeout::Int64,   connect_timeout::Int64, logerrors::Bool, logtag::Nothing,   kw::@Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool})
@ HTTP.ConnectionRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/ConnectionRequest.jl:141
[2]   (::Base.var"#96#98"{Base.var"#96#97#99"{ExponentialBackOff,   HTTP.RetryRequest.var"#2#5"{Int64, typeof(HTTP.RetryRequest.FALSE),   HTTP.Messages.Request, Base.RefValue{Int64}}, HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(args::HTTP.Messages.Request;   kwargs::@Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool, readtimeout::Int64})
@ Base ./error.jl:296
[3]   (::HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(req::HTTP.Messages.Request;   retry::Bool, retries::Int64, retry_delays::ExponentialBackOff,   retry_check::Function, retry_non_idempotent::Bool,   kw::@Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool, readtimeout::Int64})
@ HTTP.RetryRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/RetryRequest.jl:75
[4] manageretries
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/RetryRequest.jl:30   [inlined]
[5]   (::HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(req::HTTP.Messages.Request;   cookies::Bool, cookiejar::HTTP.Cookies.CookieJar,   kw::@Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool, readtimeout::Int64})
@ HTTP.CookieRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/CookieRequest.jl:42
[6] managecookies
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/CookieRequest.jl:19   [inlined]
[7]   (::HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}})(req::HTTP.Messages.Request;   iofunction::Nothing, decompress::Nothing, basicauth::Bool,   detect_content_type::Bool, canonicalize_headers::Bool,   kw::@Kwargs{verbose::Int64, credentials::Nothing, aws::Bool, readtimeout::Int64})
@ HTTP.HeadersRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/HeadersRequest.jl:71
[8] defaultheaders
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/HeadersRequest.jl:14   [inlined]
[9]   (::HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request;   redirect::Bool, redirect_limit::Int64, redirect_method::Nothing,   forwardheaders::Bool, response_stream::Nothing, kw::@Kwargs{verbose::Int64,   credentials::Nothing, aws::Bool, readtimeout::Int64})
@ HTTP.RedirectRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/RedirectRequest.jl:25
[10] redirects
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/RedirectRequest.jl:14   [inlined]
[11]   (::CloudBase.var"#cloudmetrics#61"{CloudBase.var"#cloudmetrics#60#62"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request;   logexceptionalduration::Int64, kw::@Kwargs{verbose::Int64,   response_stream::Nothing, credentials::Nothing, aws::Bool,   readtimeout::Int64})
@ CloudBase   /usr/local/.julia/packages/CloudBase/ZAkU5/src/CloudBase.jl:67
[12] cloudmetrics
@ /usr/local/.julia/packages/CloudBase/ZAkU5/src/CloudBase.jl:60   [inlined]
[13] #1
@ /usr/local/.julia/packages/CloudBase/ZAkU5/src/CloudBase.jl:122   [inlined]
[14]   (::HTTP.MessageRequest.var"#makerequest#3"{HTTP.MessageRequest.var"#makerequest#1#4"{CloudBase.AWS.var"#1#3"{CloudBase.AWS.var"#1#2#4"{CloudBase.var"#cloudmetrics#61"{CloudBase.var"#cloudmetrics#60#62"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}})(method::String,   url::URIs.URI, headers::Vector{Any}, body::Dict{String, String};   copyheaders::Bool, response_stream::Nothing,   http_version::HTTP.Strings.HTTPVersion, verbose::Int64,   kw::@Kwargs{credentials::Nothing})
@ HTTP.MessageRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/MessageRequest.jl:35
[15] makerequest
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/MessageRequest.jl:24   [inlined]
[16]   request(stack::HTTP.MessageRequest.var"#makerequest#3"{HTTP.MessageRequest.var"#makerequest#1#4"{CloudBase.AWS.var"#1#3"{CloudBase.AWS.var"#1#2#4"{CloudBase.var"#cloudmetrics#61"{CloudBase.var"#cloudmetrics#60#62"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}},   method::String, url::String, h::Vector{Any}, b::Dict{String, String},   q::Nothing; headers::Vector{Any}, body::Dict{String, String}, query::Nothing,   kw::@Kwargs{credentials::Nothing})
@ HTTP /usr/local/.julia/packages/HTTP/LsfSO/src/HTTP.jl:457
[17] post(::String, ::Vararg{Any};   kw::@Kwargs{credentials::Nothing})
@ CloudBase.AWS   /usr/local/.julia/packages/CloudBase/ZAkU5/src/CloudBase.jl:124
[18] loadRoleArn(roleArn::String,   credFile::String, configFile::String)
@ CloudBase /usr/local/.julia/packages/CloudBase/ZAkU5/src/aws.jl:233
[19] awsLoadConfig!(profile::String,   expireThreshold::Dates.Minute)
@ CloudBase /usr/local/.julia/packages/CloudBase/ZAkU5/src/aws.jl:84
[20] #AWSCredentials#2
@ /usr/local/.julia/packages/CloudBase/ZAkU5/src/aws.jl:49 [inlined]
[21] AWSCredentials (repeats 2 times)
@ /usr/local/.julia/packages/CloudBase/ZAkU5/src/aws.jl:48 [inlined]
[22]   consolidate_partitions(queue_url::String, dead_letter_queue_url::String;   bin_column::String, column_dtype::Type, batch_size::Int64,   scan_precision::Int64, row_group_chunk_size::Int64, chunks_per_file::Int64,   big_file_cutoff::Int64, skip_clustering::Bool, max_batch_length::Int64)
@ Main.TableConsolidator /usr/local/src/TableConsolidator.jl:136
[23] top-level scope
@ /usr/local/consolidator_script.jl:12
in expression starting at   /usr/local/consolidator_script.jl:12
caused by:   HTTP.Exceptions.StatusError(403, "POST", "/",   HTTP.Messages.Response:
"""
HTTP/1.1 403 Forbidden
x-amzn-RequestId:   3b41cd5c-8c20-48e8-9092-3dcfed024966
Content-Type: text/xml
Content-Length: 297
Date: Mon, 05 Aug 2024 20:03:08 GMT
 
<ErrorResponse   xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
<Error>
<Type>Sender</Type>
<Code>MissingAuthenticationToken</Code>
<Message>Request is missing Authentication Token</Message>
</Error>
<RequestId>3b41cd5c-8c20-48e8-9092-3dcfed024966</RequestId>
</ErrorResponse>
""")
Stacktrace:
[1]   (::HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}})(stream::HTTP.Streams.Stream{HTTP.Messages.Response,   HTTP.Connections.Connection{OpenSSL.SSLStream}}; status_exception::Bool,   timedout::ConcurrentUtilities.TimedOut{HTTP.Messages.Response},   logerrors::Bool, logtag::Nothing, kw::@Kwargs{iofunction::Nothing,   decompress::Nothing, verbose::Int64, credentials::Nothing, aws::Bool})
@ HTTP.ExceptionRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/ExceptionRequest.jl:19
[2] exceptions
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/ExceptionRequest.jl:13   [inlined]
[3] #2
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/TimeoutRequest.jl:22   [inlined]
[4] macro expansion
@   /usr/local/.julia/packages/ConcurrentUtilities/QOkoO/src/try_with_timeout.jl:82   [inlined]
[5]   (::ConcurrentUtilities.var"#2#4"{HTTP.Messages.Response, Timer,   ConcurrentUtilities.TimedOut{HTTP.Messages.Response},   Channel{HTTP.Messages.Response},   HTTP.TimeoutRequest.var"#2#5"{Bool, Nothing,   @Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool}, HTTP.Streams.Stream{HTTP.Messages.Response,   HTTP.Connections.Connection{OpenSSL.SSLStream}},   HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}})()
@ ConcurrentUtilities   /usr/local/.julia/packages/ConcurrentUtilities/QOkoO/src/ConcurrentUtilities.jl:9
Stacktrace:
[1] try_yieldto(undo::typeof(Base.ensure_rescheduled))
@ Base ./task.jl:931
[2] wait()
@ Base ./task.jl:995
[3] wait(c::Base.GenericCondition{ReentrantLock}; first::Bool)
@ Base ./condition.jl:130
[4] wait
@ ./condition.jl:125 [inlined]
[5] take_unbuffered(c::Channel{HTTP.Messages.Response})
@ Base ./channels.jl:494
[6] take!
@ ./channels.jl:471 [inlined]
[7] try_with_timeout(f::Function, timeout::Int64,   ::Type{HTTP.Messages.Response})
@ ConcurrentUtilities   /usr/local/.julia/packages/ConcurrentUtilities/QOkoO/src/try_with_timeout.jl:89
[8]   (::HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}})(stream::HTTP.Streams.Stream{HTTP.Messages.Response,   HTTP.Connections.Connection{OpenSSL.SSLStream}}; readtimeout::Int64,   logerrors::Bool, logtag::Nothing, kw::@Kwargs{iofunction::Nothing,   decompress::Nothing, verbose::Int64, credentials::Nothing, aws::Bool})
@ HTTP.TimeoutRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/TimeoutRequest.jl:21
[9]   (::HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(req::HTTP.Messages.Request;   proxy::Nothing, socket_type::Type, socket_type_tls::Type, readtimeout::Int64,   connect_timeout::Int64, logerrors::Bool, logtag::Nothing,   kw::@Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool})
@ HTTP.ConnectionRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/ConnectionRequest.jl:119
[10]   (::Base.var"#96#98"{Base.var"#96#97#99"{ExponentialBackOff,   HTTP.RetryRequest.var"#2#5"{Int64, typeof(HTTP.RetryRequest.FALSE),   HTTP.Messages.Request, Base.RefValue{Int64}}, HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(args::HTTP.Messages.Request;   kwargs::@Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool, readtimeout::Int64})
@ Base ./error.jl:296
[11]   (::HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(req::HTTP.Messages.Request;   retry::Bool, retries::Int64, retry_delays::ExponentialBackOff,   retry_check::Function, retry_non_idempotent::Bool,   kw::@Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool, readtimeout::Int64})
@ HTTP.RetryRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/RetryRequest.jl:75
[12] manageretries
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/RetryRequest.jl:30   [inlined]
[13]   (::HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(req::HTTP.Messages.Request;   cookies::Bool, cookiejar::HTTP.Cookies.CookieJar,   kw::@Kwargs{iofunction::Nothing, decompress::Nothing, verbose::Int64,   credentials::Nothing, aws::Bool, readtimeout::Int64})
@ HTTP.CookieRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/CookieRequest.jl:42
[14] managecookies
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/CookieRequest.jl:19   [inlined]
[15]   (::HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}})(req::HTTP.Messages.Request;   iofunction::Nothing, decompress::Nothing, basicauth::Bool,   detect_content_type::Bool, canonicalize_headers::Bool,   kw::@Kwargs{verbose::Int64, credentials::Nothing, aws::Bool, readtimeout::Int64})
@ HTTP.HeadersRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/HeadersRequest.jl:71
[16] defaultheaders
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/HeadersRequest.jl:14   [inlined]
[17]   (::HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request;   redirect::Bool, redirect_limit::Int64, redirect_method::Nothing,   forwardheaders::Bool, response_stream::Nothing, kw::@Kwargs{verbose::Int64,   credentials::Nothing, aws::Bool, readtimeout::Int64})
@ HTTP.RedirectRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/RedirectRequest.jl:25
[18] redirects
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/RedirectRequest.jl:14   [inlined]
[19]   (::CloudBase.var"#cloudmetrics#61"{CloudBase.var"#cloudmetrics#60#62"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request;   logexceptionalduration::Int64, kw::@Kwargs{verbose::Int64,   response_stream::Nothing, credentials::Nothing, aws::Bool,   readtimeout::Int64})
@ CloudBase   /usr/local/.julia/packages/CloudBase/ZAkU5/src/CloudBase.jl:67
[20] cloudmetrics
@ /usr/local/.julia/packages/CloudBase/ZAkU5/src/CloudBase.jl:60   [inlined]
[21] #1
@ /usr/local/.julia/packages/CloudBase/ZAkU5/src/CloudBase.jl:122   [inlined]
[22]   (::HTTP.MessageRequest.var"#makerequest#3"{HTTP.MessageRequest.var"#makerequest#1#4"{CloudBase.AWS.var"#1#3"{CloudBase.AWS.var"#1#2#4"{CloudBase.var"#cloudmetrics#61"{CloudBase.var"#cloudmetrics#60#62"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}})(method::String,   url::URIs.URI, headers::Vector{Any}, body::Dict{String, String};   copyheaders::Bool, response_stream::Nothing,   http_version::HTTP.Strings.HTTPVersion, verbose::Int64,   kw::@Kwargs{credentials::Nothing})
@ HTTP.MessageRequest   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/MessageRequest.jl:35
[23] makerequest
@   /usr/local/.julia/packages/HTTP/LsfSO/src/clientlayers/MessageRequest.jl:24   [inlined]
[24]   request(stack::HTTP.MessageRequest.var"#makerequest#3"{HTTP.MessageRequest.var"#makerequest#1#4"{CloudBase.AWS.var"#1#3"{CloudBase.AWS.var"#1#2#4"{CloudBase.var"#cloudmetrics#61"{CloudBase.var"#cloudmetrics#60#62"{HTTP.RedirectRequest.var"#redirects#3"{HTTP.RedirectRequest.var"#redirects#1#4"{HTTP.HeadersRequest.var"#defaultheaders#2"{HTTP.HeadersRequest.var"#defaultheaders#1#3"{HTTP.CookieRequest.var"#managecookies#4"{HTTP.CookieRequest.var"#managecookies#1#5"{HTTP.RetryRequest.var"#manageretries#3"{HTTP.RetryRequest.var"#manageretries#1#4"{HTTP.ConnectionRequest.var"#connections#4"{HTTP.ConnectionRequest.var"#connections#1#5"{HTTP.TimeoutRequest.var"#timeouts#3"{HTTP.TimeoutRequest.var"#timeouts#1#4"{HTTP.ExceptionRequest.var"#exceptions#2"{HTTP.ExceptionRequest.var"#exceptions#1#3"{CloudBase.var"#cloudsign#64"{CloudBase.var"#cloudsign#63#65"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}},   method::String, url::String, h::Vector{Any}, b::Dict{String, String},   q::Nothing; headers::Vector{Any}, body::Dict{String, String}, query::Nothing,   kw::@Kwargs{credentials::Nothing})
@ HTTP /usr/local/.julia/packages/HTTP/LsfSO/src/HTTP.jl:457
[25] post(::String, ::Vararg{Any};   kw::@Kwargs{credentials::Nothing})
@ CloudBase.AWS   /usr/local/.julia/packages/CloudBase/ZAkU5/src/CloudBase.jl:124
[26] loadRoleArn(roleArn::String,   credFile::String, configFile::String)
@ CloudBase /usr/local/.julia/packages/CloudBase/ZAkU5/src/aws.jl:233
[27] awsLoadConfig!(profile::String,   expireThreshold::Dates.Minute)
@ CloudBase /usr/local/.julia/packages/CloudBase/ZAkU5/src/aws.jl:84
[28] #AWSCredentials#2
@ /usr/local/.julia/packages/CloudBase/ZAkU5/src/aws.jl:49 [inlined]
[29] AWSCredentials (repeats 2 times)
@ /usr/local/.julia/packages/CloudBase/ZAkU5/src/aws.jl:48 [inlined]
[30]   consolidate_partitions(queue_url::String, dead_letter_queue_url::String;   bin_column::String, column_dtype::Type, batch_size::Int64,   scan_precision::Int64, row_group_chunk_size::Int64, chunks_per_file::Int64,   big_file_cutoff::Int64, skip_clustering::Bool, max_batch_length::Int64)
@ Main.TableConsolidator /usr/local/src/TableConsolidator.jl:136
[31] top-level scope
@ /usr/local/consolidator_script.jl:12
mrufsvold commented 3 months ago

Bump @quinnj

Ended up running with AWS.jl this quarter, but getting the script working locally is just so tantalizing. Really hoping to get this running before next quarter!

mrufsvold commented 1 month ago

Circling back with a bump since I saw @quinnj had some new commits on his branch. My work has slowed down a bit, so I have time to try to get this running again.