search

JuliaCI / Nanosoldier.jl

A package for running JuliaCI services on MIT's Nanosoldier cluster
Other
29 stars 15 forks source link

Hang during processing #163

Open maleadt opened 1 year ago

maleadt commented 1 year ago

Errors like this shouldn't hang the server:

Encountered error: encountered job loop error: HTTP.Exceptions.ConnectError("https://api.github.com/repos/JuliaLang/julia/commits/114e675fc26c5bfda5ac48f5efbe4fd0
3ec59116/comments", Base.IOError("connect: operation canceled (ECANCELED)", -125))

Stacktrace:
  [1] (::HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#
4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, socket_type_tls::Type, readtimeout::Int64, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose,
:status_exception, :idle_timeout), Tuple{Nothing, Nothing, Int64, Bool, Int64}}})
    @ HTTP.ConnectionRequest /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/ConnectionRequest.jl:80
  [2] (::HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var
"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}})(req::HTTP.Messages.Request; canonicalize_headers::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:iofunction,
 :decompress, :verbose, :status_exception, :idle_timeout), Tuple{Nothing, Nothing, Int64, Bool, Int64}}})
    @ HTTP.CanonicalizeRequest /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/CanonicalizeRequest.jl:17
  [3] (::Base.var"#88#90"{Base.var"#88#89#91"{ExponentialBackOff, HTTP.RetryRequest.var"#3#6"{Int64, typeof(HTTP.RetryRequest.FALSE), HTTP.Messages.Request, Base.RefValue{Int64}}, HTTP.CanonicalizeReque
st.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamReques
t.streamlayer)}}}}}}}}}})(args::HTTP.Messages.Request; kwargs::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :status_exception, :idle_timeout), Tuple{Nothing, Nothing, Int64, Bool, Int64}}})
    @ Base ./error.jl:296
  [4] (::HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#
2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_delays::ExponentialBackO
ff, retry_check::Function, retry_non_idempotent::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :status_exception, :idle_timeout), Tuple{Nothing, Nothing, Int64, Bool, Int64}}})
    @ HTTP.RetryRequest /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/RetryRequest.jl:75
  [5] (::HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTT
P.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}})(req::HTTP.Messages.Reques
t; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :status_exception, :idle_timeout), Tuple{Nothing, Nothing, Int64, Bool, Int64}}})
    @ HTTP.CookieRequest /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/CookieRequest.jl:42
  [6] (::HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}})(req::HTTP.Messages.Request; detect_content_type::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :status_exception, :idle_timeout), Tuple{Nothing, Nothing, Int64, Bool, Int64}}})
    @ HTTP.ContentTypeDetection /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/ContentTypeRequest.jl:23
  [7] (::HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; basicauth::Bool, kw::Base.Pairs{Symbol, Union{Nothing, Integer}, NTuple{5, Symbol}, NamedTuple{(:iofunction, :decompress, :verbose, :status_exception, :idle_timeout), Tuple{Nothing, Nothing, Int64, Bool, Int64}}})
    @ HTTP.BasicAuthRequest /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/BasicAuthRequest.jl:22
  [8] (::HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; iofunction::Nothing, decompress::Nothing, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:verbose, :status_exception, :idle_timeout), Tuple{Int64, Bool, Int64}}})
    @ HTTP.DefaultHeadersRequest /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/DefaultHeadersRequest.jl:47
  [9] (::HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Nothing, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:verbose, :status_exception, :idle_timeout), Tuple{Int64, Bool, Int64}}})
    @ HTTP.RedirectRequest /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/RedirectRequest.jl:25
 [10] #1#2
    @ /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/DebugRequest.jl:23 [inlined]
 [11] (::HTTP.MessageRequest.var"#1#3"{HTTP.MessageRequest.var"#1#2#4"{HTTP.DebugRequest.var"#1#4"{HTTP.DebugRequest.var"#1#2#5"{HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}}}})(method::String, url::URIs.URI, headers::Dict{String, String}, body::String; copyheaders::Bool, response_stream::Nothing, http_version::HTTP.Strings.HTTPVersion, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:redirect, :status_exception, :idle_timeout), Tuple{Bool, Bool, Int64}}})
    @ HTTP.MessageRequest /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/clientlayers/MessageRequest.jl:27
 [12] request(stack::HTTP.MessageRequest.var"#1#3"{HTTP.MessageRequest.var"#1#2#4"{HTTP.DebugRequest.var"#1#4"{HTTP.DebugRequest.var"#1#2#5"{HTTP.RedirectRequest.var"#1#4"{HTTP.RedirectRequest.var"#1#2#5"{HTTP.DefaultHeadersRequest.var"#1#3"{HTTP.DefaultHeadersRequest.var"#1#2#4"{HTTP.BasicAuthRequest.var"#1#3"{HTTP.BasicAuthRequest.var"#1#2#4"{HTTP.ContentTypeDetection.var"#1#3"{HTTP.ContentTypeDetection.var"#1#2#4"{HTTP.CookieRequest.var"#1#5"{HTTP.CookieRequest.var"#1#2#6"{HTTP.RetryRequest.var"#1#4"{HTTP.RetryRequest.var"#1#2#5"{HTTP.CanonicalizeRequest.var"#1#3"{HTTP.CanonicalizeRequest.var"#1#2#4"{HTTP.ConnectionRequest.var"#1#7"{HTTP.ConnectionRequest.var"#1#2#8"{HTTP.TimeoutRequest.var"#1#6"{HTTP.TimeoutRequest.var"#1#2#7"{HTTP.ExceptionRequest.var"#1#3"{HTTP.ExceptionRequest.var"#1#2#4"{typeof(HTTP.StreamRequest.streamlayer)}}}}}}}}}}}}}}}}}}}}}}}}, method::String, url::String, h::Dict{String, String}, b::String, q::Nothing; headers::Dict{String, String}, body::String, query::Nothing, kw::Base.Pairs{Symbol, Integer, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:redirect, :status_exception, :idle_timeout), Tuple{Bool, Bool, Int64}}})
    @ HTTP /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/HTTP.jl:426
 [13] #request#16
    @ /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/HTTP.jl:286 [inlined]
 [14] #post#36
    @ /srv/nanosoldier/depot/packages/HTTP/RxlHW/src/HTTP.jl:501 [inlined]
 [15] github_request(api::GitHub.GitHubWebAPI, request_method::typeof(HTTP.post), endpoint::String; auth::GitHub.OAuth2, handle_error::Bool, headers::Dict{Any, Any}, params::Dict{String, String}, allowr
edirects::Bool)
    @ GitHub /srv/nanosoldier/depot/packages/GitHub/Ij6eH/src/utils/requests.jl:66
 [16] #gh_post#5
    @ /srv/nanosoldier/depot/packages/GitHub/Ij6eH/src/utils/requests.jl:73 [inlined]
 [17] gh_post_json(api::GitHub.GitHubWebAPI, endpoint::String; options::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:auth, :params), Tuple{GitHub.OAuth2, Dict{String, String}}}})
    @ GitHub /srv/nanosoldier/depot/packages/GitHub/Ij6eH/src/utils/requests.jl:79
 [18] create_comment(api::GitHub.GitHubWebAPI, repo::String, item::String, kind::Symbol; options::Base.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:auth, :params), Tuple{GitHub.OAuth2, Dict{St
ring, String}}}})
    @ GitHub /srv/nanosoldier/depot/packages/GitHub/Ij6eH/src/issues/comments.jl:71
 [19] #create_comment#210
    @ ./none:0 [inlined]
 [20] reply_comment(sub::Nanosoldier.JobSubmission, message::String)
    @ Nanosoldier /srv/nanosoldier/pkg/Nanosoldier/src/submission.jl:153
 [21] delegate_job(server::Nanosoldier.Server, job::Nanosoldier.PkgEvalJob, node::Int64)
    @ Nanosoldier /srv/nanosoldier/pkg/Nanosoldier/src/server.jl:141
 [22] macro expansion
    @ /srv/nanosoldier/pkg/Nanosoldier/src/server.jl:93 [inlined]
 [23] (::Nanosoldier.var"#76#77"{Nanosoldier.Server, DataType, Int64, Vector{Int64}})()
    @ Nanosoldier ./task.jl:484