search

JuliaGeo / GADM.jl

A Julia package for obtaining geographical data from the GADM dataset
https://gadm.org
MIT License
38 stars 4 forks source link

GADM is down... and maybe the error could be more informative. #41

Closed rafaqz closed 3 years ago

rafaqz commented 3 years ago

Running the readme command 

 GADM.coordinates("IND")

the prompt asks to download the data twice, and does start... but ends with a 403 error.

Turns out this is because GADM is down: this file wont load in the browser either. https://data.biogeo.ucdavis.edu/data/gadm3.6/gpkg/gadm36_IND_gpkg.zip

Seems ucdavis doesn't run reliable servers https://stackoverflow.com/questions/58265951/http-status-was-403-forbidden-when-trying-to-download-spatial-polygon-datafram

Its a minor point, but it's the error output is pretty long, with an apparently duplicated stack trace. I don't handle this possibility in RasterDataSources.jl either, but it seems like we should account for it.

We could maybe have some `The host site appears to be down" message and a url to the actual file to test in the browser, and reassure people its not a code problem, and that waiting or contacting GADM is all they can do.

Here's the current stack trace:

┌ Info: Downloading
│   source = "https://biogeo.ucdavis.edu/data/gadm3.6/gpkg/gadm36_IND_gpkg.zip"
│   dest = "/home/raf/.julia/datadeps/GADM_IND/gadm36_IND_gpkg.zip"
│   progress = 1.0
│   time_taken = "0.0 s"
│   time_remaining = "0.0 s"
│   average_speed = "∞ B/s"
│   downloaded = "289 bytes"
│   remaining = "0 bytes"
└   total = "289 bytes"
ERROR: HTTP.ExceptionRequest.StatusError(403, "GET", "/data/gadm3.6/gpkg/gadm36_IND_gpkg.zip", HTTP.Messages.Response:
"""
HTTP/1.1 403 Forbidden
Date: Sun, 03 Oct 2021 09:43:07 GMT
Server: Apache/2.4.18 (Ubuntu)
Content-Length: 289
Content-Type: text/html; charset=iso-8859-1

""")
Stacktrace:
  [1] request(::Type{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}, ::URIs.URI, ::Vararg{Any, N} where N; kw::Base.Iterators.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:iofunction, :reached_redirect_limit), Tuple{HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, Bool}}})
    @ HTTP.ExceptionRequest ~/.julia/packages/HTTP/qszg7/src/ExceptionRequest.jl:22
  [2] (::Base.var"#70#72"{Base.var"#70#71#73"{ExponentialBackOff, HTTP.RetryRequest.var"#2#3"{Bool, HTTP.Messages.Request}, typeof(HTTP.request)}})(::Type, ::Vararg{Any, N} where N; kwargs::Base.Iterators.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:iofunction, :reached_redirect_limit), Tuple{HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, Bool}}})
    @ Base ./error.jl:301
  [3] #request#1
    @ ~/.julia/packages/HTTP/qszg7/src/RetryRequest.jl:44 [inlined]
  [4] request(::Type{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}, method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::Nothing; http_version::VersionNumber, target::String, parent::HTTP.Messages.Response, iofunction::Function, kw::Base.Iterators.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:reached_redirect_limit,), Tuple{Bool}}})
    @ HTTP.MessageRequest ~/.julia/packages/HTTP/qszg7/src/MessageRequest.jl:66
  [5] request(::Type{HTTP.BasicAuthRequest.BasicAuthLayer{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}}, method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::Nothing; kw::Base.Iterators.Pairs{Symbol, Any, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:reached_redirect_limit, :iofunction, :parent), Tuple{Bool, HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, HTTP.Messages.Response}}})
    @ HTTP.BasicAuthRequest ~/.julia/packages/HTTP/qszg7/src/BasicAuthRequest.jl:28
  [6] request(::Type{HTTP.RedirectRequest.RedirectLayer{HTTP.BasicAuthRequest.BasicAuthLayer{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}}}, method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::Nothing; redirect_limit::Int64, forwardheaders::Bool, kw::Base.Iterators.Pairs{Symbol, HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, Tuple{Symbol}, NamedTuple{(:iofunction,), Tuple{HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}}}})
    @ HTTP.RedirectRequest ~/.julia/packages/HTTP/qszg7/src/RedirectRequest.jl:28
  [7] #request#1
    @ ~/.julia/packages/HTTP/qszg7/src/TopRequest.jl:15 [inlined]
  [8] request(method::String, url::String, h::Vector{Pair{SubString{String}, SubString{String}}}, b::Nothing; headers::Vector{Pair{SubString{String}, SubString{String}}}, body::Nothing, query::Nothing, kw::Base.Iterators.Pairs{Symbol, HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, Tuple{Symbol}, NamedTuple{(:iofunction,), Tuple{HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}}}})
    @ HTTP ~/.julia/packages/HTTP/qszg7/src/HTTP.jl:330
  [9] #open#11
    @ ~/.julia/packages/HTTP/qszg7/src/HTTP.jl:364 [inlined]
 [10] open
    @ ~/.julia/packages/HTTP/qszg7/src/HTTP.jl:364 [inlined]
 [11] #download#27
    @ ~/.julia/packages/HTTP/qszg7/src/download.jl:110 [inlined]
 [12] fetch_http(remotepath::String, localdir::String; update_period::Float32)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/fetch_helpers.jl:80
 [13] fetch_http
    @ ~/.julia/packages/DataDeps/ooWXe/src/fetch_helpers.jl:79 [inlined]
 [14] fetch_default(remotepath::String, localdir::String)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/fetch_helpers.jl:33
 [15] run_fetch
    @ ~/.julia/packages/DataDeps/ooWXe/src/resolution_automatic.jl:99 [inlined]
 [16] download(datadep::DataDeps.DataDep{Nothing, String, typeof(DataDeps.fetch_default), typeof(DataDeps.unpack)}, localdir::String; remotepath::String, i_accept_the_terms_of_use::Nothing, skip_checksum::Bool)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/resolution_automatic.jl:78
 [17] download
    @ ~/.julia/packages/DataDeps/ooWXe/src/resolution_automatic.jl:70 [inlined]
 [18] handle_missing
    @ ~/.julia/packages/DataDeps/ooWXe/src/resolution_automatic.jl:10 [inlined]
 [19] _resolve(datadep::DataDeps.DataDep{Nothing, String, typeof(DataDeps.fetch_default), typeof(DataDeps.unpack)}, calling_filepath::String)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/resolution.jl:83
 [20] resolve(datadep::DataDeps.DataDep{Nothing, String, typeof(DataDeps.fetch_default), typeof(DataDeps.unpack)}, inner_filepath::String, calling_filepath::String)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/resolution.jl:29
 [21] resolve(datadep_name::String, inner_filepath::String, calling_filepath::String)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/resolution.jl:54
 [22] resolve
    @ ~/.julia/packages/DataDeps/ooWXe/src/resolution.jl:73 [inlined]
 [23] download(country::String)
    @ GADM ~/.julia/packages/GADM/OFrEW/src/GADM.jl:36
 [24] |>
    @ ./operators.jl:858 [inlined]
 [25] getdataset
    @ ~/.julia/packages/GADM/OFrEW/src/GADM.jl:63 [inlined]
 [26] get(::String; children::Bool)
    @ GADM ~/.julia/packages/GADM/OFrEW/src/GADM.jl:104
 [27] get
    @ ~/.julia/packages/GADM/OFrEW/src/GADM.jl:104 [inlined]
 [28] coordinates(::String)
    @ GADM ~/.julia/packages/GADM/OFrEW/src/GADM.jl:160
 [29] top-level scope
    @ REPL[10]:1

caused by: HTTP.ExceptionRequest.StatusError(403, "GET", "/data/gadm3.6/gpkg/gadm36_IND_gpkg.zip", HTTP.Messages.Response:
"""
HTTP/1.1 403 Forbidden
Date: Sun, 03 Oct 2021 09:42:53 GMT
Server: Apache/2.4.18 (Ubuntu)
Content-Length: 289
Content-Type: text/html; charset=iso-8859-1

""")
Stacktrace:
  [1] request(::Type{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}, ::URIs.URI, ::Vararg{Any, N} where N; kw::Base.Iterators.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:iofunction, :reached_redirect_limit), Tuple{HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, Bool}}})
    @ HTTP.ExceptionRequest ~/.julia/packages/HTTP/qszg7/src/ExceptionRequest.jl:22
  [2] (::Base.var"#70#72"{Base.var"#70#71#73"{ExponentialBackOff, HTTP.RetryRequest.var"#2#3"{Bool, HTTP.Messages.Request}, typeof(HTTP.request)}})(::Type, ::Vararg{Any, N} where N; kwargs::Base.Iterators.Pairs{Symbol, Any, Tuple{Symbol, Symbol}, NamedTuple{(:iofunction, :reached_redirect_limit), Tuple{HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, Bool}}})
    @ Base ./error.jl:301
  [3] #request#1
    @ ~/.julia/packages/HTTP/qszg7/src/RetryRequest.jl:44 [inlined]
  [4] request(::Type{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}, method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::Nothing; http_version::VersionNumber, target::String, parent::HTTP.Messages.Response, iofunction::Function, kw::Base.Iterators.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:reached_redirect_limit,), Tuple{Bool}}})
    @ HTTP.MessageRequest ~/.julia/packages/HTTP/qszg7/src/MessageRequest.jl:66
  [5] request(::Type{HTTP.BasicAuthRequest.BasicAuthLayer{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}}, method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::Nothing; kw::Base.Iterators.Pairs{Symbol, Any, Tuple{Symbol, Symbol, Symbol}, NamedTuple{(:reached_redirect_limit, :iofunction, :parent), Tuple{Bool, HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, HTTP.Messages.Response}}})
    @ HTTP.BasicAuthRequest ~/.julia/packages/HTTP/qszg7/src/BasicAuthRequest.jl:28
  [6] request(::Type{HTTP.RedirectRequest.RedirectLayer{HTTP.BasicAuthRequest.BasicAuthLayer{HTTP.MessageRequest.MessageLayer{HTTP.RetryRequest.RetryLayer{HTTP.ExceptionRequest.ExceptionLayer{HTTP.ConnectionRequest.ConnectionPoolLayer{HTTP.StreamRequest.StreamLayer{Union{}}}}}}}}}, method::String, url::URIs.URI, headers::Vector{Pair{SubString{String}, SubString{String}}}, body::Nothing; redirect_limit::Int64, forwardheaders::Bool, kw::Base.Iterators.Pairs{Symbol, HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, Tuple{Symbol}, NamedTuple{(:iofunction,), Tuple{HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}}}})
    @ HTTP.RedirectRequest ~/.julia/packages/HTTP/qszg7/src/RedirectRequest.jl:28
  [7] #request#1
    @ ~/.julia/packages/HTTP/qszg7/src/TopRequest.jl:15 [inlined]
  [8] request(method::String, url::String, h::Vector{Pair{SubString{String}, SubString{String}}}, b::Nothing; headers::Vector{Pair{SubString{String}, SubString{String}}}, body::Nothing, query::Nothing, kw::Base.Iterators.Pairs{Symbol, HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}, Tuple{Symbol}, NamedTuple{(:iofunction,), Tuple{HTTP.var"#28#34"{Float32, String, String, Vector{String}, HTTP.var"#format_bytes_per_second#33"{HTTP.var"#format_bytes#31"}, HTTP.var"#format_seconds#32", HTTP.var"#format_bytes#31", HTTP.var"#format_progress#30"}}}})
    @ HTTP ~/.julia/packages/HTTP/qszg7/src/HTTP.jl:330
  [9] #open#11
    @ ~/.julia/packages/HTTP/qszg7/src/HTTP.jl:364 [inlined]
 [10] open
    @ ~/.julia/packages/HTTP/qszg7/src/HTTP.jl:364 [inlined]
 [11] #download#27
    @ ~/.julia/packages/HTTP/qszg7/src/download.jl:110 [inlined]
 [12] fetch_http(remotepath::String, localdir::String; update_period::Float32)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/fetch_helpers.jl:80
 [13] fetch_http
    @ ~/.julia/packages/DataDeps/ooWXe/src/fetch_helpers.jl:79 [inlined]
 [14] fetch_default(remotepath::String, localdir::String)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/fetch_helpers.jl:33
 [15] run_fetch
    @ ~/.julia/packages/DataDeps/ooWXe/src/resolution_automatic.jl:99 [inlined]
 [16] download(datadep::DataDeps.DataDep{Nothing, String, typeof(DataDeps.fetch_default), typeof(DataDeps.unpack)}, localdir::String; remotepath::String, i_accept_the_terms_of_use::Nothing, skip_checksum::Bool)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/resolution_automatic.jl:78
 [17] download
    @ ~/.julia/packages/DataDeps/ooWXe/src/resolution_automatic.jl:70 [inlined]
 [18] handle_missing
    @ ~/.julia/packages/DataDeps/ooWXe/src/resolution_automatic.jl:10 [inlined]
 [19] _resolve(datadep::DataDeps.DataDep{Nothing, String, typeof(DataDeps.fetch_default), typeof(DataDeps.unpack)}, calling_filepath::String)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/resolution.jl:83
 [20] resolve(datadep::DataDeps.DataDep{Nothing, String, typeof(DataDeps.fetch_default), typeof(DataDeps.unpack)}, inner_filepath::String, calling_filepath::String)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/resolution.jl:29
 [21] resolve(datadep_name::String, inner_filepath::String, calling_filepath::String)
    @ DataDeps ~/.julia/packages/DataDeps/ooWXe/src/resolution.jl:54
 [22] resolve
    @ ~/.julia/packages/DataDeps/ooWXe/src/resolution.jl:73 [inlined]
 [23] download(country::String)
    @ GADM ~/.julia/packages/GADM/OFrEW/src/GADM.jl:28
 [24] |>
    @ ./operators.jl:858 [inlined]
 [25] getdataset
    @ ~/.julia/packages/GADM/OFrEW/src/GADM.jl:63 [inlined]
 [26] get(::String; children::Bool)
    @ GADM ~/.julia/packages/GADM/OFrEW/src/GADM.jl:104
 [27] get
    @ ~/.julia/packages/GADM/OFrEW/src/GADM.jl:104 [inlined]
 [28] coordinates(::String)
    @ GADM ~/.julia/packages/GADM/OFrEW/src/GADM.jl:160
 [29] top-level scope
    @ REPL[10]:1
juliohm commented 3 years ago

More generally, I think this error message should go in DataDeps.jl whenever the download is unsuccessful. Can you please open an issue there and reference this one?

rafaqz commented 3 years ago

Maybe leave this open until the package is actually working again ;)