search

EcoJulia / RasterDataSources.jl

Easily download and use raster data sets in Julia
MIT License
21 stars 10 forks source link

Cannot download WorldClim Weather data #64

Closed felixcremer closed 1 month ago

felixcremer commented 2 months ago

When I try to download WorldClim Weather data I get a 404 hidden in some long stacktraces. It seems that that data is not available where it used to be. Could we fail more gracefully here?

julia> getraster(WorldClim{Weather}, :tavg, date=Date(2000,1,1))
[ Info: Starting download for https://biogeo.ucdavis.edu/data/worldclim/v2.1/hist/wc2.1_2.5m_tavg_2000-2009.zip
┌ Info: Downloading
│   source = "https://biogeo.ucdavis.edu/data/worldclim/v2.1/hist/wc2.1_2.5m_tavg_2000-2009.zip"
│   dest = "/home/fcremer/Daten/WorldClim/Weather/zips/wc2.1_2.5m_tavg_2000-2009.zip"
│   progress = 1.0
│   time_taken = "0.0 s"
│   time_remaining = "0.0 s"
│   average_speed = "346.680 KiB/s"
│   downloaded = "355 bytes"
│   remaining = "0 bytes"
└   total = "355 bytes"
┌ Info: Downloading
│   source = "https://biogeo.ucdavis.edu/data/worldclim/v2.1/hist/wc2.1_2.5m_tavg_2000-2009.zip"
│   dest = "/home/fcremer/Daten/WorldClim/Weather/zips/wc2.1_2.5m_tavg_2000-2009.zip"
│   progress = 1.0
│   time_taken = "0.0 s"
│   time_remaining = "0.0 s"
│   average_speed = "∞ B/s"
│   downloaded = "223 bytes"
│   remaining = "0 bytes"
└   total = "223 bytes"
ERROR: HTTP.Exceptions.StatusError(404, "GET", "/data/worldclim/v2.1/hist/wc2.1_2.5m_tavg_2000-2009.zip", HTTP.Messages.Response:
"""
HTTP/1.1 404 Not Found
Date: Fri, 26 Apr 2024 09:28:37 GMT
Server: Apache/2.4.18 (Ubuntu)
Last-Modified: Mon, 12 Oct 2020 23:20:08 GMT
ETag: "df-5b1818a0b093c;59fea9729ba29"
Accept-Ranges: bytes
Content-Length: 223
Content-Type: text/html

""")
Stacktrace:
 [1] _maybe_download(uri::URIs.URI, filepath::String, headers::Vector{Any})
   @ RasterDataSources ~/.julia/packages/RasterDataSources/xWdNX/src/shared.jl:71
 [2] _maybe_download
   @ ~/.julia/packages/RasterDataSources/xWdNX/src/shared.jl:63 [inlined]
 [3] _getraster(T::Type{WorldClim{Weather}}, layer::Symbol, date::Date)
   @ RasterDataSources ~/.julia/packages/RasterDataSources/xWdNX/src/worldclim/weather.jl:38
 [4] #getraster#24
   @ ~/.julia/packages/RasterDataSources/xWdNX/src/worldclim/weather.jl:18 [inlined]
 [5] top-level scope
   @ REPL[14]:1

caused by: HTTP.Exceptions.StatusError(404, "GET", "/data/worldclim/v2.1/hist/wc2.1_2.5m_tavg_2000-2009.zip", HTTP.Messages.Response:
"""
HTTP/1.1 404 Not Found
Date: Fri, 26 Apr 2024 09:28:37 GMT
Server: Apache/2.4.18 (Ubuntu)
Last-Modified: Mon, 12 Oct 2020 23:20:08 GMT
ETag: "df-5b1818a0b093c;59fea9729ba29"
Accept-Ranges: bytes
Content-Length: 223
Content-Type: text/html

""")
Stacktrace:
  [1] (::HTTP.ConnectionRequest.var"#connections#4"{…})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, socket_type_tls::Nothing, readtimeout::Int64, connect_timeout::Int64, logerrors::Bool, logtag::Nothing, kw::@Kwargs{…})
    @ HTTP.ConnectionRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/ConnectionRequest.jl:141
  [2] connections
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/ConnectionRequest.jl:58 [inlined]
  [3] (::Base.var"#96#98"{Base.var"#96#97#99"{…}})(args::HTTP.Messages.Request; kwargs::@Kwargs{iofunction::HTTP.var"#11#17"{…}, decompress::Nothing, verbose::Int64})
    @ Base ./error.jl:296
  [4] (::HTTP.RetryRequest.var"#manageretries#3"{…})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_delays::ExponentialBackOff, retry_check::Function, retry_non_idempotent::Bool, kw::@Kwargs{…})
    @ HTTP.RetryRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/RetryRequest.jl:75
  [5] manageretries
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/RetryRequest.jl:30 [inlined]
  [6] (::HTTP.CookieRequest.var"#managecookies#4"{…})(req::HTTP.Messages.Request; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::@Kwargs{…})
    @ HTTP.CookieRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/CookieRequest.jl:42
  [7] (::HTTP.HeadersRequest.var"#defaultheaders#2"{…})(req::HTTP.Messages.Request; iofunction::Function, decompress::Nothing, basicauth::Bool, detect_content_type::Bool, canonicalize_headers::Bool, kw::@Kwargs{…})
    @ HTTP.HeadersRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/HeadersRequest.jl:71
  [8] defaultheaders
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/HeadersRequest.jl:14 [inlined]
  [9] (::HTTP.RedirectRequest.var"#redirects#3"{…})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Nothing, kw::@Kwargs{…})
    @ HTTP.RedirectRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/RedirectRequest.jl:25
 [10] redirects
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/RedirectRequest.jl:14 [inlined]
 [11] (::HTTP.MessageRequest.var"#makerequest#3"{…})(method::String, url::URIs.URI, headers::Vector{…}, body::Nothing; copyheaders::Bool, response_stream::Nothing, http_version::HTTP.Strings.HTTPVersion, verbose::Int64, kw::@Kwargs{…})
    @ HTTP.MessageRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/MessageRequest.jl:35
 [12] makerequest
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/MessageRequest.jl:24 [inlined]
 [13] request(stack::HTTP.MessageRequest.var"#makerequest#3"{…}, method::String, url::String, h::Vector{…}, b::Nothing, q::Nothing; headers::Vector{…}, body::Nothing, query::Nothing, kw::@Kwargs{…})
    @ HTTP ~/.julia/packages/HTTP/PnoHb/src/HTTP.jl:457
 [14] #request#20
    @ ~/.julia/packages/HTTP/PnoHb/src/HTTP.jl:315 [inlined]
 [15] request
    @ ~/.julia/packages/HTTP/PnoHb/src/HTTP.jl:313 [inlined]
 [16] open
    @ ~/.julia/packages/HTTP/PnoHb/src/HTTP.jl:579 [inlined]
 [17] download(url::String, local_path::String, headers::Vector{Any}; update_period::Int64, kw::@Kwargs{})
    @ HTTP ~/.julia/packages/HTTP/PnoHb/src/download.jl:106
 [18] download
    @ ~/.julia/packages/HTTP/PnoHb/src/download.jl:96 [inlined]
 [19] _maybe_download(uri::URIs.URI, filepath::String, headers::Vector{Any})
    @ RasterDataSources ~/.julia/packages/RasterDataSources/xWdNX/src/shared.jl:67
 [20] _maybe_download
    @ ~/.julia/packages/RasterDataSources/xWdNX/src/shared.jl:63 [inlined]
 [21] _getraster(T::Type{WorldClim{Weather}}, layer::Symbol, date::Date)
    @ RasterDataSources ~/.julia/packages/RasterDataSources/xWdNX/src/worldclim/weather.jl:38
 [22] #getraster#24
    @ ~/.julia/packages/RasterDataSources/xWdNX/src/worldclim/weather.jl:18 [inlined]
 [23] top-level scope
    @ REPL[14]:1

caused by: HTTP.Exceptions.StatusError(404, "GET", "/data/worldclim/v2.1/hist/wc2.1_2.5m_tavg_2000-2009.zip", HTTP.Messages.Response:
"""
HTTP/1.1 404 Not Found
Date: Fri, 26 Apr 2024 09:28:37 GMT
Server: Apache/2.4.18 (Ubuntu)
Last-Modified: Mon, 12 Oct 2020 23:20:08 GMT
ETag: "df-5b1818a0b093c;59fea9729ba29"
Accept-Ranges: bytes
Content-Length: 223
Content-Type: text/html

""")
Stacktrace:
  [1] (::HTTP.ExceptionRequest.var"#exceptions#2"{…})(stream::HTTP.Streams.Stream{…}; status_exception::Bool, timedout::Nothing, logerrors::Bool, logtag::Nothing, kw::@Kwargs{…})
    @ HTTP.ExceptionRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/ExceptionRequest.jl:19
  [2] exceptions
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/ExceptionRequest.jl:13 [inlined]
  [3] (::HTTP.TimeoutRequest.var"#timeouts#3"{…})(stream::HTTP.Streams.Stream{…}; readtimeout::Int64, logerrors::Bool, logtag::Nothing, kw::@Kwargs{…})
    @ HTTP.TimeoutRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/TimeoutRequest.jl:18
  [4] (::HTTP.ConnectionRequest.var"#connections#4"{…})(req::HTTP.Messages.Request; proxy::Nothing, socket_type::Type, socket_type_tls::Nothing, readtimeout::Int64, connect_timeout::Int64, logerrors::Bool, logtag::Nothing, kw::@Kwargs{…})
    @ HTTP.ConnectionRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/ConnectionRequest.jl:119
  [5] connections
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/ConnectionRequest.jl:58 [inlined]
  [6] (::Base.var"#96#98"{Base.var"#96#97#99"{…}})(args::HTTP.Messages.Request; kwargs::@Kwargs{iofunction::HTTP.var"#11#17"{…}, decompress::Nothing, verbose::Int64})
    @ Base ./error.jl:296
  [7] (::HTTP.RetryRequest.var"#manageretries#3"{…})(req::HTTP.Messages.Request; retry::Bool, retries::Int64, retry_delays::ExponentialBackOff, retry_check::Function, retry_non_idempotent::Bool, kw::@Kwargs{…})
    @ HTTP.RetryRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/RetryRequest.jl:75
  [8] manageretries
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/RetryRequest.jl:30 [inlined]
  [9] (::HTTP.CookieRequest.var"#managecookies#4"{…})(req::HTTP.Messages.Request; cookies::Bool, cookiejar::HTTP.Cookies.CookieJar, kw::@Kwargs{…})
    @ HTTP.CookieRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/CookieRequest.jl:42
 [10] (::HTTP.HeadersRequest.var"#defaultheaders#2"{…})(req::HTTP.Messages.Request; iofunction::Function, decompress::Nothing, basicauth::Bool, detect_content_type::Bool, canonicalize_headers::Bool, kw::@Kwargs{…})
    @ HTTP.HeadersRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/HeadersRequest.jl:71
 [11] defaultheaders
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/HeadersRequest.jl:14 [inlined]
 [12] (::HTTP.RedirectRequest.var"#redirects#3"{…})(req::HTTP.Messages.Request; redirect::Bool, redirect_limit::Int64, redirect_method::Nothing, forwardheaders::Bool, response_stream::Nothing, kw::@Kwargs{…})
    @ HTTP.RedirectRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/RedirectRequest.jl:25
 [13] redirects
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/RedirectRequest.jl:14 [inlined]
 [14] (::HTTP.MessageRequest.var"#makerequest#3"{…})(method::String, url::URIs.URI, headers::Vector{…}, body::Nothing; copyheaders::Bool, response_stream::Nothing, http_version::HTTP.Strings.HTTPVersion, verbose::Int64, kw::@Kwargs{…})
    @ HTTP.MessageRequest ~/.julia/packages/HTTP/PnoHb/src/clientlayers/MessageRequest.jl:35
 [15] makerequest
    @ ~/.julia/packages/HTTP/PnoHb/src/clientlayers/MessageRequest.jl:24 [inlined]
 [16] request(stack::HTTP.MessageRequest.var"#makerequest#3"{…}, method::String, url::String, h::Vector{…}, b::Nothing, q::Nothing; headers::Vector{…}, body::Nothing, query::Nothing, kw::@Kwargs{…})
    @ HTTP ~/.julia/packages/HTTP/PnoHb/src/HTTP.jl:457
 [17] #request#20
    @ ~/.julia/packages/HTTP/PnoHb/src/HTTP.jl:315 [inlined]
 [18] request
    @ ~/.julia/packages/HTTP/PnoHb/src/HTTP.jl:313 [inlined]
 [19] open
    @ ~/.julia/packages/HTTP/PnoHb/src/HTTP.jl:579 [inlined]
 [20] download(url::String, local_path::String, headers::Vector{Any}; update_period::Int64, kw::@Kwargs{})
    @ HTTP ~/.julia/packages/HTTP/PnoHb/src/download.jl:106
 [21] download
    @ ~/.julia/packages/HTTP/PnoHb/src/download.jl:96 [inlined]
 [22] _maybe_download(uri::URIs.URI, filepath::String, headers::Vector{Any})
    @ RasterDataSources ~/.julia/packages/RasterDataSources/xWdNX/src/shared.jl:67
 [23] _maybe_download
    @ ~/.julia/packages/RasterDataSources/xWdNX/src/shared.jl:63 [inlined]
 [24] _getraster(T::Type{WorldClim{Weather}}, layer::Symbol, date::Date)
    @ RasterDataSources ~/.julia/packages/RasterDataSources/xWdNX/src/worldclim/weather.jl:38
 [25] #getraster#24
    @ ~/.julia/packages/RasterDataSources/xWdNX/src/worldclim/weather.jl:18 [inlined]
 [26] top-level scope
    @ REPL[14]:1
Some type information was truncated. Use `show(err)` to see complete types.
rafaqz commented 2 months ago

This is just ucdavis IT being truly awful. Its down a few times a year.

If you want to make the errors nicer go for it.

sylvaticus commented 1 month ago

I think they changed the URL. Going to WorldClimate.org links are now as https://geodata.ucdavis.edu/climate/worldclim/2_1/base/..., while getraster() is looking at https://biogeo.ucdavis.edu/data/worldclim/v2.1/base/...

sylvaticus commented 1 month ago

Pull request to update WORLDCLIM_URI here .