Something is broken in download option in eurostat package

[rickecon]

@jpycroft @nikhilwoodruff. I posted this issue in the Bitbucket repository for the eurostat package. I have been working on @jpycroft's demographics.py PR #36 (my branch is here, which I will submit as a PR to @jpycroft's PR branch as soon as mine is ready). I have made some updates to the data cleaning and imputation at the tails of the births data.

These operations were working just fine as recently as last evening. But when I tried to download U.K. population and births data by age from EuroStat today using the Python eurostat package (v. 0.2.3), I got a persistent error today Server Error 500: DSD_demo_pjan not found in the Eurostat server. It looks like it could be an issue with the pandasdmx package. Or it could be that the EuroStat REST API server is down. I would be interested to see if anyone else has the same problem. The code I executed that created this error is the following:

import eurostat

Country = "UK"
Year = 2018
StartPeriod = Year
EndPeriod = Year
filter_pars = {"GEO": [Country]}
df_pop = eurostat.get_sdmx_data_df(
    "demo_pjan",
    StartPeriod,
    EndPeriod,
    filter_pars,
    flags=True,
    verbose=True
)

As I said, this download method was working just fine as of last night. The error traceback is the following:

Server Error 500: DSD_demo_pjan not found in the Eurostat server
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/pandasdmx/model.py in __call__(self, request, target_only, **kwargs)
    790         try:
--> 791             rc = getattr(self._reader.message, rc_name)
    792             if self.maintainable_parent_id:

AttributeError: 'StructureMessage' object has no attribute 'datastructure'

During handling of the above exception, another exception occurred:

SDMXException                             Traceback (most recent call last)
/var/folders/_2/fkr0rfrx2fz2r4_zpbgtdvzw0000gn/T/ipykernel_32395/19781219.py in <module>
     11     filter_pars,
     12     flags=True,
---> 13     verbose=True
     14 )

~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/eurostat/eurostat.py in get_sdmx_data_df(code, StartPeriod, EndPeriod, filter_pars, flags, verbose)
    302     """
    303 
--> 304     d = get_sdmx_data(code, StartPeriod, EndPeriod, filter_pars, flags, verbose)
    305 
    306     if d != None:

~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/eurostat/_decorators.py in wrapper(*args, **kwargs)
     59             proxydic = None
     60         estat = Request('ESTAT', timeout = 100., proxies = proxydic)
---> 61         res = func(estat, *args, **kwargs)
     62         return res
     63     return wrapper

~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/eurostat/eurostat.py in get_sdmx_data(estat, code, StartPeriod, EndPeriod, filter_pars, flags, verbose)
    267 
    268     for c in cart:
--> 269         resp = __pandasdmx_get_data(estat, code, key=dict(c), params={'startPeriod': str(StartPeriod), 'endPeriod': str(EndPeriod)})
    270         if flags:
    271             for s in resp.data.series:

~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/eurostat/_decorators.py in wrapper(*args, **kwargs)
     17         for att in range(max_att + 1):
     18             try:
---> 19                 resp = func(*args, **kwargs)
     20                 return resp
     21             except ValueError as value_error:

~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/eurostat/eurostat.py in __pandasdmx_get_data(estat, code, key, params)
    369 @robust_pandasdmx_request
    370 def __pandasdmx_get_data(estat, code, key, params):
--> 371     resp = estat.data(code, key=key, params=params)
    372     return resp
    373 

~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/pandasdmx/api.py in get(self, resource_type, resource_id, agency, version, key, params, headers, fromfile, tofile, url, get_footer_url, memcache, writer, dsd, series_keys)
    307                     if not hasattr(val_resp.msg, 'datastructure'):
    308                         val_resp = val_resp.dataflow[resource_id].structure(
--> 309                             request=True, target_only=False)
    310                 val_msg = val_resp.msg
    311                 # validate key

~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/pandasdmx/model.py in __call__(self, request, target_only, **kwargs)
    804                 resp = req.get(resource_type=rc_name,
    805                                resource_id=self.maintainable_parent_id or self.id,
--> 806                                agency=self.agency_id, **kwargs)
    807                 if target_only:
    808                     rc = getattr(resp.msg, rc_name)

~/opt/anaconda3/envs/oguk-dev/lib/python3.7/site-packages/pandasdmx/api.py in get(self, resource_type, resource_id, agency, version, key, params, headers, fromfile, tofile, url, get_footer_url, memcache, writer, dsd, series_keys)
    361             base_url, params=params, headers=headers, fromfile=fromfile)
    362         if source is None:
--> 363             raise SDMXException('Server error:', status_code, url)
    364         logger.info(
    365             'Loaded file into memory from URL/file: %s', (url or fromfile))

SDMXException: ('Server error:', 500, 'http://ec.europa.eu/eurostat/SDMX/diss-web/rest/datastructure/ESTAT/DSD_demo_pjan/latest?references=all')

I also received the same error for U.K. births data by age I was downloading using the following code:

df_births = eurostat.get_sdmx_data_df(
    "demo_fasec",
    StartPeriod,
    EndPeriod,
    filter_pars,
    flags=True,
    verbose=True,
)

[jpycroft]

Hi @rickecon, I get the same as you. Running the snippet of code, I got the DSD_demo_pjan not found error (full output below). The same error for demo_fasec: Server Error 500: DSD_demo_fasec not found in the Eurostat server

(oguk-dev) C:\Users\jonat\repos\anotherPracticeRepo>python anotherEurostatTest.py
Server Error 500: DSD_demo_pjan not found in the Eurostat server
Traceback (most recent call last):
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\pandasdmx\model.py", line 791, in __call__
    rc = getattr(self._reader.message, rc_name)
AttributeError: 'StructureMessage' object has no attribute 'datastructure'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "anotherEurostatTest.py", line 14, in <module>
    verbose=True
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\eurostat\eurostat.py", line 304, in get_sdmx_data_df
    d = get_sdmx_data(code, StartPeriod, EndPeriod, filter_pars, flags, verbose)
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\eurostat\_decorators.py", line 61, in wrapper
    res = func(estat, *args, **kwargs)
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\eurostat\eurostat.py", line 269, in get_sdmx_data
    resp = __pandasdmx_get_data(estat, code, key=dict(c), params={'startPeriod': str(StartPeriod), 'endPeriod': str(EndPeriod)})
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\eurostat\_decorators.py", line 19, in wrapper
    resp = func(*args, **kwargs)
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\eurostat\eurostat.py", line 371, in __pandasdmx_get_data
    resp = estat.data(code, key=key, params=params)
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\pandasdmx\api.py", line 309, in get
    request=True, target_only=False)
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\pandasdmx\model.py", line 806, in __call__
    agency=self.agency_id, **kwargs)
  File "C:\Users\jonat\anaconda3\envs\oguk-dev\lib\site-packages\pandasdmx\api.py", line 363, in get
    raise SDMXException('Server error:', status_code, url)
pandasdmx.api.SDMXException: ('Server error:', 500, 'http://ec.europa.eu/eurostat/SDMX/diss-web/rest/datastructure/ESTAT/DSD_demo_pjan/latest?references=all')

[rickecon]

@jpycroft @nikhilwoodruff. It is working now. Must have been the EuroStat REST API was down. Closing this issue.