Correct degradation example is failing consistently #125

Closed nabobalis closed 1 year ago

In GitLab by @wtbarnes on Sep 4, 2023, 13:08

The gallery example entry that shows the effect of degradation as a function of time over the first 11 years of the mission (https://aiapy.readthedocs.io/en/stable/generated/gallery/correct_degradation.html#sphx-glr-generated-gallery-correct-degradation-py) is failing consistently. This seems to be due to the JSOC query failing repeatedly: https://gitlab.com/wtbarnes/aiapy/-/jobs/5005334717#L240. This is currently causing all of our doc builds to fail.

Notably, this example is not making an export request. It is only querying the metadata associated with each record. I suspect that the large date range is what is causing this failure. Until recently (a few days ago) this has not been issue.

I am also able to reproduce this failure locally,

import astropy.units as u
from sunpy.net import Fido
from sunpy.net import attrs as a

results = Fido.search(
    a.Time("2010-06-01T00:00:00", "2021-06-01T00:00:00"),
    a.Sample(30 * u.day),
    a.jsoc.Series.aia_lev1_euv_12s,
    a.jsoc.Wavelength(335 * u.angstrom),
)

---------------------------------------------------------------------------
HTTPError                                 Traceback (most recent call last)
Cell In[2], line 1
----> 1 results = Fido.search(
      2     a.Time("2010-06-01T00:00:00", "2021-06-01T00:00:00"),
      3     a.Sample(30 * u.day),
      4     a.jsoc.Series.aia_lev1_euv_12s,
      5     a.jsoc.Wavelength(335 * u.angstrom),
      6 )

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/sunpy/net/fido_factory.py:315, in UnifiedDownloaderFactory.search(self, *query)
    269 """
    270 Query for data in form of multiple parameters.
    271
   (...)
    312 parts individually.
    313 """
    314 query = attr.and_(*query)
--> 315 results = query_walker.create(query, self)
    317 # If we have searched the VSO but no results were returned, but another
    318 # client generated results, we drop the empty VSO results for tidiness.
    319 # This is because the VSO _can_handle_query is very broad because we
    320 # don't know the full list of supported values we can search for (yet).
    321 results = [r for r in results if not isinstance(r, vso.VSOQueryResponseTable) or len(r) > 0]

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/sunpy/net/attr.py:613, in AttrWalker.create(self, *args, **kwargs)
    609 def create(self, *args, **kwargs):
    610     """
    611     Call the create function(s) matching the arguments to this method.
    612     """
--> 613     return self.createmm(self, *args, **kwargs)

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/sunpy/util/functools.py:18, in seconddispatch.<locals>.wrapper(*args, **kwargs)
     17 def wrapper(*args, **kwargs):
---> 18     return dispatcher.dispatch(args[1].__class__)(*args, **kwargs)

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/sunpy/net/fido_factory.py:244, in _create_and(walker, query, factory)
    242 @query_walker.add_creator(attr.AttrAnd)
    243 def _create_and(walker, query, factory):
--> 244     return factory._make_query_to_client(*query.attrs)

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/sunpy/net/fido_factory.py:486, in UnifiedDownloaderFactory._make_query_to_client(self, *query)
    484     if isinstance(tmpclient, vso.VSOClient):
    485         kwargs = dict(response_format="table")
--> 486     results.append(tmpclient.search(*query, **kwargs))
    488 # This method is called by `search` and the results are fed into a
    489 # UnifiedResponse object.
    490 return results

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/sunpy/net/jsoc/jsoc.py:307, in JSOCClient.search(self, *query, **kwargs)
    305     # Update blocks with deep copy of iargs because in _make_recordset we use .pop() on element from iargs
    306     blocks.append(copy.deepcopy(iargs))
--> 307     return_results = astropy.table.vstack([return_results, self._lookup_records(iargs)])
    308 return_results.query_args = blocks
    309 return_results._original_num_rows = len(return_results)

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/sunpy/net/jsoc/jsoc.py:768, in JSOCClient._lookup_records(self, iargs)
    766 log.debug(f"Running following query: {ds}")
    767 log.debug(f"Requesting following keywords: {key}")
--> 768 result = client.query(ds, key=key, rec_index=isMeta)
    769 if result is None or result.empty:
    770     return astropy.table.Table()

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/drms/client.py:1058, in Client.query(self, ds, key, seg, link, convert_numeric, skip_conversion, pkeys, rec_index, n)
   1055     key = [k for k in key if k not in pk]
   1056     key = pk + key
-> 1058 lres = self._json.rs_list(ds, key, seg, link, recinfo=rec_index, n=n)
   1059 status = lres.get("status")
   1060 if status != 0:

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/drms/json.py:265, in HttpJsonClient.rs_list(self, ds, key, seg, link, recinfo, n, uid)
    263     d["userhandle"] = uid
    264 query = f"?{urlencode(d)}"
--> 265 req = self._json_request(self._server.url_jsoc_info + query)
    266 return req.data

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/drms/json.py:120, in HttpJsonClient._json_request(self, url)
    118 if self.debug:
    119     print(url)
--> 120 return HttpJsonRequest(url, self._server.encoding)

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/drms/json.py:70, in HttpJsonRequest.__init__(self, url, encoding)
     68 except HTTPError as e:
     69     e.msg = f"Failed to open URL: {e.url} with {e.code} - {e.msg}"
---> 70     raise e
     71 self._data_str = None
     72 self._data = None

File ~/anaconda/envs/aiapy-dev/lib/python3.11/site-packages/drms/json.py:67, in HttpJsonRequest.__init__(self, url, encoding)
     65 self._encoding = encoding
     66 try:
---> 67     self._http = urlopen(url)
     68 except HTTPError as e:
     69     e.msg = f"Failed to open URL: {e.url} with {e.code} - {e.msg}"

File ~/anaconda/envs/aiapy-dev/lib/python3.11/urllib/request.py:216, in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    214 else:
    215     opener = _opener
--> 216 return opener.open(url, data, timeout)

File ~/anaconda/envs/aiapy-dev/lib/python3.11/urllib/request.py:525, in OpenerDirector.open(self, fullurl, data, timeout)
    523 for processor in self.process_response.get(protocol, []):
    524     meth = getattr(processor, meth_name)
--> 525     response = meth(req, response)
    527 return response

File ~/anaconda/envs/aiapy-dev/lib/python3.11/urllib/request.py:634, in HTTPErrorProcessor.http_response(self, request, response)
    631 # According to RFC 2616, "2xx" code indicates that the client's
    632 # request was successfully received, understood, and accepted.
    633 if not (200 <= code < 300):
--> 634     response = self.parent.error(
    635         'http', request, response, code, msg, hdrs)
    637 return response

File ~/anaconda/envs/aiapy-dev/lib/python3.11/urllib/request.py:563, in OpenerDirector.error(self, proto, *args)
    561 if http_err:
    562     args = (dict, 'default', 'http_error_default') + orig_args
--> 563     return self._call_chain(*args)

File ~/anaconda/envs/aiapy-dev/lib/python3.11/urllib/request.py:496, in OpenerDirector._call_chain(self, chain, kind, meth_name, *args)
    494 for handler in handlers:
    495     func = getattr(handler, meth_name)
--> 496     result = func(*args)
    497     if result is not None:
    498         return result

File ~/anaconda/envs/aiapy-dev/lib/python3.11/urllib/request.py:643, in HTTPDefaultErrorHandler.http_error_default(self, req, fp, code, msg, hdrs)
    642 def http_error_default(self, req, fp, code, msg, hdrs):
--> 643     raise HTTPError(req.full_url, code, msg, hdrs, fp)

HTTPError: HTTP Error 500: Failed to open URL: http://jsoc.stanford.edu/cgi-bin/ajax/jsoc_info?op=rs_list&ds=aia.lev1_euv_12s%5B2010.06.01_00%3A00%3A34_TAI-2021.06.01_00%3A00%3A37_TAI%402592000.0s%5D%5B335%5D&key=[v0.2](https://github.com/LM-SAL/aiapy/milestone/2)A%2AALL%2A%2A with 500 - Internal Server Error

In GitLab by @wtbarnes on Oct 31, 2023, 13:35

mentioned in commit a9acf79c8d20aed154a8596258f1ff4853e64279

In GitLab by @wtbarnes on Oct 31, 2023, 13:36

mentioned in commit 2f2010e2b8593883f53e915cbf68ae163c652a11

LM-SAL / aiapy

Correct degradation example is failing consistently #125