Unidata / siphon

Siphon - A collection of Python utilities for retrieving atmospheric and oceanic data from remote sources, focusing on being able to retrieve data from Unidata data technologies, such as the THREDDS data server.
https://unidata.github.io/siphon
BSD 3-Clause "New" or "Revised" License
216 stars 75 forks source link

Error parsing time data, genfromtxt related #246

Open scollis opened 6 years ago

scollis commented 6 years ago

Hey Developers. Could be a thredds issue.. Please close if not the right place. trying to find obs in Alaska for a project.. Specifically narrowed the variables required so I don't just get Anchorage..

Code:

metar_cat_url = 'http://thredds.ucar.edu/thredds/catalog/nws/metar/ncdecoded/catalog.xml?dataset=nws/metar/ncdecoded/Metar_Station_Data_fc.cdmr'

# parse the xml
metar_cat = TDSCatalog(metar_cat_url)

# what datasets are here? only one "dataset" in this catalog
dataset = list(metar_cat.datasets.values())[0]
ncss_url = dataset.access_urls["NetcdfSubset"]
ncss = NCSS(ncss_url)
#Bounding box for our query

#Utqiagvik @71.2741555,-156.8642725,12.47z
bb={'west':-170.0, 'east':-140.0,'north':75, 'south':60.0}

#create a new query for the server
query = ncss.query()

#Pass the bounding box through
query.lonlat_box(**bb)

#what measurements do we want?
query.variables('air_temperature', 'dew_point_temperature', 'inches_ALTIM',
                'wind_speed', 'wind_from_direction', 'precipitation_amount_24')

#                , 'cloud_area_fraction', 'weather',
#               'report_id', 'precipitation_amount_hourly', 'precipitation_amount_24')

#restrict to last 1200 hours.. that should be enough data
query.time_range(datetime(2018, 7, 1) , datetime(2018, 8, 7) )
query.accept('csv')

#run the query!
data = ncss.get_data(query)

Error:


ValueError                                Traceback (most recent call last)
<ipython-input-25-031e4482d4f0> in <module>()
     22 
     23 #run the query!
---> 24 data = ncss.get_data(query)

~/anaconda3/envs/pchange/lib/python3.6/site-packages/siphon/ncss.py in get_data(self, query)
    113         """
    114         resp = self.get_query(query)
--> 115         return response_handlers(resp, self.unit_handler)
    116 
    117     def get_data_raw(self, query):

~/anaconda3/envs/pchange/lib/python3.6/site-packages/siphon/ncss.py in __call__(self, resp, unit_handler)
    292         """Process the HTTP response using the appropriate handler."""
    293         mimetype = resp.headers['content-type'].split(';')[0]
--> 294         return self._reg.get(mimetype, self.default)(resp.content, unit_handler)
    295 
    296 

~/anaconda3/envs/pchange/lib/python3.6/site-packages/siphon/ncss.py in parse_csv_response(data, unit_handler)
    403 def parse_csv_response(data, unit_handler):
    404     """Handle CSV-formatted HTTP responses."""
--> 405     return squish([parse_csv_dataset(d, unit_handler) for d in data.split(b'\n\n')])
    406 
    407 

~/anaconda3/envs/pchange/lib/python3.6/site-packages/siphon/ncss.py in <listcomp>(.0)
    403 def parse_csv_response(data, unit_handler):
    404     """Handle CSV-formatted HTTP responses."""
--> 405     return squish([parse_csv_dataset(d, unit_handler) for d in data.split(b'\n\n')])
    406 
    407 

~/anaconda3/envs/pchange/lib/python3.6/site-packages/siphon/ncss.py in parse_csv_dataset(data, handle_units)
    431     names, units = parse_csv_header(fobj.readline().decode('utf-8'))
    432     arrs = np.genfromtxt(fobj, dtype=None, names=names, delimiter=',', unpack=True,
--> 433                          converters={'date': lambda s: parse_iso_date(s.decode('utf-8'))})
    434     d = {}
    435     for f in arrs.dtype.fields:

~/anaconda3/envs/pchange/lib/python3.6/site-packages/numpy/lib/npyio.py in genfromtxt(fname, dtype, comments, delimiter, skip_header, skip_footer, converters, missing_values, filling_values, usecols, names, excludelist, deletechars, replace_space, autostrip, case_sensitive, defaultfmt, unpack, usemask, loose, invalid_raise, max_rows, encoding)
   2029             # Raise an exception ?
   2030             if invalid_raise:
-> 2031                 raise ValueError(errmsg)
   2032             # Issue a warning ?
   2033             else:

ValueError: Some errors were detected !
    Line #222 (got 5 columns instead of 10)```

When more vars added no time parsing error..
scollis commented 6 years ago

Note: no error when using time instead of time_range

scollis commented 6 years ago

now when using

                                             datetime.utcnow() - timedelta(minutes=5))

I get

HTTPError                                 Traceback (most recent call last)
<ipython-input-61-1687871e6b78> in <module>()
     12 
     13 
---> 14 data = ncss.get_data(query)
     15 #Pass the bounding box through
     16 #query.lonlat_box(**bb)

~/anaconda3/envs/pchange/lib/python3.6/site-packages/siphon/ncss.py in get_data(self, query)
    112 
    113         """
--> 114         resp = self.get_query(query)
    115         return response_handlers(resp, self.unit_handler)
    116 

~/anaconda3/envs/pchange/lib/python3.6/site-packages/siphon/http_util.py in get_query(self, query)
    400         """
    401         url = self._base[:-1] if self._base[-1] == '/' else self._base
--> 402         return self.get(url, query)
    403 
    404     def url_path(self, path):

~/anaconda3/envs/pchange/lib/python3.6/site-packages/siphon/http_util.py in get(self, path, params)
    485                                      'Server Error ({1:d}: {2})'.format(resp.request.url,
    486                                                                         resp.status_code,
--> 487                                                                         text))
    488         return resp
    489 

HTTPError: Error accessing http://thredds.ucar.edu/thredds/ncss/nws/metar/ncdecoded/Metar_Station_Data_fc.cdmr?var=wind_speed&var=weather&var=dew_point_temperature&var=report_id&var=cloud_area_fraction&var=wind_from_direction&var=inches_ALTIM&var=air_temperature&time_start=2018-09-07T03%3A17%3A09.613948&time_end=2018-09-07T15%3A12%3A09.613958&west=-170.0&east=-140.0&south=60.0&north=75&accept=csv
Server Error (502: badgateway)

:)