jbusecke
commented
2 years ago Reporting back from the OSM tutorial, where @selipot, Ian Carroll, and I tried our luck with this.
We unfortunately failed to even open datasets from within the binder environment. Using the example url https://datahub.geos.tamu.edu:8880/thredds/fileServer/B.E.13.B1850C5.ne120_t12.sehires38.003.sunway_02/ocn/SSH/B.E.13.B1850C5.ne120_t12.sehires38.003.sunway_02.pop.h.SSH.032001-032912.nc, which I can download to my local computer by clicking it, but was unable to load into xarray:
path = 'https://datahub.geos.tamu.edu:8880/thredds/fileServer/B.E.13.B1850C5.ne120_t12.sehires38.003.sunway_02/ocn/SSH/B.E.13.B1850C5.ne120_t12.sehires38.003.sunway_02.pop.h.SSH.032001-032912.nc'
xr.open_dataset(path)gives:
Note:Caching=1
Error:curl error: SSL peer certificate or SSH remote key was not OK
curl error details:
Warning:oc_open: Could not read url
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
198 try:
--> 199 file = self._cache[self._key]
200 except KeyError:
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/lru_cache.py in __getitem__(self, key)
52 with self._lock:
---> 53 value = self._cache[key]
54 self._cache.move_to_end(key)
KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('https://datahub.geos.tamu.edu:8880/thredds/fileServer/B.E.13.B1850C5.ne120_t12.sehires38.003.sunway_02/ocn/SSH/B.E.13.B1850C5.ne120_t12.sehires38.003.sunway_02.pop.h.SSH.032001-032912.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False))]
During handling of the above exception, another exception occurred:
OSError Traceback (most recent call last)
/var/folders/_1/1k9jtjl51z333f21s7yht0340000gn/T/ipykernel_1917/1179327591.py in <module>
----> 1 xr.open_dataset(path)
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/api.py in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, backend_kwargs, *args, **kwargs)
493
494 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 495 backend_ds = backend.open_dataset(
496 filename_or_obj,
497 drop_variables=drop_variables,
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, format, clobber, diskless, persist, lock, autoclose)
548
549 filename_or_obj = _normalize_path(filename_or_obj)
--> 550 store = NetCDF4DataStore.open(
551 filename_or_obj,
552 mode=mode,
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in open(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)
377 netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
378 )
--> 379 return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
380
381 def _acquire(self, needs_lock=True):
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in __init__(self, manager, group, mode, lock, autoclose)
325 self._group = group
326 self._mode = mode
--> 327 self.format = self.ds.data_model
328 self._filename = self.ds.filepath()
329 self.is_remote = is_remote_uri(self._filename)
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in ds(self)
386 @property
387 def ds(self):
--> 388 return self._acquire()
389
390 def open_store_variable(self, name, var):
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in _acquire(self, needs_lock)
380
381 def _acquire(self, needs_lock=True):
--> 382 with self._manager.acquire_context(needs_lock) as root:
383 ds = _nc4_require_group(root, self._group, self._mode)
384 return ds
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/contextlib.py in __enter__(self)
117 del self.args, self.kwds, self.func
118 try:
--> 119 return next(self.gen)
120 except StopIteration:
121 raise RuntimeError("generator didn't yield") from None
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/file_manager.py in acquire_context(self, needs_lock)
185 def acquire_context(self, needs_lock=True):
186 """Context manager for acquiring a file."""
--> 187 file, cached = self._acquire_with_cache_info(needs_lock)
188 try:
189 yield file
~/miniconda/envs/pangeo_forge_cm26/lib/python3.9/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
203 kwargs = kwargs.copy()
204 kwargs["mode"] = self._mode
--> 205 file = self._opener(*self._args, **kwargs)
206 if self._mode == "w":
207 # ensure file doesn't get overriden when opened again
src/netCDF4/_netCDF4.pyx in netCDF4._netCDF4.Dataset.__init__()
src/netCDF4/_netCDF4.pyx in netCDF4._netCDF4._ensure_nc_success()
OSError: [Errno -68] NetCDF: I/O failure: b'https://datahub.geos.tamu.edu:8880/thredds/fileServer/B.E.13.B1850C5.ne120_t12.sehires38.003.sunway_02/ocn/SSH/B.E.13.B1850C5.ne120_t12.sehires38.003.sunway_02.pop.h.SSH.032001-032912.nc'We tried to download an example file with curl, which worked on my local machine from the terminal, which only succeeded from my local machine.
We did not get any further, but I would love to understand what went wrong here. I guess it is not technically necessary to look at the data first, but I do not know much about it, and so exploring it first seemed like a good idea.
rabernat
abishekg7
Source Dataset
Transformation / Alignment / Merging
The data should merge cleanly.
Output Dataset
Zarr with all time slice files unified into a single timeseries. Possibly merging of different variables as well.
cc @paigem, @abishekg7