search

COSIMA / cosima-cookbook

Framework for indexing and querying ocean-sea ice model output.
https://cosima-recipes.readthedocs.io/en/latest/
Apache License 2.0
58 stars 25 forks source link

can't access jra55 data #275

Closed fzhang0 closed 3 years ago

fzhang0 commented 3 years ago

When I try 

cc.querying.getvar(expt='MRI-JRA55-do-1-4-0', variable='uas', 
                          session=session, frequency='3 hourly',
                          start_time='1957-12-31 22:30:00', 
                          end_time='2019-01-05 22:30:00')

I get 

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
    198             try:
--> 199                 file = self._cache[self._key]
    200             except KeyError:

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/lru_cache.py in __getitem__(self, key)
     52         with self._lock:
---> 53             value = self._cache[key]
     54             self._cache.move_to_end(key)

KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('/g/data/ik11/inputs/JRA-55/MRI-JRA55-do/MRI-JRA55-do-1-4-0/atmos/uas/uas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_196501010000-196512312100.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False))]

During handling of the above exception, another exception occurred:

FileNotFoundError                         Traceback (most recent call last)
/jobfs/28073817.gadi-pbs/ipykernel_1224218/2750106142.py in <module>
----> 1 cc.querying.getvar(expt='MRI-JRA55-do-1-4-0', variable='uas', 
      2                           session=session, frequency='3 hourly',
      3                           start_time='1957-12-31 22:30:00',
      4                           end_time='2019-01-05 22:30:00')

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/cosima_cookbook/querying.py in getvar(expt, variable, session, ncfile, start_time, end_time, n, frequency, attrs, attrs_unique, **kwargs)
    230         return d[variables]
    231 
--> 232     ds = xr.open_mfdataset(
    233         (str(f.NCFile.ncfile_path) for f in ncfiles),
    234         parallel=True,

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/api.py in open_mfdataset(paths, chunks, concat_dim, compat, preprocess, engine, data_vars, coords, combine, parallel, join, attrs_file, combine_attrs, **kwargs)
    919         # calling compute here will return the datasets/file_objs lists,
    920         # the underlying datasets will still be stored as dask arrays
--> 921         datasets, closers = dask.compute(datasets, closers)
    922 
    923     # Combine all datasets, closing them in case of a ValueError

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/dask/base.py in compute(*args, **kwargs)
    566         postcomputes.append(x.__dask_postcompute__())
    567 
--> 568     results = schedule(dsk, keys, **kwargs)
    569     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    570 

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, pool, **kwargs)
     77             pool = MultiprocessingPoolExecutor(pool)
     78 
---> 79     results = get_async(
     80         pool.submit,
     81         pool._max_workers,

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/dask/local.py in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)
    515                             _execute_task(task, data)  # Re-execute locally
    516                         else:
--> 517                             raise_exception(exc, tb)
    518                     res, worker_id = loads(res_info)
    519                     state["cache"][key] = res

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/dask/local.py in reraise(exc, tb)
    323     if exc.__traceback__ is not tb:
    324         raise exc.with_traceback(tb)
--> 325     raise exc
    326 
    327 

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    221     try:
    222         task, data = loads(task_info)
--> 223         result = _execute_task(task, data)
    224         id = get_id()
    225         result = dumps((result, id))

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
    119         # temporaries by their reference count and can execute certain
    120         # operations in-place.
--> 121         return func(*(_execute_task(a, cache) for a in args))
    122     elif not ishashable(arg):
    123         return arg

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/dask/utils.py in apply(func, args, kwargs)
     33 def apply(func, args, kwargs=None):
     34     if kwargs:
---> 35         return func(*args, **kwargs)
     36     else:
     37         return func(*args)

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/api.py in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, backend_kwargs, *args, **kwargs)
    495 
    496     overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 497     backend_ds = backend.open_dataset(
    498         filename_or_obj,
    499         drop_variables=drop_variables,

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, group, mode, format, clobber, diskless, persist, lock, autoclose)
    549 
    550         filename_or_obj = _normalize_path(filename_or_obj)
--> 551         store = NetCDF4DataStore.open(
    552             filename_or_obj,
    553             mode=mode,

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in open(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)
    378             netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
    379         )
--> 380         return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
    381 
    382     def _acquire(self, needs_lock=True):

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in __init__(self, manager, group, mode, lock, autoclose)
    326         self._group = group
    327         self._mode = mode
--> 328         self.format = self.ds.data_model
    329         self._filename = self.ds.filepath()
    330         self.is_remote = is_remote_uri(self._filename)

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in ds(self)
    387     @property
    388     def ds(self):
--> 389         return self._acquire()
    390 
    391     def open_store_variable(self, name, var):

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/netCDF4_.py in _acquire(self, needs_lock)
    381 
    382     def _acquire(self, needs_lock=True):
--> 383         with self._manager.acquire_context(needs_lock) as root:
    384             ds = _nc4_require_group(root, self._group, self._mode)
    385         return ds

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/contextlib.py in __enter__(self)
    117         del self.args, self.kwds, self.func
    118         try:
--> 119             return next(self.gen)
    120         except StopIteration:
    121             raise RuntimeError("generator didn't yield") from None

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/file_manager.py in acquire_context(self, needs_lock)
    185     def acquire_context(self, needs_lock=True):
    186         """Context manager for acquiring a file."""
--> 187         file, cached = self._acquire_with_cache_info(needs_lock)
    188         try:
    189             yield file

/g/data/hh5/public/apps/miniconda3/envs/analysis3-21.07/lib/python3.9/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
    203                     kwargs = kwargs.copy()
    204                     kwargs["mode"] = self._mode
--> 205                 file = self._opener(*self._args, **kwargs)
    206                 if self._mode == "w":
    207                     # ensure file doesn't get overriden when opened again

src/netCDF4/_netCDF4.pyx in netCDF4._netCDF4.Dataset.__init__()

src/netCDF4/_netCDF4.pyx in netCDF4._netCDF4._ensure_nc_success()

FileNotFoundError: [Errno 2] No such file or directory: b'/g/data/ik11/inputs/JRA-55/MRI-JRA55-do/MRI-JRA55-do-1-4-0/atmos/uas/uas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_196501010000-196512312100.nc'

I don't have this problem when I access other data in ik11.

aekiss commented 3 years ago

The weird thing is I don't have this issue. We are both using analysis3-21.07. Fan is a member of ik11 and can see the file when he does

ls /g/data/ik11/inputs/JRA-55/MRI-JRA55-do/MRI-JRA55-do-1-4-0/atmos/uas/uas_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_196501010000-196512312100.nc
aekiss commented 3 years ago

(I deleted my last post, which was meant for a different issue)

aidanheerdegen commented 3 years ago

The data in there are symbolic links to the input4MIPs data in /g/data/qv56:

$ ls -l  /g/data/ik11/inputs/JRA-55/MRI-JRA55-do/MRI-JRA55-do-1-4-0/land/friver/*2019*    
lrwxrwxrwx 1 aph502 ik11 178 Aug 21  2020 /g/data/ik11/inputs/JRA-55/MRI-JRA55-do/MRI-JRA55-do-1-4-0/land/friver/friver_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_20190101-20190105.nc -> /g/data/qv56/replicas/input4MIPs/CMIP6/OMIP/MRI/MRI-JRA55-do-1-4-0/land/day/friver/gr/v20190429/friver_input4MIPs_atmosphericState_OMIP_MRI-JRA55-do-1-4-0_gr_20190101-20190105.nc

You need to go to my.nci.org.au and request membership to qv56

I have added a note that qv56 membership is required to access this data in the Notes in metadata.yaml

aekiss commented 3 years ago

d'oh I should have noticed that - thanks @aidanheerdegen

fzhang0 commented 3 years ago

The issue has been solved. Thanks @aidanheerdegen