RCM reader not handling HH files

[vincelhx]

test file is :

/home/datawork-cersat-public/provider/asc-csa/satellite/l1/rcm/rcm2/SCSDA/GRD/2020/051/RCM2_OK1051101_PK1052225_1_SCSDA_20200220_090253_HH_GRD

with fct_meta = xsar.RcmMeta

Error =>

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[26], line 22
     15     raise FileNotFoundError('config_path do not exists, got %s '  % config_path)
     20 # 2 - Add raster and load dataset at 1km resoltuion
---> 22 meta = fct_meta(filename)

File ~/libs_last/xsar_forkVinc/xsar/src/xsar/utils.py:94, in timing.<locals>.wrapper(*args, **kwargs)
     92     startrss = process.memory_info().rss
     93 starttime = time.time()
---> 94 result = f(*args, **kwargs)
     95 endtime = time.time()
     96 if mem_monitor:

File ~/libs_last/xsar_forkVinc/xsar/src/xsar/rcm_meta.py:34, in RcmMeta.__init__(self, name)
     32     self.dt = api.open_rcm(name.split(':')[1])
     33 else:
---> 34     self.dt = api.open_rcm(name)
     35 if not name.startswith('RCM_DS:'):
     36     name = 'RCM_DS:%s:' % name

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/api.py:97, in open_rcm(url, backend_kwargs, manifest_ignores, **dataset_kwargs)
     91 if missing_files:
     92     raise ExceptionGroup(
     93         "not all files declared in the manifest are available",
     94         [ValueError(f"{p} does not exist") for p in missing_files],
     95     )
---> 97 tree = read_product(mapper, "metadata/product.xml")
     99 calibration_root = "metadata/calibration"
    100 lookup_table_structure = {
    101     "/incidenceAngles": {
    102         "path": "/imageReferenceAttributes",
   (...)
    135     },
    136 }

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/product/reader.py:274, in read_product(mapper, product_path)
     35 decoded = read_xml(mapper, product_path)
     37 layout = {
     38     "/": {
     39         "path": "/",
   (...)
    271     },
    272 }
--> 274 converted = valmap(
    275     lambda x: execute(**x)(decoded),
    276     layout,
    277 )
    278 return datatree.DataTree.from_dict(converted)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/dicttoolz.pyx:178, in cytoolz.dicttoolz.valmap()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/dicttoolz.pyx:203, in cytoolz.dicttoolz.valmap()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/product/reader.py:275, in read_product.<locals>.<lambda>(x)
     35 decoded = read_xml(mapper, product_path)
     37 layout = {
     38     "/": {
     39         "path": "/",
   (...)
    271     },
    272 }
    274 converted = valmap(
--> 275     lambda x: execute(**x)(decoded),
    276     layout,
    277 )
    278 return datatree.DataTree.from_dict(converted)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/functoolz.pyx:253, in cytoolz.functoolz.curry.__call__()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/functoolz.pyx:249, in cytoolz.functoolz.curry.__call__()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/product/reader.py:31, in execute(mapping, f, path)
     27 @curry
     28 def execute(mapping, f, path):
     29     subset = query(path, mapping)
---> 31     return compose_left(f, attach_path(path=path))(subset)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/functoolz.pyx:502, in cytoolz.functoolz.Compose.__call__()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/functoolz.pyx:504, in cytoolz.functoolz.Compose.__call__()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/functoolz.pyx:721, in genexpr()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/functoolz.pyx:721, in genexpr()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/functoolz.pyx:504, in cytoolz.functoolz.Compose.__call__()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/product/transformers.py:133, in extract_dataset(obj, dims, default_dims)
    129 variables_ = keymap(lambda k: k.lstrip("@"), variables)
    131 filtered_variables = valfilter(lambda x: not is_nested_dataset(x), variables_)
--> 133 data_vars = itemmap(
    134     lambda item: (
    135         item[0],
    136         extract_entry(*item, dims=dims, default_dims=default_dims),
    137     ),
    138     filtered_variables,
    139 )
    140 return xr.Dataset(data_vars=data_vars, attrs=attrs)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/dicttoolz.pyx:236, in cytoolz.dicttoolz.itemmap()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/dicttoolz.pyx:261, in cytoolz.dicttoolz.itemmap()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/product/transformers.py:136, in extract_dataset.<locals>.<lambda>(item)
    129 variables_ = keymap(lambda k: k.lstrip("@"), variables)
    131 filtered_variables = valfilter(lambda x: not is_nested_dataset(x), variables_)
    133 data_vars = itemmap(
    134     lambda item: (
    135         item[0],
--> 136         extract_entry(*item, dims=dims, default_dims=default_dims),
    137     ),
    138     filtered_variables,
    139 )
    140 return xr.Dataset(data_vars=data_vars, attrs=attrs)

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/product/transformers.py:116, in extract_entry(name, obj, dims, default_dims)
    114     return extract_variable(obj, dims=dims)
    115 elif is_nested_array(obj):
--> 116     return extract_nested_array(obj, dims=dims).pipe(rename, name)
    117 else:
    118     raise ValueError(f"unknown datastructure:\n{obj}")

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/safe_rcm/product/transformers.py:209, in extract_nested_array(obj, dims)
    200     dims = ["$"]
    202 coords = itemmap(
    203     lambda it: (it[0], to_variable_tuple(*it, dims=dims)),
    204     indexes,
    205 )
    207 arr = xr.DataArray(
    208     data=preprocessed_data["$"],
--> 209     attrs=valmap(first, attrs_),
    210     dims=dims,
    211     coords=coords,
    212 )
    213 if originally_stacked:
    214     return arr

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/dicttoolz.pyx:178, in cytoolz.dicttoolz.valmap()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/dicttoolz.pyx:203, in cytoolz.dicttoolz.valmap()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/itertoolz.pyx:549, in cytoolz.itertoolz.first()

File /home1/datawork/vlheureu/conda-env/xsar_N/lib/python3.10/site-packages/cytoolz/itertoolz.pyx:556, in cytoolz.itertoolz.first()

TypeError: iteration over a 0-d array

[sparshgarg23]

I am facing the same issue.Wanted to know if you were able to resolve it?

[agrouaze]

I will try to have a look at this issue next week.

[agrouaze]

issue opened in the reader repo : https://github.com/umr-lops/xarray-safe-rcm/issues/86