ValueError: '_index' is a reserved name for dataframe columns when writing multiple tables in Zarr

[Bisho2122]

I got an error from https://github.com/scverse/spatialdata/blob/2c6ae3f162d64d83780c2accaa73fa0f498eb4aa/src/spatialdata/_io/io_table.py#L24

when trying to write sdata to zarr with multiple-tables

@LucaMarconato @melonora any idea why ?

SpatialData version is

sd.__version__
'0.0.16.dev50+g421c315'

{
    "name": "ValueError",
    "message": "'_index' is a reserved name for dataframe columns.",
    "stack": "---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[46], line 1
----> 1 sdata.write('../Data/SpatialData_obj_Mar_26_2024',overwrite=True)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/spatialdata/_core/spatialdata.py:1065, in SpatialData.write(self, file_path, storage_options, overwrite, consolidate_metadata)
   1063 except Exception as e:  # noqa: B902
   1064     self._path = None
-> 1065     raise e
   1067 if consolidate_metadata:
   1068     # consolidate metadata to more easily support remote reading
   1069     # bug in zarr, 'zmetadata' is written instead of '.zmetadata'
   1070     # see discussion https://github.com/zarr-developers/zarr-python/issues/1121
   1071     zarr.consolidate_metadata(store, metadata_key=\".zmetadata\")

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/spatialdata/_core/spatialdata.py:1061, in SpatialData.write(self, file_path, storage_options, overwrite, consolidate_metadata)
   1059         elem_group = root.create_group(name=\"tables\")
   1060         for key in self.tables:
-> 1061             write_table(table=self.tables[key], group=elem_group, name=key)
   1063 except Exception as e:  # noqa: B902
   1064     self._path = None

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/spatialdata/_io/io_table.py:24, in write_table(table, group, name, group_type, fmt)
     22 else:
     23     region, region_key, instance_key = (None, None, None)
---> 24 write_adata(group, name, table)  # creates group[name]
     25 tables_group = group[name]
     26 tables_group.attrs[\"spatialdata-encoding-type\"] = group_type

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/registry.py:359, in write_elem(store, k, elem, dataset_kwargs)
    335 def write_elem(
    336     store: GroupStorageType,
    337     k: str,
   (...)
    340     dataset_kwargs: Mapping[str, Any] = MappingProxyType({}),
    341 ) -> None:
    342     \"\"\"
    343     Write an element to a storage group using anndata encoding.
    344 
   (...)
    357         E.g. for zarr this would be `chunks`, `compressor`.
    358     \"\"\"
--> 359     Writer(_REGISTRY).write_elem(store, k, elem, dataset_kwargs=dataset_kwargs)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/utils.py:243, in report_write_key_on_error.<locals>.func_wrapper(*args, **kwargs)
    241     raise ValueError(\"No element found in args.\")
    242 try:
--> 243     return func(*args, **kwargs)
    244 except Exception as e:
    245     path = _get_display_path(store)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/registry.py:309, in Writer.write_elem(self, store, k, elem, dataset_kwargs, modifiers)
    303 write_func = partial(
    304     self.find_writer(dest_type, elem, modifiers),
    305     _writer=self,
    306 )
    308 if self.callback is None:
--> 309     return write_func(store, k, elem, dataset_kwargs=dataset_kwargs)
    310 return self.callback(
    311     write_func,
    312     store,
   (...)
    316     iospec=self.registry.get_spec(elem),
    317 )

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/registry.py:57, in write_spec.<locals>.decorator.<locals>.wrapper(g, k, *args, **kwargs)
     55 @wraps(func)
     56 def wrapper(g: GroupStorageType, k: str, *args, **kwargs):
---> 57     result = func(g, k, *args, **kwargs)
     58     g[k].attrs.setdefault(\"encoding-type\", spec.encoding_type)
     59     g[k].attrs.setdefault(\"encoding-version\", spec.encoding_version)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/methods.py:259, in write_anndata(f, k, adata, _writer, dataset_kwargs)
    257 _writer.write_elem(g, \"layers\", dict(adata.layers), dataset_kwargs=dataset_kwargs)
    258 _writer.write_elem(g, \"uns\", dict(adata.uns), dataset_kwargs=dataset_kwargs)
--> 259 _writer.write_elem(g, \"raw\", adata.raw, dataset_kwargs=dataset_kwargs)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/utils.py:243, in report_write_key_on_error.<locals>.func_wrapper(*args, **kwargs)
    241     raise ValueError(\"No element found in args.\")
    242 try:
--> 243     return func(*args, **kwargs)
    244 except Exception as e:
    245     path = _get_display_path(store)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/registry.py:309, in Writer.write_elem(self, store, k, elem, dataset_kwargs, modifiers)
    303 write_func = partial(
    304     self.find_writer(dest_type, elem, modifiers),
    305     _writer=self,
    306 )
    308 if self.callback is None:
--> 309     return write_func(store, k, elem, dataset_kwargs=dataset_kwargs)
    310 return self.callback(
    311     write_func,
    312     store,
   (...)
    316     iospec=self.registry.get_spec(elem),
    317 )

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/registry.py:57, in write_spec.<locals>.decorator.<locals>.wrapper(g, k, *args, **kwargs)
     55 @wraps(func)
     56 def wrapper(g: GroupStorageType, k: str, *args, **kwargs):
---> 57     result = func(g, k, *args, **kwargs)
     58     g[k].attrs.setdefault(\"encoding-type\", spec.encoding_type)
     59     g[k].attrs.setdefault(\"encoding-version\", spec.encoding_version)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/methods.py:292, in write_raw(f, k, raw, _writer, dataset_kwargs)
    290 g = f.require_group(k)
    291 _writer.write_elem(g, \"X\", raw.X, dataset_kwargs=dataset_kwargs)
--> 292 _writer.write_elem(g, \"var\", raw.var, dataset_kwargs=dataset_kwargs)
    293 _writer.write_elem(g, \"varm\", dict(raw.varm), dataset_kwargs=dataset_kwargs)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/utils.py:243, in report_write_key_on_error.<locals>.func_wrapper(*args, **kwargs)
    241     raise ValueError(\"No element found in args.\")
    242 try:
--> 243     return func(*args, **kwargs)
    244 except Exception as e:
    245     path = _get_display_path(store)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/registry.py:309, in Writer.write_elem(self, store, k, elem, dataset_kwargs, modifiers)
    303 write_func = partial(
    304     self.find_writer(dest_type, elem, modifiers),
    305     _writer=self,
    306 )
    308 if self.callback is None:
--> 309     return write_func(store, k, elem, dataset_kwargs=dataset_kwargs)
    310 return self.callback(
    311     write_func,
    312     store,
   (...)
    316     iospec=self.registry.get_spec(elem),
    317 )

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/registry.py:57, in write_spec.<locals>.decorator.<locals>.wrapper(g, k, *args, **kwargs)
     55 @wraps(func)
     56 def wrapper(g: GroupStorageType, k: str, *args, **kwargs):
---> 57     result = func(g, k, *args, **kwargs)
     58     g[k].attrs.setdefault(\"encoding-type\", spec.encoding_type)
     59     g[k].attrs.setdefault(\"encoding-version\", spec.encoding_version)

File /g/scb2/alexandr/bwadie/miniconda3/envs/spatialdata/lib/python3.10/site-packages/anndata/_io/specs/methods.py:664, in write_dataframe(f, key, df, _writer, dataset_kwargs)
    662 for reserved in (\"_index\",):
    663     if reserved in df.columns:
--> 664         raise ValueError(f\"{reserved!r} is a reserved name for dataframe columns.\")
    665 group = f.require_group(key)
    666 col_names = [check_key(c) for c in df.columns]

ValueError: '_index' is a reserved name for dataframe columns."
}

[LucaMarconato]

Hi @Bisho2122, can you please post a reproducible code example?

[Bisho2122]

Hi @LucaMarconato , while I was trying to make a reproducible code example, I discovered that the problem was in the anndata object itself and the error has been reported before here https://github.com/theislab/scvelo/issues/255#issuecomment-739995301.

And this line

adata.__dict__['_raw'].__dict__['_var'] = adata.__dict__['_raw'].__dict__['_var'].rename(columns={'_index': 'features'})

solved it. So all good, closing this issue now.