error reading output of tutorial/ecephys in pynwb

[bendichter]

When I try to read it I get the following error:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    883         try:
--> 884             obj = cls(**kwargs)
    885             obj.container_source = builder.source

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:

~/dev/pynwb/src/pynwb/core.py in __init__(self, **kwargs)
    873             col_dict = {col.name: col for col in columns}
--> 874             self.columns = [col_dict[name] for name in self.colnames]
    875 

~/dev/pynwb/src/pynwb/core.py in <listcomp>(.0)
    873             col_dict = {col.name: col for col in columns}
--> 874             self.columns = [col_dict[name] for name in self.colnames]
    875 

KeyError: 'x         '

The above exception was the direct cause of the following exception:

Exception                                 Traceback (most recent call last)
<ipython-input-3-7cc58965b729> in <module>()
      4 #file = '/Users/bendichter/Desktop/Chang/data/TDTBackup/EC125_B22.nwb'
      5 with NWBHDF5IO(file,'r') as io:
----> 6     io.read()

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/backends/io.py in read(self, **kwargs)
     31     def read(self, **kwargs):
     32         f_builder = self.read_builder()
---> 33         container = self.__manager.construct(f_builder)
     34         return container
     35 

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    186         result = self.__containers.get(builder_id)
    187         if result is None:
--> 188             result = self.__type_map.construct(builder, self)
    189             parent_builder = self.__get_parent_dt_builder(builder)
    190             if parent_builder is not None:

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
   1293                              % str(container.__class__.__name__))  # noqa: F821
   1294         else:
-> 1295             return attr_map.construct(builder, build_manager)
   1296 
   1297     @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"},

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    862         cls = manager.get_cls(builder)
    863         # gather all subspecs
--> 864         subspecs = self.__get_subspec_values(builder, self.spec, manager)
    865         # get the constructor argument each specification corresponds to
    866         const_args = dict()

~/dev/pynwb/src/pynwb/form/build/map.py in __get_subspec_values(self, builder, spec, manager)
    848                             ret[subspec] = val
    849                     else:
--> 850                         result = self.__get_subspec_values(sub_builder, subspec, manager)
    851                         ret.update(result)
    852         else:

~/dev/pynwb/src/pynwb/form/build/map.py in __get_subspec_values(self, builder, spec, manager)
    839                     if self.__data_type_key in sub_builder.attributes or \
    840                        not (subspec.data_type_inc is None and subspec.data_type_def is None):
--> 841                         val = manager.construct(sub_builder)
    842                         if subspec.is_many():
    843                             if subspec in ret:

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    186         result = self.__containers.get(builder_id)
    187         if result is None:
--> 188             result = self.__type_map.construct(builder, self)
    189             parent_builder = self.__get_parent_dt_builder(builder)
    190             if parent_builder is not None:

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
   1293                              % str(container.__class__.__name__))  # noqa: F821
   1294         else:
-> 1295             return attr_map.construct(builder, build_manager)
   1296 
   1297     @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"},

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    862         cls = manager.get_cls(builder)
    863         # gather all subspecs
--> 864         subspecs = self.__get_subspec_values(builder, self.spec, manager)
    865         # get the constructor argument each specification corresponds to
    866         const_args = dict()

~/dev/pynwb/src/pynwb/form/build/map.py in __get_subspec_values(self, builder, spec, manager)
    839                     if self.__data_type_key in sub_builder.attributes or \
    840                        not (subspec.data_type_inc is None and subspec.data_type_def is None):
--> 841                         val = manager.construct(sub_builder)
    842                         if subspec.is_many():
    843                             if subspec in ret:

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    186         result = self.__containers.get(builder_id)
    187         if result is None:
--> 188             result = self.__type_map.construct(builder, self)
    189             parent_builder = self.__get_parent_dt_builder(builder)
    190             if parent_builder is not None:

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
   1293                              % str(container.__class__.__name__))  # noqa: F821
   1294         else:
-> 1295             return attr_map.construct(builder, build_manager)
   1296 
   1297     @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"},

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    862         cls = manager.get_cls(builder)
    863         # gather all subspecs
--> 864         subspecs = self.__get_subspec_values(builder, self.spec, manager)
    865         # get the constructor argument each specification corresponds to
    866         const_args = dict()

~/dev/pynwb/src/pynwb/form/build/map.py in __get_subspec_values(self, builder, spec, manager)
    818                 continue
    819             if isinstance(h5attr_val, (GroupBuilder, DatasetBuilder)):
--> 820                 ret[subspec] = manager.construct(h5attr_val)
    821             elif isinstance(h5attr_val, RegionBuilder):
    822                 raise ValueError("RegionReferences as attributes is not yet supported")

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    186         result = self.__containers.get(builder_id)
    187         if result is None:
--> 188             result = self.__type_map.construct(builder, self)
    189             parent_builder = self.__get_parent_dt_builder(builder)
    190             if parent_builder is not None:

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
   1293                              % str(container.__class__.__name__))  # noqa: F821
   1294         else:
-> 1295             return attr_map.construct(builder, build_manager)
   1296 
   1297     @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"},

~/dev/pynwb/src/pynwb/form/utils.py in func_call(*args, **kwargs)
    341                     msg = ', '.join(parse_err)
    342                     raise_from(TypeError(msg), None)
--> 343                 return func(self, **parsed['args'])
    344         else:
    345             def func_call(*args, **kwargs):

~/dev/pynwb/src/pynwb/form/build/map.py in construct(self, **kwargs)
    886         except Exception as ex:
    887             msg = 'Could not construct %s object' % (cls.__name__,)
--> 888             raise_from(Exception(msg), ex)
    889         return obj
    890 

~/anaconda3/lib/python3.6/site-packages/six.py in raise_from(value, from_value)

Exception: Could not construct DynamicTable object

When I look at the attributes of electrodes in HDFView, I see

which is pretty similar to what I see from files generated using pynwb that read without error:

So I'm not sure what is causing the error

[bendichter]

These inspections check out. I'm pretty stumped.

nwbtable.colnames

ans =

  1×8 cell array

  Columns 1 through 4

    {'x'}    {'y'}    {'z'}    {'imp'}

  Columns 5 through 7

    {'location'}    {'filtering'}    {'group'}

  Column 8

    {'group_name'}

>>electrode_table.tablecolumn.keys

ans =

  1×8 cell array

  Columns 1 through 3

    {'filtering'}    {'group'}    {'group_name'}

  Columns 4 through 7

    {'imp'}    {'location'}    {'x'}    {'y'}

  Column 8

    {'z'}

[lawrence-mbf]

This might be related to #43, does it match the schema used in pynwb?

[bendichter]

It looks like this might be an issue on the pynwb side

[lawrence-mbf]

Strip self.colnames before populating self.columns and see if that works.

Instead of using variable length strings, matnwb opts for strictly sized strings. However, an array of strings requires padding so that they can be stored as an array.

[lawrence-mbf]

953555ee14c676863391c5033a3503aa32dc0259 better yet. Try running it with this.

[bendichter]

works!