Error when reading a file that contains DecompositionSeries without 'source_timeseries'

[luiztauffer]

1) Bug When reading from a NWB file that contains a DecompositionSeries without 'source_timeseries', an error is raised. It happens with other classes as well. Error present in both 1.1.0 and 1.1.0.post0.dev2 Here's a snippet:

import os
import numpy as np
from datetime import datetime
from dateutil.tz import tzlocal
import pynwb
from pynwb import NWBFile, NWBHDF5IO, get_manager, ProcessingModule
from pynwb.core import DynamicTable, DynamicTableRegion, VectorData
from pynwb.misc import DecompositionSeries

manager = get_manager()

# Creates file 1
nwb = NWBFile(session_description='session', identifier='1', session_start_time=datetime.now(tzlocal()))

# data: (ndarray) dims: num_times * num_channels * num_bands
Xp = np.zeros((1000,10,3))

# Spectral band power
# bands: (DynamicTable) frequency bands that signal was decomposed into
band_param_0V = VectorData(name='filter_param_0',
                  description='frequencies for bandpass filters',
                  data=np.array([1.,2.,3.]))
band_param_1V = VectorData(name='filter_param_1',
                  description='frequencies for bandpass filters',
                  data=np.array([1.,2.,3.]))
bandsTable = DynamicTable(name='bands',
                          description='Series of filters used for Hilbert transform.',
                          columns=[band_param_0V,band_param_1V],
                          colnames=['filter_param_0','filter_param_1'])
decs = DecompositionSeries(name='DecompositionSeries',
                           data=Xp,
                           description='Analytic amplitude estimated with Hilbert transform.',
                           metric='amplitude',
                           unit='V',
                           bands=bandsTable,
                           #source_timeseries=lfp
                           rate=1.)

ecephys_module = ProcessingModule(name='ecephys',
                                  description='Extracellular electrophysiology data.')
nwb.add_processing_module(ecephys_module)
ecephys_module.add_data_interface(decs)

with NWBHDF5IO('file_1.nwb', mode='w', manager=manager) as io:
    io.write(nwb)

# Open file 1
with NWBHDF5IO('file_1.nwb', 'r', manager=manager) as io:
    nwb = io.read()

Notice that the files saves alright. The error happens on reading:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-6-3c4f12e3d123> in <module>
     48 # Open file 1
     49 with NWBHDF5IO('file_1.nwb', 'r', manager=manager) as io:
---> 50     nwb = io.read()
     51 

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\backends\hdf5\h5tools.py in read(self, **kwargs)
    244                                        % (self.__path, self.__mode))
    245         try:
--> 246             return call_docval_func(super(HDF5IO, self).read, kwargs)
    247         except UnsupportedOperation as e:
    248             if str(e) == 'Cannot build data. There are no values.':

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in call_docval_func(func, kwargs)
    279 def call_docval_func(func, kwargs):
    280     fargs, fkwargs = fmt_docval_args(func, kwargs)
--> 281     return func(*fargs, **fkwargs)
    282 
    283 

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\backends\io.py in read(self, **kwargs)
     33             # TODO also check that the keys are appropriate. print a better error message
     34             raise UnsupportedOperation('Cannot build data. There are no values.')
---> 35         container = self.__manager.construct(f_builder)
     36         return container
     37 

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
    198         result = self.__containers.get(builder_id)
    199         if result is None:
--> 200             result = self.__type_map.construct(builder, self)
    201             parent_builder = self.__get_parent_dt_builder(builder)
    202             if parent_builder is not None:

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
   1670             raise ValueError('No ObjectMapper found for builder of type %s' % dt)
   1671         else:
-> 1672             return attr_map.construct(builder, build_manager)
   1673 
   1674     @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"},

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
   1181         cls = manager.get_cls(builder)
   1182         # gather all subspecs
-> 1183         subspecs = self.__get_subspec_values(builder, self.spec, manager)
   1184         # get the constructor argument that each specification corresponds to
   1185         const_args = dict()

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __get_subspec_values(self, builder, spec, manager)
   1125                         ret[subspec] = self.__flatten(sub_builder, subspec, manager)
   1126             # now process groups and datasets
-> 1127             self.__get_sub_builders(groups, spec.groups, manager, ret)
   1128             self.__get_sub_builders(datasets, spec.datasets, manager, ret)
   1129         elif isinstance(spec, DatasetSpec):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __get_sub_builders(self, sub_builders, subspecs, manager, ret)
   1163                 if dt is None:
   1164                     # recurse
-> 1165                     ret.update(self.__get_subspec_values(sub_builder, subspec, manager))
   1166                 else:
   1167                     ret[subspec] = manager.construct(sub_builder)

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __get_subspec_values(self, builder, spec, manager)
   1125                         ret[subspec] = self.__flatten(sub_builder, subspec, manager)
   1126             # now process groups and datasets
-> 1127             self.__get_sub_builders(groups, spec.groups, manager, ret)
   1128             self.__get_sub_builders(datasets, spec.datasets, manager, ret)
   1129         elif isinstance(spec, DatasetSpec):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __get_sub_builders(self, sub_builders, subspecs, manager, ret)
   1155                 sub_builder = builder_dt.get(dt)
   1156                 if sub_builder is not None:
-> 1157                     sub_builder = self.__flatten(sub_builder, subspec, manager)
   1158                     ret[subspec] = sub_builder
   1159             else:

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __flatten(self, sub_builder, subspec, manager)
   1168 
   1169     def __flatten(self, sub_builder, subspec, manager):
-> 1170         tmp = [manager.construct(b) for b in sub_builder]
   1171         if len(tmp) == 1 and not subspec.is_many():
   1172             tmp = tmp[0]

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in <listcomp>(.0)
   1168 
   1169     def __flatten(self, sub_builder, subspec, manager):
-> 1170         tmp = [manager.construct(b) for b in sub_builder]
   1171         if len(tmp) == 1 and not subspec.is_many():
   1172             tmp = tmp[0]

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
    198         result = self.__containers.get(builder_id)
    199         if result is None:
--> 200             result = self.__type_map.construct(builder, self)
    201             parent_builder = self.__get_parent_dt_builder(builder)
    202             if parent_builder is not None:

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
   1670             raise ValueError('No ObjectMapper found for builder of type %s' % dt)
   1671         else:
-> 1672             return attr_map.construct(builder, build_manager)
   1673 
   1674     @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"},

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
   1181         cls = manager.get_cls(builder)
   1182         # gather all subspecs
-> 1183         subspecs = self.__get_subspec_values(builder, self.spec, manager)
   1184         # get the constructor argument that each specification corresponds to
   1185         const_args = dict()

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __get_subspec_values(self, builder, spec, manager)
   1125                         ret[subspec] = self.__flatten(sub_builder, subspec, manager)
   1126             # now process groups and datasets
-> 1127             self.__get_sub_builders(groups, spec.groups, manager, ret)
   1128             self.__get_sub_builders(datasets, spec.datasets, manager, ret)
   1129         elif isinstance(spec, DatasetSpec):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __get_sub_builders(self, sub_builders, subspecs, manager, ret)
   1155                 sub_builder = builder_dt.get(dt)
   1156                 if sub_builder is not None:
-> 1157                     sub_builder = self.__flatten(sub_builder, subspec, manager)
   1158                     ret[subspec] = sub_builder
   1159             else:

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __flatten(self, sub_builder, subspec, manager)
   1168 
   1169     def __flatten(self, sub_builder, subspec, manager):
-> 1170         tmp = [manager.construct(b) for b in sub_builder]
   1171         if len(tmp) == 1 and not subspec.is_many():
   1172             tmp = tmp[0]

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in <listcomp>(.0)
   1168 
   1169     def __flatten(self, sub_builder, subspec, manager):
-> 1170         tmp = [manager.construct(b) for b in sub_builder]
   1171         if len(tmp) == 1 and not subspec.is_many():
   1172             tmp = tmp[0]

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
    198         result = self.__containers.get(builder_id)
    199         if result is None:
--> 200             result = self.__type_map.construct(builder, self)
    201             parent_builder = self.__get_parent_dt_builder(builder)
    202             if parent_builder is not None:

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
   1670             raise ValueError('No ObjectMapper found for builder of type %s' % dt)
   1671         else:
-> 1672             return attr_map.construct(builder, build_manager)
   1673 
   1674     @docval({"name": "container", "type": Container, "doc": "the container to convert to a Builder"},

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\utils.py in func_call(*args, **kwargs)
    386                         raise_from(ExceptionType(msg), None)
    387 
--> 388                 return func(self, **parsed['args'])
    389         else:
    390             def func_call(*args, **kwargs):

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in construct(self, **kwargs)
   1181         cls = manager.get_cls(builder)
   1182         # gather all subspecs
-> 1183         subspecs = self.__get_subspec_values(builder, self.spec, manager)
   1184         # get the constructor argument that each specification corresponds to
   1185         const_args = dict()

~\Anaconda3\envs\ecog_vis\lib\site-packages\hdmf\build\map.py in __get_subspec_values(self, builder, spec, manager)
   1119             for subspec in spec.links:
   1120                 if subspec.name is not None:
-> 1121                     ret[subspec] = manager.construct(links[subspec.name].builder)
   1122                 else:
   1123                     sub_builder = link_dt.get(subspec.target_type)

KeyError: 'source_timeseries'

Python Executable: Conda
Python Version: Python 3.7
Operating System: Windows
HDMF Version: 1.1.0

Checklist

• [x] Have you ensured the feature or change was not already reported ?
• [x] Have you included a brief and descriptive title?
• [x] Have you included a clear description of the problem you are trying to solve?
• [x] Have you included a minimal code snippet that reproduces the issue you are encountering?
• [x] Have you checked our Contributing document?

[luiztauffer]

Update: this happens whenever the file contains a container (not only DecompositionSeries) with empty 'source_timeseries' attribute.