Cannot iteratively write rois for PlaneSegmentation

[luiztauffer]

pynwb==1.2.1 and hdmf==1.6.0

For a PlaneSegmentation object, when trying to add_roi() using a DataChunkIterator it raises the error: TypeError: 'DataChunk' object does not support indexing

Here's a code for reproducing it:

from pynwb import NWBFile, NWBHDF5IO
from pynwb.ophys import TwoPhotonSeries, OpticalChannel, ImageSegmentation
from pynwb.device import Device
from hdmf.data_utils import DataChunkIterator
from datetime import datetime
from dateutil.tz import tzlocal
import numpy as np

def iter_dataset(datasetview_obj):
    n = datasetview_obj.shape[0]
    for i in range(n):
        curr_data = datasetview_obj[i]
        yield curr_data
    return

# Create nwbfile
nwbfile = NWBFile('a', 'id', datetime.now(tzlocal()))

# Device
device = Device('imaging_device_1')
nwbfile.add_device(device)

# Optical Channels
op_names = ['my_optchan1', 'my_optchan2']
optical_channel1 = OpticalChannel('my_optchan1', 'description1', 500.)
optical_channel2 = OpticalChannel('my_optchan2', 'description2', 600.)

# Imaging Plane
imaging_plane = nwbfile.create_imaging_plane(
    name = 'my_imgpln',
    optical_channel = [optical_channel1, optical_channel2],
    description='a very interesting part of the brain',
    device=device,
    excitation_lambda=600.,
    imaging_rate=300.,
    indicator='GFP',
    location='my favorite brain location',
)

# TwoPhotonSeries
n_samples, n_x, n_y = 10, 5, 5
data_raw = np.random.rand(n_samples, n_x, n_y)
data_iterator = DataChunkIterator(
    data=iter_dataset(data_raw),
)
image_series = TwoPhotonSeries(
    name='test_iS',
    data=data_iterator,
    imaging_plane=imaging_plane,
    rate=1.0
)
nwbfile.add_acquisition(image_series)

# Processing module / ImageSegmentation
mod = nwbfile.create_processing_module('ophys', 'contains optical physiology processed data')
img_seg = ImageSegmentation()
mod.add(img_seg)

# PlaneSegmentation
ps = img_seg.create_plane_segmentation(
    description='output from segmenting my favorite imaging plane',
    imaging_plane=imaging_plane,
    name='my_planeseg',
)

# create 4 points mask
pix_mask1 = [[0, 0, 1], [0, 1, 1], [1, 0, 1], [1, 1, 1]]

mask_iterator = DataChunkIterator(
    data=iter_dataset(np.array(pix_mask1)),
)
ps.add_roi(pixel_mask=mask_iterator)     # With this it doesn't work!
#ps.add_roi(pixel_mask=pix_mask1)        # With this it works!

# Prints content of plane segmentation rois
print(nwbfile.processing['ophys'].data_interfaces['ImageSegmentation'].plane_segmentations['my_planeseg'].columns[1][:])

# Write file
with NWBHDF5IO('ophys_example_test.nwb', 'w') as io:
    io.write(nwbfile)

# Read file
with NWBHDF5IO('ophys_example_test.nwb', 'r') as io:
    nwbfile = io.read()
    print(nwbfile.processing['ophys'].data_interfaces['ImageSegmentation'].plane_segmentations['my_planeseg'].columns[1][:])

The TwoPhotonSeries data uses DataChunkIterator and gets properly stored, if rois are not passed. Rois are properly stored not using DataChunkIterator.

Before writing, this prints for plane_segmentations['my_planeseg'].columns[1]: [<hdmf.data_utils.DataChunk object at 0x0000015017864470>, <hdmf.data_utils.DataChunk object at 0x00000150178646D8>, <hdmf.data_utils.DataChunk object at 0x0000015017864780>, <hdmf.data_utils.DataChunk object at 0x00000150178647B8>]

and the full error track is:

Traceback (most recent call last):
  File "ophys.py", line 99, in <module>
    io.write(nwbfile)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 453, in func_call
    return func(args[0], **parsed['args'])
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 269, in write
    call_docval_func(super().write, kwargs)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 343, in call_docval_func
    return func(*fargs, **fkwargs)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 453, in func_call
    return func(args[0], **parsed['args'])
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\io.py", line 44, in write
    self.write_builder(f_builder, **kwargs)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 453, in func_call
    return func(args[0], **parsed['args'])
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 536, in write_builder
    self.write_group(self.__file, gbldr, exhaust_dci=exhaust_dci)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 453, in func_call
    return func(args[0], **parsed['args'])
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 696, in write_group
    self.write_group(group, sub_builder, exhaust_dci=exhaust_dci)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 453, in func_call
    return func(args[0], **parsed['args'])
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 696, in write_group
    self.write_group(group, sub_builder, exhaust_dci=exhaust_dci)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 453, in func_call
    return func(args[0], **parsed['args'])
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 696, in write_group
    self.write_group(group, sub_builder, exhaust_dci=exhaust_dci)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 453, in func_call
    return func(args[0], **parsed['args'])
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 701, in write_group
    self.write_dataset(group, sub_builder, exhaust_dci=exhaust_dci)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 453, in func_call
    return func(args[0], **parsed['args'])
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 898, in write_dataset
    dset = self.__list_fill__(parent, name, data, options)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\backends\hdf5\h5tools.py", line 1060, in __list_fill__
    data_shape = get_data_shape(data)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 633, in get_data_shape
    return __get_shape_helper(data)
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 623, in __get_shape_helper
    shape.extend(__get_shape_helper(local_data[0]))
  File "C:\Users\Luiz\Anaconda3\envs\jaeger_nwb\lib\site-packages\hdmf\utils.py", line 622, in __get_shape_helper
    if len(local_data) and not isinstance(local_data[0], (str, bytes)):
TypeError: 'DataChunk' object does not support indexing

[bendichter]

@luiztauffer, PlaneSegmentation is a column that concatenates all of the individual rows into columnar data. We don't have a way to concatenate DataChunkIterators, so this won't work. I'd be surprised if someone needed to iterator over a single cell of a table like this, but if this is needed there are ways of doing it by adding the column data directly.