creating custom MultiContainerInterface extensions

NeurodataWithoutBorders / pynwb

A Python API for working with Neurodata stored in the NWB Format

https://pynwb.readthedocs.io

Other

176 stars 85 forks source link

creating custom MultiContainerInterface extensions #566

Closed bendichter closed 6 years ago

bendichter commented 6 years ago

I'm trying to make my own multicontainerinterfaces but I am finding that when I try to use them they are empty.

Here's the minimal code:

from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec
from pynwb import register_class, load_namespaces, NWBHDF5IO, NWBFile
from pynwb.file import MultiContainerInterface, NWBContainer

name = 'test_multicontainerinterface'
ns_path = name + ".namespace.yaml"
ext_source = name + ".extensions.yaml"

potato = NWBGroupSpec(neurodata_type_def='Potato',
                      doc='time of multicontainer', quantity='*',
                      attributes=[
                          NWBAttributeSpec(name='weight',
                                           doc='weight of potato',
                                           dtype='float',
                                           required=True),
                          NWBAttributeSpec(name='age',
                                           doc='age of potato',
                                           dtype='float',
                                           required=False)
                      ])

potato_sack = NWBGroupSpec(neurodata_type_def='PotatoSack', name='potato_sack',
                           doc='test of multicontainer', quantity='?',
                           groups=[potato])

ns_builder = NWBNamespaceBuilder(name + ' extensions', name)
ns_builder.add_spec(ext_source, potato_sack)
ns_builder.export(ns_path)

###

load_namespaces(ns_path)

@register_class('Potato', name)
class Potato(NWBContainer):
    __nwbfields__ = ('name', 'weight', 'age', 'source')

    def __init__(self, **kwargs):
        super(Potato, self).__init__(kwargs['source'], name=kwargs['name'])
        self.weight = kwargs['weight']
        self.age = kwargs['age']

@register_class('PotatoSack', name)
class PotatoSack(MultiContainerInterface):

    __nwbfields__ = ('potato',)

    __clsconf__ = {
        'attr': 'potato',
        'type': Potato,
        'add': 'add_potato',
        'get': 'get_potato',
        'create': 'create_potato',
    }

    __help = 'info about potatoes'

####

potato_sack = PotatoSack(source='pantry',
                         potato=Potato(name='potato1', age=2.3, weight=3.0, source='the ground'))

nwbfile = NWBFile("source", "a file with metadata", "NB123A", '2018-06-01T00:00:00')

pmod = nwbfile.create_processing_module('module_name', 'source', 'desc')
pmod.add_container(potato_sack)

with NWBHDF5IO('test_multicontainerinterface.nwb', 'w') as io:
    io.write(nwbfile)

Here's the HDFView. Notice, no 'potato1':

Am I missing something?

ajtritt commented 6 years ago

Your new types are not actually NWBDataInterfaces, so the autogenerated class stuff breaks down. Add neurodata_type_inc=NWBDataInterface the PotatoSack spec

bendichter commented 6 years ago

@ajtritt thanks for looking into this. I don't think that fixed it. I'm assuming you mean neurodata_type_inc='NWBDataInterface'.

from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec
from pynwb import register_class, load_namespaces, NWBHDF5IO, NWBFile
from pynwb.file import MultiContainerInterface, NWBContainer, NWBDataInterface

name = 'test_multicontainerinterface'
ns_path = name + ".namespace.yaml"
ext_source = name + ".extensions.yaml"

potato = NWBGroupSpec(neurodata_type_def='Potato',
                      doc='time of multicontainer', quantity='*',
                      attributes=[
                          NWBAttributeSpec(name='weight',
                                           doc='weight of potato',
                                           dtype='float',
                                           required=True),
                          NWBAttributeSpec(name='age',
                                           doc='age of potato',
                                           dtype='float',
                                           required=False)
                      ])

potato_sack = NWBGroupSpec(neurodata_type_def='PotatoSack',
                           neurodata_type_inc='NWBDataInterface',
                           name='potato_sack',
                           doc='test of multicontainer', quantity='?',
                           groups=[potato],
                           attributes=[
                               NWBAttributeSpec(name='help',
                                                doc='help',
                                                dtype='text',
                                                value="It's a sack of potatoes")
                           ])

ns_builder = NWBNamespaceBuilder(name + ' extensions', name)
ns_builder.add_spec(ext_source, potato_sack)
ns_builder.export(ns_path)

###

load_namespaces(ns_path)

@register_class('Potato', name)
class Potato(NWBContainer):
    __nwbfields__ = ('name', 'weight', 'age', 'source')

    def __init__(self, **kwargs):
        super(Potato, self).__init__(kwargs['source'], name=kwargs['name'])
        self.weight = kwargs['weight']
        self.age = kwargs['age']

@register_class('PotatoSack', name)
class PotatoSack(MultiContainerInterface):

    __nwbfields__ = ('potato',)

    __clsconf__ = {
        'attr': 'potato',
        'type': Potato,
        'add': 'add_potato',
        'get': 'get_potato',
        'create': 'create_potato',
    }

    __help = 'info about potatoes'

####

potato_sack = PotatoSack(source='pantry',
                         potato=Potato(name='potato1', age=2.3, weight=3.0, source='the ground'))

nwbfile = NWBFile("source", "a file with metadata", "NB123A", '2018-06-01T00:00:00')

pmod = nwbfile.create_processing_module('module_name', 'source', 'desc')
pmod.add_container(potato_sack)

with NWBHDF5IO('test_multicontainerinterface.nwb', 'w') as io:
    io.write(nwbfile)

same result.

ajtritt commented 6 years ago

A few things:

1) Potato should have a neurodata_type_inc -- either NWBContainer or NWBDataInterface. I don't think this contributed anything to this problem, but you should always do this to avoid other problems.

2) The default ObjectMapper will look for Potato objects under PotatoSack.potatos not PotatoSack.potato. This is actually the cause of the problem.

There was no warning about the missing Potato objects because, from the ObjectMapper's perspective, nothing is missing--you have quantity set to zero-or-more

bendichter commented 6 years ago

It works!

from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec
from pynwb import register_class, load_namespaces, NWBHDF5IO, NWBFile
from pynwb.file import MultiContainerInterface, NWBContainer

name = 'test_multicontainerinterface'
ns_path = name + ".namespace.yaml"
ext_source = name + ".extensions.yaml"

potato = NWBGroupSpec(neurodata_type_def='Potato',
                      neurodata_type_inc='NWBDataInterface',
                      doc='time of multicontainer', quantity='*',
                      attributes=[
                          NWBAttributeSpec(name='weight',
                                           doc='weight of potato',
                                           dtype='float',
                                           required=True),
                          NWBAttributeSpec(name='age',
                                           doc='age of potato',
                                           dtype='float',
                                           required=False)
                      ])

potato_sack = NWBGroupSpec(neurodata_type_def='PotatoSack',
                           neurodata_type_inc='NWBDataInterface',
                           name='potato_sack',
                           doc='test of multicontainer', quantity='?',
                           groups=[potato],
                           attributes=[
                               NWBAttributeSpec(name='help',
                                                doc='help',
                                                dtype='text',
                                                value="It's a sack of potatoes")
                           ])

ns_builder = NWBNamespaceBuilder(name + ' extensions', name)
ns_builder.add_spec(ext_source, potato_sack)
ns_builder.export(ns_path)

###

load_namespaces(ns_path)

@register_class('Potato', name)
class Potato(NWBContainer):
    __nwbfields__ = ('name', 'weight', 'age', 'source')

    def __init__(self, **kwargs):
        super(Potato, self).__init__(kwargs['source'], name=kwargs['name'])
        self.weight = kwargs['weight']
        self.age = kwargs['age']

@register_class('PotatoSack', name)
class PotatoSack(MultiContainerInterface):

    __nwbfields__ = ('potatos',)

    __clsconf__ = {
        'attr': 'potatos',
        'type': Potato,
        'add': 'add_potato',
        'get': 'get_potato',
        'create': 'create_potato',
    }

    __help = 'info about potatoes'

####

potato_sack = PotatoSack(source='pantry',
                         potatos=Potato(name='potato1', age=2.3, weight=3.0, 
                                        source='the ground'))

nwbfile = NWBFile("source", "a file with metadata", "NB123A", '2018-06-01T00:00:00')

pmod = nwbfile.create_processing_module('module_name', 'source', 'desc')
pmod.add_container(potato_sack)

with NWBHDF5IO('test_multicontainerinterface.nwb', 'w') as io:
    io.write(nwbfile)

The changes I made were:

__nwbfields__ = ('potatos',)

        'attr': 'potatos',

potato_sack = PotatoSack(source='pantry',
                         potatos=Potato(name='potato1', age=2.3, weight=3.0,
                                        source='the ground'))

bendichter commented 6 years ago

It works!

from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec
from pynwb import register_class, load_namespaces, NWBHDF5IO, NWBFile
from pynwb.file import MultiContainerInterface, NWBContainer

name = 'test_multicontainerinterface'
ns_path = name + ".namespace.yaml"
ext_source = name + ".extensions.yaml"

potato = NWBGroupSpec(neurodata_type_def='Potato',
                      neurodata_type_inc='NWBDataInterface',
                      doc='time of multicontainer', quantity='*',
                      attributes=[
                          NWBAttributeSpec(name='weight',
                                           doc='weight of potato',
                                           dtype='float',
                                           required=True),
                          NWBAttributeSpec(name='age',
                                           doc='age of potato',
                                           dtype='float',
                                           required=False)
                      ])

potato_sack = NWBGroupSpec(neurodata_type_def='PotatoSack',
                           neurodata_type_inc='NWBDataInterface',
                           name='potato_sack',
                           doc='test of multicontainer', quantity='?',
                           groups=[potato],
                           attributes=[
                               NWBAttributeSpec(name='help',
                                                doc='help',
                                                dtype='text',
                                                value="It's a sack of potatoes")
                           ])

ns_builder = NWBNamespaceBuilder(name + ' extensions', name)
ns_builder.add_spec(ext_source, potato_sack)
ns_builder.export(ns_path)

###

load_namespaces(ns_path)

@register_class('Potato', name)
class Potato(NWBContainer):
    __nwbfields__ = ('name', 'weight', 'age', 'source')

    def __init__(self, **kwargs):
        super(Potato, self).__init__(kwargs['source'], name=kwargs['name'])
        self.weight = kwargs['weight']
        self.age = kwargs['age']

@register_class('PotatoSack', name)
class PotatoSack(MultiContainerInterface):

    __nwbfields__ = ('potatos',)

    __clsconf__ = {
        'attr': 'potatos',
        'type': Potato,
        'add': 'add_potato',
        'get': 'get_potato',
        'create': 'create_potato',
    }

    __help = 'info about potatoes'

####

potato_sack = PotatoSack(source='pantry',
                         potatos=Potato(name='potato1', age=2.3, weight=3.0, 
                                        source='the ground'))

nwbfile = NWBFile("source", "a file with metadata", "NB123A", '2018-06-01T00:00:00')

pmod = nwbfile.create_processing_module('module_name', 'source', 'desc')
pmod.add_container(potato_sack)

with NWBHDF5IO('test_multicontainerinterface.nwb', 'w') as io:
    io.write(nwbfile)

The changes I made were:

__nwbfields__ = ('potatos',)

        'attr': 'potatos',

potato_sack = PotatoSack(source='pantry',
                         potatos=Potato(name='potato1', age=2.3, weight=3.0,
                                        source='the ground'))

bendichter commented 6 years ago

It works!

from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec
from pynwb import register_class, load_namespaces, NWBHDF5IO, NWBFile
from pynwb.file import MultiContainerInterface, NWBContainer

name = 'test_multicontainerinterface'
ns_path = name + ".namespace.yaml"
ext_source = name + ".extensions.yaml"

potato = NWBGroupSpec(neurodata_type_def='Potato',
                      neurodata_type_inc='NWBDataInterface',
                      doc='time of multicontainer', quantity='*',
                      attributes=[
                          NWBAttributeSpec(name='weight',
                                           doc='weight of potato',
                                           dtype='float',
                                           required=True),
                          NWBAttributeSpec(name='age',
                                           doc='age of potato',
                                           dtype='float',
                                           required=False)
                      ])

potato_sack = NWBGroupSpec(neurodata_type_def='PotatoSack',
                           neurodata_type_inc='NWBDataInterface',
                           name='potato_sack',
                           doc='test of multicontainer', quantity='?',
                           groups=[potato],
                           attributes=[
                               NWBAttributeSpec(name='help',
                                                doc='help',
                                                dtype='text',
                                                value="It's a sack of potatoes")
                           ])

ns_builder = NWBNamespaceBuilder(name + ' extensions', name)
ns_builder.add_spec(ext_source, potato_sack)
ns_builder.export(ns_path)

###

load_namespaces(ns_path)

@register_class('Potato', name)
class Potato(NWBContainer):
    __nwbfields__ = ('name', 'weight', 'age', 'source')

    def __init__(self, **kwargs):
        super(Potato, self).__init__(kwargs['source'], name=kwargs['name'])
        self.weight = kwargs['weight']
        self.age = kwargs['age']

@register_class('PotatoSack', name)
class PotatoSack(MultiContainerInterface):

    __nwbfields__ = ('potatos',)

    __clsconf__ = {
        'attr': 'potatos',
        'type': Potato,
        'add': 'add_potato',
        'get': 'get_potato',
        'create': 'create_potato',
    }

    __help = 'info about potatoes'

####

potato_sack = PotatoSack(source='pantry',
                         potatos=Potato(name='potato1', age=2.3, weight=3.0, 
                                        source='the ground'))

nwbfile = NWBFile("source", "a file with metadata", "NB123A", '2018-06-01T00:00:00')

pmod = nwbfile.create_processing_module('module_name', 'source', 'desc')
pmod.add_container(potato_sack)

with NWBHDF5IO('test_multicontainerinterface.nwb', 'w') as io:
    io.write(nwbfile)

The changes I made were:

__nwbfields__ = ('potatos',)

        'attr': 'potatos',

potato_sack = PotatoSack(source='pantry',
                         potatos=Potato(name='potato1', age=2.3, weight=3.0,
                                        source='the ground'))

ajtritt commented 6 years ago

For your reference, you can forgo

__nwbfields__ = ('potatos',)

This part will do that for you:

        'attr': 'potatos',

bendichter commented 6 years ago

OK good to know, thanks!

On Mon, Jul 30, 2018 at 1:41 PM Andrew Tritt notifications@github.com wrote:

For your reference, you can forgo

nwbfields = ('potatos',)

This part will do that for you:
    'attr': 'potatos',
— You are receiving this because you were assigned. Reply to this email directly, view it on GitHub https://github.com/NeurodataWithoutBorders/pynwb/issues/566#issuecomment-408948899, or mute the thread https://github.com/notifications/unsubscribe-auth/AAziEsg5wM6MaWWQlAxVqLrLttCpBG6Aks5uL0U_gaJpZM4VgmvF .

--

Ben Dichter, PhD Data Science Consultant