KeyError/PicklingError: Can't pickle class.. it's not found as numba.experimental.jitclass

[rpopovici]

Hello guys,

I am trying to use numba with @jitclass. I have multiple classes with class attribute members and typed lists

Do you have any idea why I get this error?

This is a sample code..

import numba as nb
from numba import jit, types, typed
from numba.experimental import jitclass
from numba import int64, float64

@jitclass([('x', float64), ('y', float64), ('z', float64)])
class Vector(object):
    def __init__(self, x, y, z):
        self.x = x
        self.y = y
        self.z = z

VectorType = Vector.class_type.instance_type

# VertexType = nb.deferred_type()
@jitclass([('pos', Vector.class_type.instance_type)])     
class Vertex(object):
    def __init__(self, pos):
        self.pos = pos

    def clone(self):
        return Vertex(self.pos)       

# VertexType.define(Vertex.class_type.instance_type)
VertexType = Vertex.class_type.instance_type   

# @jit(VertexType(VertexType), nopython=False)
# def CloneVertex(v):
#     return Vertex(Vector(v.pos.x, v.pos.y, v.pos.z))

@jitclass([('vertices', types.ListType(Vertex.class_type.instance_type)), ('shared', int64)])  
class Polygon(object):
    def __init__(self, vertices, shared):
        self.vertices = vertices
        self.shared = shared

PolygonType = Polygon.class_type.instance_type

def load_mesh(mesh_data):
    polygons = typed.List.empty_list(PolygonType)

    for polygon in mesh_data:
        vertices = typed.List.empty_list(VertexType)
        for v in polygon['vertices']:
             vertices.append( Vertex(Vector(v[0], v[1], v[2])) )
        polygons.append( Polygon(vertices, polygon['shared']) )

    return polygons

@jit(nopython=False)
def put_back(p):
    return p[0].vertices[0].x

mesh = [{'vertices': [(0.0, 0.0, 0.0), (1.0, 0.0, 1.0), (-1.0, -1.0, 0.0)], 'shared': 0}, {'vertices': [(1.0, 0.0, 0.0), (0.0, 0.0, 1.0), (1.0, -1.0, 0.0)], 'shared': 1}, {'vertices': [(1.0, 1.0, 0.0), (0.0, 1.0, 1.0), (1.0, -1.0, 1.0)], 'shared': 2}]

And I am using it from this code:

# load mesh..
p = load_mesh(mesh)
print(put_back(p))

the mesh object is a plain python list of dicts with floats inside..

[rpopovici]

Debug client attached. Traceback (most recent call last): File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 518, in save self.save_global(obj) File "...\python\lib\pickle.py", line 960, in save_global (obj, module_name, name)) from None _pickle.PicklingError: Can't pickle <class 'numba.experimental.jitclass.base.Vector'>: it's not found as numba.experimental.jitclass.base.Vector

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "...\scripts\modules\addon_utils.py", line 351, in enable mod = import(module_name) File "...__init.py", line 34, in auto_load.init() File "...\auto_load.py", line 23, in init modules = get_all_submodules(Path(file).parent) File "...\auto_load.py", line 51, in get_all_submodules return list(iter_submodules(directory, directory.name)) File "...\auto_load.py", line 55, in iter_submodules yield importlib.import_module("." + name, package_name) File "...\python\lib\importlib\init__.py", line 127, in import_module return _bootstrap._gcd_import(name[level:], package, level) File "...\csg\numba\core.py", line 3, in from .geom import Vertex, Polygon, VertexfromPoints, PolygonfromVertices, PolygonType, VertexType, BSPNode, BSPNodeType File "...\csg\numba\geom.py", line 360, in @jit(Polygon.class_type.instance_type(types.ListType(Vertex.class_type.instance_type), int32), nopython=True) File "...\python\lib\site-packages\numba\core\decorators.py", line 213, in wrapper disp.compile(sig) File "...\python\lib\site-packages\numba\core\compiler_lock.py", line 32, in _acquire_compile_lock return func(*args, kwargs) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 819, in compile cres = self._compiler.compile(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 78, in compile status, retval = self._compile_cached(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 92, in _compile_cached retval = self._compile_core(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 110, in _compile_core pipeline_class=self.pipeline_class) File "...\python\lib\site-packages\numba\core\compiler.py", line 625, in compile_extra return pipeline.compile_extra(func) File "...\python\lib\site-packages\numba\core\compiler.py", line 361, in compile_extra return self._compile_bytecode() File "...\python\lib\site-packages\numba\core\compiler.py", line 423, in _compile_bytecode return self._compile_core() File "...\python\lib\site-packages\numba\core\compiler.py", line 403, in _compile_core raise e File "...\python\lib\site-packages\numba\core\compiler.py", line 394, in _compile_core pm.run(self.state) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 341, in run raise patched_exception File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 332, in run self._runPass(idx, pass_inst, state) File "...\python\lib\site-packages\numba\core\compiler_lock.py", line 32, in _acquire_compile_lock return func(*args, *kwargs) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 291, in _runPass mutated |= check(pss.run_pass, internal_state) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 264, in check mangled = func(compiler_state) File "...\python\lib\site-packages\numba\core\typed_passes.py", line 442, in run_pass NativeLowering().run_pass(state) File "...\python\lib\site-packages\numba\core\typed_passes.py", line 372, in run_pass lower.create_cpython_wrapper(flags.release_gil) File "...\python\lib\site-packages\numba\core\lowering.py", line 244, in create_cpython_wrapper release_gil=release_gil) File "...\python\lib\site-packages\numba\core\cpu.py", line 161, in create_cpython_wrapper builder.build() File "...\python\lib\site-packages\numba\core\callwrapper.py", line 122, in build self.build_wrapper(api, builder, closure, args, kws) File "...\python\lib\site-packages\numba\core\callwrapper.py", line 176, in build_wrapper obj = api.from_native_return(retty, retval, env_manager) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1359, in from_native_return out = self.from_native_value(typ, val, env_manager) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1373, in from_native_value return impl(typ, val, c) File "...\python\lib\site-packages\numba\experimental\jitclass\boxing.py", line 139, in _box_class_instance box_subclassed = _specialize_box(typ) File "...\python\lib\site-packages\numba\experimental\jitclass\boxing.py", line 122, in _specialize_box fast_fget = fget.compile((typ,)) File "...\python\lib\site-packages\numba\core\compiler_lock.py", line 32, in _acquire_compile_lock return func(args, kwargs) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 819, in compile cres = self._compiler.compile(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 78, in compile status, retval = self._compile_cached(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 92, in _compile_cached retval = self._compile_core(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 110, in _compile_core pipeline_class=self.pipeline_class) File "...\python\lib\site-packages\numba\core\compiler.py", line 625, in compile_extra return pipeline.compile_extra(func) File "...\python\lib\site-packages\numba\core\compiler.py", line 361, in compile_extra return self._compile_bytecode() File "...\python\lib\site-packages\numba\core\compiler.py", line 423, in _compile_bytecode return self._compile_core() File "...\python\lib\site-packages\numba\core\compiler.py", line 403, in _compile_core raise e File "...\python\lib\site-packages\numba\core\compiler.py", line 394, in _compile_core pm.run(self.state) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 341, in run raise patched_exception File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 332, in run self._runPass(idx, pass_inst, state) File "...\python\lib\site-packages\numba\core\compiler_lock.py", line 32, in _acquire_compile_lock return func(*args, *kwargs) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 291, in _runPass mutated |= check(pss.run_pass, internal_state) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 264, in check mangled = func(compiler_state) File "...\python\lib\site-packages\numba\core\typed_passes.py", line 442, in run_pass NativeLowering().run_pass(state) File "...\python\lib\site-packages\numba\core\typed_passes.py", line 372, in run_pass lower.create_cpython_wrapper(flags.release_gil) File "...\python\lib\site-packages\numba\core\lowering.py", line 244, in create_cpython_wrapper release_gil=release_gil) File "...\python\lib\site-packages\numba\core\cpu.py", line 161, in create_cpython_wrapper builder.build() File "...\python\lib\site-packages\numba\core\callwrapper.py", line 122, in build self.build_wrapper(api, builder, closure, args, kws) File "...\python\lib\site-packages\numba\core\callwrapper.py", line 176, in build_wrapper obj = api.from_native_return(retty, retval, env_manager) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1359, in from_native_return out = self.from_native_value(typ, val, env_manager) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1373, in from_native_value return impl(typ, val, c) File "...\python\lib\site-packages\numba\typed\typedlist.py", line 437, in box_lsttype lsttype_obj = c.pyapi.unserialize(c.pyapi.serialize_object(typ)) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1334, in serialize_object struct = self.serialize_uncached(obj) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1313, in serialize_uncached data = serialize.dumps(obj) File "...\python\lib\site-packages\numba\core\serialize.py", line 142, in dumps p.dump(obj) File "...\python\lib\pickle.py", line 437, in dump self.save(obj) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 549, in save self.save_reduce(obj=obj, rv) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 549, in save self.save_reduce(obj=obj, rv) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 549, in save self.save_reduce(obj=obj, rv) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 549, in save self.save_reduce(obj=obj, rv) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\site-packages\numba\core\serialize.py", line 323, in _save_custom_pickled return self.save_reduce(cp._reduce()) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\site-packages\numba\core\serialize.py", line 309, in _save_function self.save_reduce(_rebuild_function, args, obj=func) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 789, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 288, in save raise _TracedPicklingError(m) numba.core.serialize._TracedPicklingError: Failed in nopython mode pipeline (step: nopython mode backend) Failed in nopython mode pipeline (step: nopython mode backend) Failed to pickle because of PicklingError: Can't pickle <class 'numba.experimental.jitclass.base.Vector'>: it's not found as numba.experimental.jitclass.base.Vector tracing... [0]: <class 'numba.core.types.containers.ListType'>: 2258524412744 [1]: <class 'tuple'>: 2258527165000 [2]: <class 'dict'>: 2258524023688 [3]: <class 'numba.core.types.misc.ClassInstanceType'>: 2258517313736 [4]: <class 'tuple'>: 2258527164120 [5]: <class 'dict'>: 2258481013368 [6]: <class 'numba.core.types.misc.ClassType'>: 2258517312520 [7]: <class 'tuple'>: 2258527165320 [8]: <class 'dict'>: 2258517300072 [9]: <class 'dict'>: 2258517235976 [10]: <class 'numba.core.registry.CPUDispatcher'>: 2258517219528 [11]: <class 'tuple'>: 2258523633352 [12]: <class 'numba.core.serialize._CustomPickled'>: 2258523559944 [13]: <class 'tuple'>: 2258527350280 [14]: <class 'dict'>: 2258527351336 [15]: <class 'function'>: 2258517275704 [16]: <class 'tuple'>: 2258525186952 [17]: <class 'dict'>: 2258527351976 [18]: <class 'numba.experimental.jitclass.base.JitClassType'>: 2258490097240

Error: Traceback (most recent call last): File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 518, in save self.save_global(obj) File "...\python\lib\pickle.py", line 960, in save_global (obj, module_name, name)) from None _pickle.PicklingError: Can't pickle <class 'numba.experimental.jitclass.base.Vector'>: it's not found as numba.experimental.jitclass.base.Vector

During handling of the above exception, another exception occurred:

Traceback (most recent call last): RuntimeError: Error: Traceback (most recent call last): File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 518, in save self.save_global(obj) File "...\python\lib\pickle.py", line 960, in save_global (obj, module_name, name)) from None _pickle.PicklingError: Can't pickle <class 'numba.experimental.jitclass.base.Vector'>: it's not found as numba.experimental.jitclass.base.Vector

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "...\scripts\modules\addon_utils.py", line 351, in enable mod = import(module_name) File "...__init.py", line 34, in auto_load.init() File "...\auto_load.py", line 23, in init modules = get_all_submodules(Path(file).parent) File "...\auto_load.py", line 51, in get_all_submodules return list(iter_submodules(directory, directory.name)) File "...\auto_load.py", line 55, in iter_submodules yield importlib.import_module("." + name, package_name) File "...\python\lib\importlib\init__.py", line 127, in import_module return _bootstrap._gcd_import(name[level:], package, level) File "...\csg\numba\core.py", line 3, in from .geom import Vertex, Polygon, VertexfromPoints, PolygonfromVertices, PolygonType, VertexType, BSPNode, BSPNodeType File "...\csg\numba\geom.py", line 360, in @jit(Polygon.class_type.instance_type(types.ListType(Vertex.class_type.instance_type), int32), nopython=True) File "...\python\lib\site-packages\numba\core\decorators.py", line 213, in wrapper disp.compile(sig) File "...\python\lib\site-packages\numba\core\compiler_lock.py", line 32, in _acquire_compile_lock return func(*args, kwargs) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 819, in compile cres = self._compiler.compile(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 78, in compile status, retval = self._compile_cached(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 92, in _compile_cached retval = self._compile_core(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 110, in _compile_core pipeline_class=self.pipeline_class) File "...\python\lib\site-packages\numba\core\compiler.py", line 625, in compile_extra return pipeline.compile_extra(func) File "...\python\lib\site-packages\numba\core\compiler.py", line 361, in compile_extra return self._compile_bytecode() File "...\python\lib\site-packages\numba\core\compiler.py", line 423, in _compile_bytecode return self._compile_core() File "...\python\lib\site-packages\numba\core\compiler.py", line 403, in _compile_core raise e File "...\python\lib\site-packages\numba\core\compiler.py", line 394, in _compile_core pm.run(self.state) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 341, in run raise patched_exception File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 332, in run self._runPass(idx, pass_inst, state) File "...\python\lib\site-packages\numba\core\compiler_lock.py", line 32, in _acquire_compile_lock return func(*args, *kwargs) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 291, in _runPass mutated |= check(pss.run_pass, internal_state) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 264, in check mangled = func(compiler_state) File "...\python\lib\site-packages\numba\core\typed_passes.py", line 442, in run_pass NativeLowering().run_pass(state) File "...\python\lib\site-packages\numba\core\typed_passes.py", line 372, in run_pass lower.create_cpython_wrapper(flags.release_gil) File "...\python\lib\site-packages\numba\core\lowering.py", line 244, in create_cpython_wrapper release_gil=release_gil) File "...\python\lib\site-packages\numba\core\cpu.py", line 161, in create_cpython_wrapper builder.build() File "...\python\lib\site-packages\numba\core\callwrapper.py", line 122, in build self.build_wrapper(api, builder, closure, args, kws) File "...\python\lib\site-packages\numba\core\callwrapper.py", line 176, in build_wrapper obj = api.from_native_return(retty, retval, env_manager) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1359, in from_native_return out = self.from_native_value(typ, val, env_manager) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1373, in from_native_value return impl(typ, val, c) File "...\python\lib\site-packages\numba\experimental\jitclass\boxing.py", line 139, in _box_class_instance box_subclassed = _specialize_box(typ) File "...\python\lib\site-packages\numba\experimental\jitclass\boxing.py", line 122, in _specialize_box fast_fget = fget.compile((typ,)) File "...\python\lib\site-packages\numba\core\compiler_lock.py", line 32, in _acquire_compile_lock return func(args, kwargs) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 819, in compile cres = self._compiler.compile(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 78, in compile status, retval = self._compile_cached(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 92, in _compile_cached retval = self._compile_core(args, return_type) File "...\python\lib\site-packages\numba\core\dispatcher.py", line 110, in _compile_core pipeline_class=self.pipeline_class) File "...\python\lib\site-packages\numba\core\compiler.py", line 625, in compile_extra return pipeline.compile_extra(func) File "...\python\lib\site-packages\numba\core\compiler.py", line 361, in compile_extra return self._compile_bytecode() File "...\python\lib\site-packages\numba\core\compiler.py", line 423, in _compile_bytecode return self._compile_core() File "...\python\lib\site-packages\numba\core\compiler.py", line 403, in _compile_core raise e File "...\python\lib\site-packages\numba\core\compiler.py", line 394, in _compile_core pm.run(self.state) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 341, in run raise patched_exception File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 332, in run self._runPass(idx, pass_inst, state) File "...\python\lib\site-packages\numba\core\compiler_lock.py", line 32, in _acquire_compile_lock return func(*args, *kwargs) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 291, in _runPass mutated |= check(pss.run_pass, internal_state) File "...\python\lib\site-packages\numba\core\compiler_machinery.py", line 264, in check mangled = func(compiler_state) File "...\python\lib\site-packages\numba\core\typed_passes.py", line 442, in run_pass NativeLowering().run_pass(state) File "...\python\lib\site-packages\numba\core\typed_passes.py", line 372, in run_pass lower.create_cpython_wrapper(flags.release_gil) File "...\python\lib\site-packages\numba\core\lowering.py", line 244, in create_cpython_wrapper release_gil=release_gil) File "...\python\lib\site-packages\numba\core\cpu.py", line 161, in create_cpython_wrapper builder.build() File "...\python\lib\site-packages\numba\core\callwrapper.py", line 122, in build self.build_wrapper(api, builder, closure, args, kws) File "...\python\lib\site-packages\numba\core\callwrapper.py", line 176, in build_wrapper obj = api.from_native_return(retty, retval, env_manager) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1359, in from_native_return out = self.from_native_value(typ, val, env_manager) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1373, in from_native_value return impl(typ, val, c) File "...\python\lib\site-packages\numba\typed\typedlist.py", line 437, in box_lsttype lsttype_obj = c.pyapi.unserialize(c.pyapi.serialize_object(typ)) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1334, in serialize_object struct = self.serialize_uncached(obj) File "...\python\lib\site-packages\numba\core\pythonapi.py", line 1313, in serialize_uncached data = serialize.dumps(obj) File "...\python\lib\site-packages\numba\core\serialize.py", line 142, in dumps p.dump(obj) File "...\python\lib\pickle.py", line 437, in dump self.save(obj) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 549, in save self.save_reduce(obj=obj, rv) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 549, in save self.save_reduce(obj=obj, rv) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 549, in save self.save_reduce(obj=obj, rv) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 549, in save self.save_reduce(obj=obj, rv) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\site-packages\numba\core\serialize.py", line 323, in _save_custom_pickled return self.save_reduce(cp._reduce()) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 774, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\site-packages\numba\core\serialize.py", line 309, in _save_function self.save_reduce(_rebuild_function, args, obj=func) File "...\python\lib\pickle.py", line 638, in save_reduce save(args) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 789, in save_tuple save(element) File "...\python\lib\site-packages\numba\core\serialize.py", line 279, in save return super().save(obj) File "...\python\lib\pickle.py", line 504, in save f(self, obj) # Call unbound method with explicit self File "...\python\lib\pickle.py", line 859, in save_dict self._batch_setitems(obj.items()) File "...\python\lib\pickle.py", line 885, in _batch_setitems save(v) File "...\python\lib\site-packages\numba\core\serialize.py", line 288, in save raise _TracedPicklingError(m) numba.core.serialize._TracedPicklingError: Failed in nopython mode pipeline (step: nopython mode backend) Failed in nopython mode pipeline (step: nopython mode backend) Failed to pickle because of PicklingError: Can't pickle <class 'numba.experimental.jitclass.base.Vector'>: it's not found as numba.experimental.jitclass.base.Vector tracing...

[esc]

@rpopovici thanks for asking about this on the Numba issue tracker. For the time being jitclass is not picklable, xref: https://github.com/numba/numba/issues/1846

[rpopovici]

@esc can you suggest an alternative solution to this problem? thanks

[esc]

@rpopovici not sure what the intent is here. Perhaps you could supply a fully runnable reproducer, i.e. something that includes a mesh and all the other stuff like VertexFromPoints etc.. It would allow me to reproduce your issue locally.

[rpopovici]

@esc I updated the initial post. It should be runnable now..

[rpopovici]

@esc by commenting the clone() method in my example, I discovered that calling the class constructor from class method causes the pickle error. I thought that calling the class constructor from class method was supported by numba. Is that a correct assumption or I am mistaken? Can you pls suggest an alternative solution. thanks

Also the jupyter notebook kernel crashes for some reason when I try to use nb.deferred_type()..

[esc]

@rpopovici thanks for following up on this. I'm afraid I don't quite follow what you mean with ", I discovered that calling the class constructor from class method causes the pickle error." I don't see any @classmethod decorators in your code. Did you perhaps mean "instance method"?

[esc]

@rpopovici also, I am confused about the clone method, it doesn't seem to be used anywhere in the example, or did I miss something?

[rpopovici]

@esc sorry for not being very explicit. The clone method it's not used in the example code I pasted here, but it's used in my dev project.. There is no @classmethod, it's the instance method clone(). Just simply by removing any constructor being called in methods proved to make the pickle error go away, but I need to be able to instantiate new objects from methods..

[esc]

OK, that is interesting, however in the above example if I comment out the clone method, I do still get the pickling error?

[rpopovici]

@esc I simplified the code a little bit. Now there is only one clone method with one constructor call. If you remove this method, then it will work and you can call load_mesh(mesh) without errors

[esc]

@rpopovici thanks for following up on this. I was assuming, that this error arises from trying to instantiate new object from methods. However, it seems I was mistaken. I created a small reproducer to investigate this, and it seems to work fine:

from numba import int64
from numba.experimental import jitclass

spec = [
    ('value', int64),
]

@jitclass(spec)
class Bag(object):
    def __init__(self, value):
        self.value = value

    def clone(self):
        return Bag(self.value)

n = 21
mybag = Bag(n)
print(mybag.value)
clone_bag = mybag.clone()
print(clone_bag.value)

So the question is, what is it about your snippet that causes the error in the first place? I think it would be good to reduce the example to something smaller with less boilerplate to expose the original issue. I am also re-opening this, since you are not trying to pickle a jitclass explicitly and the error you are seeing seems to be a symptom of something else, which may be worth investigating.

[rpopovici]

@esc thanks. I am looking into it

[esc]

Incidentally, when taking a closer look at the error message when I run your example locally, I see:

Traceback (most recent call last):
  File "/Users/vhaenel/git/numba/numba/core/pythonapi.py", line 1328, in serialize_object
    gv = self.module.__serialized[obj]
KeyError: ListType[instance.jitclass.Vertex#10c974040<pos:instance.jitclass.Vector#10c95ef10<x:float64,y:float64,z:float64>>]

So this may in fact be related to the typed list in there.

[rpopovici]

@esc the culprit appear to be this line:

@jitclass([('vertices', types.ListType(Vertex.class_type.instance_type)), ('shared', int64)]) 
class Polygon(object):

It's unable to deal with the custom list type definition

[esc]

@rpopovici excellent, so the hypothesis is that using a numba.typed.List somehow requires something to be pickled and then it falls over?

[rpopovici]

@esc I think the problem starts with the fact that it cannot handle numba.typed.List(class_A_type) in class_B definition, hence the KeyError and the pickeling is another issue probably related to the KeyError..

[esc]

@rpopovici it might make sense to consult the following section in the docs about this: https://numba.pydata.org/numba-doc/latest/user/jitclass.html#specifying-numba-typed-containers-as-class-members

[rpopovici]

This is the KeyError I got. I thought I put it in the initial error log, but I didn't..

KeyError Traceback (most recent call last) ~\AppData\Local\Programs\Python\Python37\lib\site-packages\numba-0+unknown-py3.7-win-amd64.egg\numba\core\pythonapi.py in serialize_object(self, obj) 1331 try: -> 1332 gv = self.module.__serialized[obj] 1333 except KeyError:

KeyError: ListType[instance.jitclass.Vertex#26e9c666b48>]

During handling of the above exception, another exception occurred:

PicklingError Traceback (most recent call last) in ----> 1 p = load_mesh(mesh) ....

[esc]

yeah, it looks like the type contains an instance of a Vertex, not the type spec, (i think).

[rpopovici]

yes, I tried to preinitialize the list type with list_type= types.ListType(VertexType), I also tried typeof. None of those work. The list type contains a custom instance type, not a "standard" numba type

[rpopovici]

and the problem arises only in the class definition with @jitclass.. anywhere else in the code, you can use custom types in typed list

[stuartarchibald]

Reproducer:

from numba import typed, typeof, njit, int64
from numba.experimental import jitclass

spec = [
    ('value', int64),
]

@jitclass(spec)
class Bag(object):
    def __init__(self, value):
        self.value = value

    def clone(self):
        return Bag(self.value)

mybag = Bag(21)

x = typed.List.empty_list(Bag.class_type.instance_type)
x.append(mybag)

@njit
def foo(z):
    return z

foo(x)

the issue is that Bag when called from clone is not actually Bag as it's decorated, as a result it can't be serialized as needed by the cpython transport wrappers.

[rpopovici]

@stuartarchibald is there any workaround to this issue?

[stuartarchibald]

This help?

from numba import typed, typeof, njit, int64, types
from numba.experimental import jitclass
from numba.extending import overload_method, overload

spec = [
    ('value', int64),
]

@jitclass(spec)
class Bag(object):
    def __init__(self, value):
        self.value = value

mybag = Bag(21)

x = typed.List.empty_list(Bag.class_type.instance_type)
x.append(mybag)

@overload_method(types.misc.ClassInstanceType, 'clone')
def ol_bag_clone(inst,):
    if inst is Bag.class_type.instance_type:
        def impl(inst,):
            return Bag(inst.value)
        return impl

@njit
def foo(z):
    t = z[0]
    t.clone()

foo(x)

also think the original post needs :

@jit(nopython=False)
def put_back(p):
    return p[0].vertices[0].pos.x

.x is an attr on the pos instance.

[rpopovici]

@stuartarchibald this definitely heps. thanks!