Bug disappears when the classes/ functions in importfile are moved to main.
main.py
from importfile import test_function
from padl import transform
@transform
class ToProject:
def __init__(self, n_components=20):
self.n_components = n_components
def __call__(self, x):
return x
PROJECTION = ToProject(n_components=20)
GENERATOR_PIPELINE = test_function(
projection=PROJECTION,
in_channels=50,
)
importfile
from padl import transform
@transform
class Dummy:
def __init__(self, a):
self.a = a
def __call__(self, x):
return x + self.a
def test_function(projection, in_channels):
return Dummy(in_channels - projection.n_components)
ERROR:
(often different each time)
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:768, in find(var_name, module, i)
767 try:
--> 768 return find_in_module(var_name, module, i)
769 except TypeError as exc:
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:699, in find_in_module(var_name, module, i)
692 """Find the piece of code that assigned a value to the variable with name *var_name* in the
693 module *module*.
694
(...)
697 :returns: Tuple with source code segment and corresponding ast node.
698 """
--> 699 source = sourceget.get_module_source(module)
700 return find_in_source(var_name, source, i=i)
File ~/Deeplearning/padl/padl/dumptools/sourceget.py:61, in get_module_source(module, use_replace_cache)
60 pass
---> 61 return inspect.getsource(module)
File /usr/lib/python3.9/inspect.py:1024, in getsource(object)
1019 """Return the text of the source code for an object.
1020
1021 The argument may be a module, class, method, function, traceback, frame,
1022 or code object. The source code is returned as a single string. An
1023 OSError is raised if the source code cannot be retrieved."""
-> 1024 lines, lnum = getsourcelines(object)
1025 return ''.join(lines)
File /usr/lib/python3.9/inspect.py:1006, in getsourcelines(object)
1005 object = unwrap(object)
-> 1006 lines, lnum = findsource(object)
1008 if istraceback(object):
File /usr/lib/python3.9/inspect.py:817, in findsource(object)
810 """Return the entire source file and starting line number for an object.
811
812 The argument may be a module, class, method, function, traceback, frame,
813 or code object. The source code is returned as a list of all the lines
814 in the file and the line number indexes a line in that list. An OSError
815 is raised if the source code cannot be retrieved."""
--> 817 file = getsourcefile(object)
818 if file:
819 # Invalidate cache if needed.
File /usr/lib/python3.9/inspect.py:697, in getsourcefile(object)
694 """Return the filename that can be used to locate an object's source.
695 Return None if no way can be identified to get the source.
696 """
--> 697 filename = getfile(object)
698 all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:]
File /usr/local/lib/python3.9/dist-packages/torch/package/package_importer.py:620, in patched_getfile(object)
619 return _package_imported_modules[object.__module__].__file__
--> 620 return _orig_getfile(object)
File /usr/lib/python3.9/inspect.py:660, in getfile(object)
659 return object.__file__
--> 660 raise TypeError('{!r} is a built-in module'.format(object))
661 if isclass(object):
TypeError: <module '__main__'> is a built-in module
During handling of the above exception, another exception occurred:
NameNotFound Traceback (most recent call last)
Input In [2], in <module>
1 from padl import save
----> 2 save(cf.GENERATOR_PIPELINE, 'test.padl', force_overwrite=True)
File ~/Deeplearning/padl/padl/transforms.py:2412, in save(transform, path, force_overwrite, compress)
2410 transform.pd_zip_save(path, force_overwrite)
2411 else:
-> 2412 transform.pd_save(path, force_overwrite)
File ~/Deeplearning/padl/padl/transforms.py:382, in Transform.pd_save(self, path, force_overwrite)
380 for i, subtrans in enumerate(self._pd_all_transforms()):
381 subtrans.pd_pre_save(path, i)
--> 382 code, versions = self._pd_dumps(True, path=path)
384 with open(path / 'transform.py', 'w') as f:
385 f.write(code)
File ~/Deeplearning/padl/padl/transforms.py:586, in Transform._pd_dumps(self, return_versions, path)
578 def _pd_dumps(self, return_versions: bool = False,
579 path: Optional[Path] = None) -> Union[str, Tuple[str, str]]:
580 """Dump the transform as python code.
581
582 :param return_versions: If *True* return a tuple of the code and a file listing
583 dependencies and their versions.
584 :param path: Optional path to save at, might be required for serializer code snippets.
585 """
--> 586 graph = self._pd_build_codegraph(name='_pd_main')
587 Serializer.save_all(graph, path)
588 code = graph.dumps()
File ~/Deeplearning/padl/padl/transforms.py:510, in Transform._pd_build_codegraph(self, graph, name)
507 continue
509 # find how *next_name* came into being
--> 510 next_codenode = find_codenode(next_name)
511 graph[next_name] = next_codenode
513 todo.update(next_codenode.globals_)
File ~/Deeplearning/padl/padl/dumptools/var2mod.py:548, in find_codenode(name)
546 def find_codenode(name: ScopedName):
547 """Find the :class:`CodeNode` corresponding to a :class:`ScopedName` *name*. """
--> 548 (source, node), scope_of_next_var = find_in_scope(name)
550 # find dependencies
551 globals_ = find_globals(node)
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:636, in find_in_scope(name)
634 if scope.module is None:
635 raise NameNotFound(f'{name.name} not found in function hierarchy.')
--> 636 source, node = find(name.name, scope.module, i)
637 if getattr(node, '_globalscope', False):
638 scope = Scope.empty()
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:772, in find(var_name, module, i)
770 if module is not sys.modules['__main__']:
771 raise NameNotFound(f'"{var_name}" not found.') from exc
--> 772 return find_in_ipython(var_name, i)
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:750, in find_in_ipython(var_name, i)
748 break
749 if source is None:
--> 750 raise NameNotFound(f'"{var_name}" not found.')
751 return source, node
NameNotFound: "Dummy" not found.
OR:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:768, in find(var_name, module, i)
767 try:
--> 768 return find_in_module(var_name, module, i)
769 except TypeError as exc:
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:699, in find_in_module(var_name, module, i)
692 """Find the piece of code that assigned a value to the variable with name *var_name* in the
693 module *module*.
694
(...)
697 :returns: Tuple with source code segment and corresponding ast node.
698 """
--> 699 source = sourceget.get_module_source(module)
700 return find_in_source(var_name, source, i=i)
File ~/Deeplearning/padl/padl/dumptools/sourceget.py:61, in get_module_source(module, use_replace_cache)
60 pass
---> 61 return inspect.getsource(module)
File /usr/lib/python3.9/inspect.py:1024, in getsource(object)
1019 """Return the text of the source code for an object.
1020
1021 The argument may be a module, class, method, function, traceback, frame,
1022 or code object. The source code is returned as a single string. An
1023 OSError is raised if the source code cannot be retrieved."""
-> 1024 lines, lnum = getsourcelines(object)
1025 return ''.join(lines)
File /usr/lib/python3.9/inspect.py:1006, in getsourcelines(object)
1005 object = unwrap(object)
-> 1006 lines, lnum = findsource(object)
1008 if istraceback(object):
File /usr/lib/python3.9/inspect.py:817, in findsource(object)
810 """Return the entire source file and starting line number for an object.
811
812 The argument may be a module, class, method, function, traceback, frame,
813 or code object. The source code is returned as a list of all the lines
814 in the file and the line number indexes a line in that list. An OSError
815 is raised if the source code cannot be retrieved."""
--> 817 file = getsourcefile(object)
818 if file:
819 # Invalidate cache if needed.
File /usr/lib/python3.9/inspect.py:697, in getsourcefile(object)
694 """Return the filename that can be used to locate an object's source.
695 Return None if no way can be identified to get the source.
696 """
--> 697 filename = getfile(object)
698 all_bytecode_suffixes = importlib.machinery.DEBUG_BYTECODE_SUFFIXES[:]
File /usr/local/lib/python3.9/dist-packages/torch/package/package_importer.py:620, in patched_getfile(object)
619 return _package_imported_modules[object.__module__].__file__
--> 620 return _orig_getfile(object)
File /usr/lib/python3.9/inspect.py:660, in getfile(object)
659 return object.__file__
--> 660 raise TypeError('{!r} is a built-in module'.format(object))
661 if isclass(object):
TypeError: <module '__main__'> is a built-in module
During handling of the above exception, another exception occurred:
NameNotFound Traceback (most recent call last)
Input In [2], in <module>
1 from padl import save
----> 2 save(cf.GENERATOR_PIPELINE, 'test.padl', force_overwrite=True)
File ~/Deeplearning/padl/padl/transforms.py:2412, in save(transform, path, force_overwrite, compress)
2410 transform.pd_zip_save(path, force_overwrite)
2411 else:
-> 2412 transform.pd_save(path, force_overwrite)
File ~/Deeplearning/padl/padl/transforms.py:382, in Transform.pd_save(self, path, force_overwrite)
380 for i, subtrans in enumerate(self._pd_all_transforms()):
381 subtrans.pd_pre_save(path, i)
--> 382 code, versions = self._pd_dumps(True, path=path)
384 with open(path / 'transform.py', 'w') as f:
385 f.write(code)
File ~/Deeplearning/padl/padl/transforms.py:586, in Transform._pd_dumps(self, return_versions, path)
578 def _pd_dumps(self, return_versions: bool = False,
579 path: Optional[Path] = None) -> Union[str, Tuple[str, str]]:
580 """Dump the transform as python code.
581
582 :param return_versions: If *True* return a tuple of the code and a file listing
583 dependencies and their versions.
584 :param path: Optional path to save at, might be required for serializer code snippets.
585 """
--> 586 graph = self._pd_build_codegraph(name='_pd_main')
587 Serializer.save_all(graph, path)
588 code = graph.dumps()
File ~/Deeplearning/padl/padl/transforms.py:510, in Transform._pd_build_codegraph(self, graph, name)
507 continue
509 # find how *next_name* came into being
--> 510 next_codenode = find_codenode(next_name)
511 graph[next_name] = next_codenode
513 todo.update(next_codenode.globals_)
File ~/Deeplearning/padl/padl/dumptools/var2mod.py:548, in find_codenode(name)
546 def find_codenode(name: ScopedName):
547 """Find the :class:`CodeNode` corresponding to a :class:`ScopedName` *name*. """
--> 548 (source, node), scope_of_next_var = find_in_scope(name)
550 # find dependencies
551 globals_ = find_globals(node)
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:636, in find_in_scope(name)
634 if scope.module is None:
635 raise NameNotFound(f'{name.name} not found in function hierarchy.')
--> 636 source, node = find(name.name, scope.module, i)
637 if getattr(node, '_globalscope', False):
638 scope = Scope.empty()
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:772, in find(var_name, module, i)
770 if module is not sys.modules['__main__']:
771 raise NameNotFound(f'"{var_name}" not found.') from exc
--> 772 return find_in_ipython(var_name, i)
File ~/Deeplearning/padl/padl/dumptools/symfinder.py:750, in find_in_ipython(var_name, i)
748 break
749 if source is None:
--> 750 raise NameNotFound(f'"{var_name}" not found.')
751 return source, node
NameNotFound: "projection" not found.
🐞 Bug
python3.9 and python3.8
Bug disappears when the classes/ functions in
importfileare moved tomain.main.py
importfile
ERROR:
(often different each time)
OR: