Summary:
Patch prevents error ImportError: tokenizers>=0.13.3 is required for a normal functioning of this module, but found tokenizers==0.12.1
***/mmf/__init__.py in <module>
4 from mmf.utils.patch import patch_transformers
5
----> 6 patch_transformers()
7
8 from mmf import common, datasets, models, modules, utils
***/mmf/utils/patch.py in patch_transformers(log_incompatible)
65 if not module or module == "." or module[0] == ".":
66 continue
---> 67 sys.modules[f"transformers.{module}"] = importlib.import_module(
68 f"transformers.models.{key}.{module}"
69 )
***/runtime/lib/python3.8/importlib/__init__.py in import_module(name, package)
137 break
138 level += 1
--> 139 return _bootstrap._gcd_import(name[level:], package, level)
140
141
***/libfb/py/import_proxy.py in wrapper(module)
59 wraps(_exec_module)
60 def wrapper(module: ModuleType) -> None:
---> 61 _exec_module(module)
62 _fire_callbacks(module)
63
***/python/parsh/autoreload/measurements.py in patched_exec_module(self_, module)
51 start = timer()
52 try:
---> 53 orig_exec_module(self_, module)
54 finally:
55 _IS_TOP_LEVEL_IMPORT = is_top_level_import
***/transformers/models/llama/tokenization_llama_fast.py in <module>
22
23
---> 24 require_version("tokenizers>=0.13.3")
25
26 if is_sentencepiece_available():
***/transformers/utils/versions.py in require_version(requirement, hint)
122 if want_ver is not None:
123 for op, want_ver in wanted.items():
--> 124 _compare_versions(op, got_ver, want_ver, requirement, pkg, hint)
125
126
***/transformers/utils/versions.py in _compare_versions(op, got_ver, want_ver, requirement, pkg, hint)
48 )
49 if not ops[op](version.parse(got_ver), version.parse(want_ver)):
---> 50 raise ImportError(
51 f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}"
52 )
ImportError: tokenizers>=0.13.3 is required for a normal functioning of this module, but found tokenizers==0.12.1.
Summary: Patch prevents error
ImportError: tokenizers>=0.13.3 is required for a normal functioning of this module, but found tokenizers==0.12.1
Reviewed By: RylanC24, BruceChaun
Differential Revision: D45878336