gaotianyu1350
commented
2 years ago Hi, this seems to be a pytorch related error (not related to this repo).
Closed lisan97 closed 2 years ago
gaotianyu1350
commented
2 years ago Hi, this seems to be a pytorch related error (not related to this repo).
transformers==4.2.1 datasets==1.2.1 python3.6 when I set preprocessing_num_workers > 1，the error will happen
Traceback (most recent call last): File "train.py", line 624, in <module> main() File "train.py", line 489, in main load_from_cache_file=not data_args.overwrite_cache, File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/datasets/arrow_dataset.py", line 1318, in map transformed_shards = [r.get() for r in results] File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/datasets/arrow_dataset.py", line 1318, in <listcomp> transformed_shards = [r.get() for r in results] File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/multiprocess/pool.py", line 644, in get raise self._value File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/multiprocess/pool.py", line 424, in _handle_tasks put(task) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/multiprocess/connection.py", line 209, in send self._send_bytes(_ForkingPickler.dumps(obj)) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/multiprocess/reduction.py", line 54, in dumps cls(buf, protocol, *args, **kwds).dump(obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/dill/_dill.py", line 498, in dump StockPickler.dump(self, obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 409, in dump self.save(obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 751, in save_tuple save(element) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/dill/_dill.py", line 990, in save_module_dict StockPickler.save_dict(pickler, obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 821, in save_dict self._batch_setitems(obj.items()) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 847, in _batch_setitems save(v) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/dill/_dill.py", line 1496, in save_function obj.__dict__, fkwdefaults), obj=obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 610, in save_reduce save(args) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 751, in save_tuple save(element) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 751, in save_tuple save(element) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/dill/_dill.py", line 1227, in save_cell pickler.save_reduce(_create_cell, (f,), obj=obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 610, in save_reduce save(args) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 736, in save_tuple save(element) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 521, in save self.save_reduce(obj=obj, *rv) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 605, in save_reduce save(cls) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/dill/_dill.py", line 1423, in save_type obj.__bases__, _dict), obj=obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 610, in save_reduce save(args) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 751, in save_tuple save(element) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/dill/_dill.py", line 990, in save_module_dict StockPickler.save_dict(pickler, obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 821, in save_dict self._batch_setitems(obj.items()) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 847, in _batch_setitems save(v) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 476, in save f(self, obj) # Call unbound method with explicit self File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/dill/_dill.py", line 990, in save_module_dict StockPickler.save_dict(pickler, obj) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 821, in save_dict self._batch_setitems(obj.items()) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 847, in _batch_setitems save(v) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 507, in save self.save_global(obj, rv) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/pickle.py", line 927, in save_global (obj, module_name, name)) _pickle.PicklingError: Can't pickle typing.Union[str, NoneType]: it's not the same object as typing.Union Traceback (most recent call last): File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/runpy.py", line 193, in _run_module_as_main "__main__", mod_spec) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/runpy.py", line 85, in _run_code exec(code, run_globals) File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/torch/distributed/launch.py", line 260, in <module> main() File "/home/hdp-portrait/miniconda3/envs/hug_transformer/lib/python3.6/site-packages/torch/distributed/launch.py", line 256, in main cmd=cmd)