Closed Tian14267 closed 1 year ago
Traceback (most recent call last):
File "finetune_moss_2.py", line 310, in <module>
train(args)
File "finetune_moss_2.py", line 215, in train
model, optimizer, train_dataloader, val_dataloader, lr_scheduler = accelerator.prepare(model, optimizer, train_dataloader, val_dataloader, lr_scheduler)
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/accelerate/accelerator.py", line 1139, in prepare
result = self._prepare_deepspeed(*args)
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/accelerate/accelerator.py", line 1446, in _prepare_deepspeed
engine, optimizer, _, lr_scheduler = deepspeed.initialize(**kwargs)
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/deepspeed/__init__.py", line 165, in initialize
engine = DeepSpeedEngine(args=args,
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/deepspeed/runtime/engine.py", line 308, in __init__
self._configure_optimizer(optimizer, model_parameters)
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/deepspeed/runtime/engine.py", line 1173, in _configure_optimizer
self.optimizer = self._configure_zero_optimizer(basic_optimizer)
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/deepspeed/runtime/engine.py", line 1463, in _configure_zero_optimizer
optimizer = DeepSpeedZeroOptimizer_Stage3(
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/deepspeed/runtime/zero/stage3.py", line 130, in __init__
util_ops = UtilsBuilder().load()
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/deepspeed/ops/op_builder/builder.py", line 445, in load
return self.jit_load(verbose)
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/deepspeed/ops/op_builder/builder.py", line 480, in jit_load
op_module = load(name=self.name,
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1284, in load
return _jit_compile(
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1508, in _jit_compile
_write_ninja_file_and_build_library(
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1597, in _write_ninja_file_and_build_library
get_compiler_abi_compatibility_and_version(compiler)
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 336, in get_compiler_abi_compatibility_and_version
if not check_compiler_ok_for_platform(compiler):
File "/root/anaconda3/envs/mt5/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 290, in check_compiler_ok_for_platform
which = subprocess.check_output(['which', compiler], stderr=subprocess.STDOUT)
File "/root/anaconda3/envs/mt5/lib/python3.8/subprocess.py", line 415, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
File "/root/anaconda3/envs/mt5/lib/python3.8/subprocess.py", line 493, in run
with Popen(*popenargs, **kwargs) as process:
File "/root/anaconda3/envs/mt5/lib/python3.8/subprocess.py", line 858, in __init__
self._execute_child(args, executable, preexec_fn, close_fds,
File "/root/anaconda3/envs/mt5/lib/python3.8/subprocess.py", line 1704, in _execute_child
raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: 'which'
这个也是~~~
大神们好,我在finetune训练的时候,又遇到下面这个问题:
详细情况:
请问这是啥问题啊