i'm trying to train a lora following the tutorial, but every time i press ready, the system says error
my log below
can someone help me figure out how to solve it?
Is there an existing issue for this?
[X] I have searched the existing issues
Reproduction
working as tuttorial
Screenshot
No response
Logs
D:\text-generation-webui\installer_files\env\Lib\site-packages\transformers\training_args.py:1568: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
warnings.warn(
[2024-11-12 17:09:10,119] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect)
D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\runtime\zero\linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.
@autocast_custom_fwd
D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\runtime\zero\linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead.
@autocast_custom_bwd
W1112 17:09:10.499000 1980 torch\distributed\elastic\multiprocessing\redirects.py:28] NOTE: Redirects are currently not supported in Windows or MacOs.
Traceback (most recent call last):
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\gradio\queueing.py", line 566, in process_events
response = await route_utils.call_process_api(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\gradio\route_utils.py", line 261, in call_process_api
output = await app.get_blocks().process_api(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\gradio\blocks.py", line 1786, in process_api
result = await self.call_function(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\gradio\blocks.py", line 1350, in call_function
prediction = await utils.async_iteration(iterator)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\gradio\utils.py", line 583, in async_iteration
return await iterator.__anext__()
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\gradio\utils.py", line 576, in __anext__
return await anyio.to_thread.run_sync(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\anyio\to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\anyio\_backends\_asyncio.py", line 2441, in run_sync_in_worker_thread
return await future
^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\anyio\_backends\_asyncio.py", line 943, in run
result = context.run(func, *args)
^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\gradio\utils.py", line 559, in run_sync_iterator_async
return next(iterator)
^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\gradio\utils.py", line 742, in gen_wrapper
response = next(iterator)
^^^^^^^^^^^^^^
File "D:\text-generation-webui\modules\training.py", line 598, in do_train
trainer = transformers.Trainer(
^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\transformers\utils\deprecation.py", line 165, in wrapped_func
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\transformers\trainer.py", line 585, in __init__
unwrapped_model = self.accelerator.unwrap_model(model)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\accelerate\accelerator.py", line 2633, in unwrap_model
return extract_model_from_parallel(model, keep_fp32_wrapper)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\accelerate\utils\other.py", line 80, in extract_model_from_parallel
from deepspeed import DeepSpeedEngine
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\__init__.py", line 26, in <module>
from . import module_inject
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\module_inject\__init__.py", line 6, in <module>
from .replace_module import replace_transformer_layer, revert_transformer_layer, ReplaceWithTensorSlicing, GroupQuantizer, generic_injection
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\module_inject\replace_module.py", line 607, in <module>
from ..pipe import PipelineModule
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\pipe\__init__.py", line 6, in <module>
from ..runtime.pipe import PipelineModule, LayerSpec, TiedLayerSpec
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\runtime\pipe\__init__.py", line 6, in <module>
from .module import PipelineModule, LayerSpec, TiedLayerSpec
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\runtime\pipe\module.py", line 19, in <module>
from ..activation_checkpointing import checkpointing
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\runtime\activation_checkpointing\checkpointing.py", line 26, in <module>
from deepspeed.runtime.config import DeepSpeedConfig
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\runtime\config.py", line 42, in <module>
from ..elasticity import (
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\elasticity\__init__.py", line 10, in <module>
from .elastic_agent import DSElasticAgent
File "D:\text-generation-webui\installer_files\env\Lib\site-packages\deepspeed\elasticity\elastic_agent.py", line 9, in <module>
from torch.distributed.elastic.agent.server.api import log, _get_socket_with_port
ImportError: cannot import name 'log' from 'torch.distributed.elastic.agent.server.api' (D:\text-generation-webui\installer_files\env\Lib\site-packages\torch\distributed\elastic\agent\server\api.py)
Describe the bug
i'm trying to train a lora following the tutorial, but every time i press ready, the system says error my log below can someone help me figure out how to solve it?
Is there an existing issue for this?
Reproduction
working as tuttorial
Screenshot
No response
Logs
System Info