Closed soumickmj closed 1 year ago
Please provide minimum runnable (or, better, minified) example https://github.com/pytorch/pytorch/blob/c58264c3e9d8ce070d45cb650c3cd906acc7ef6a/docs/source/compile/faq.rst#torchinductor-errors
Please file issues at pytorch/pytorch as template suggests.
I'm using PyTorch and PyTorch lightning in my code. I'm getting the following error:
You are using a CUDA device ('NVIDIA A40-48Q') that has Tensor Cores. To properly utilize them, you should set
main()
File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Bridge/prova_recon.py", line 144, in main
helm(sys_params=sys_params)
File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Bridge/prova_recon.py", line 131, in helm
engine.engage()
File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Engineering/Engines/MainEngine.py", line 366, in engage
self.train()
File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Engineering/Engines/MainEngine.py", line 343, in train
self.trainer.fit(model=self.model,
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 538, in fit
call._call_and_handle_interrupt(
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 42, in _call_and_handle_interrupt
return trainer_fn(*args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 577, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 962, in _run
call._call_callback_hooks(self, "on_fit_start")
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 189, in _call_callback_hooks
fn(trainer, trainer.lightning_module, *args, *kwargs)
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/callbacks/model_summary.py", line 60, in on_fit_start
model_summary = self._summary(trainer, pl_module)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/callbacks/model_summary.py", line 74, in _summary
return summarize(pl_module, max_depth=self._max_depth)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/utilities/model_summary/model_summary.py", line 454, in summarize
return ModelSummary(lightning_module, max_depth=max_depth)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/utilities/model_summary/model_summary.py", line 193, in init
self._layer_summary = self.summarize()
^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/utilities/model_summary/model_summary.py", line 254, in summarize
self._forward_example_input()
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/utilities/model_summary/model_summary.py", line 286, in _forward_exampleinput
model(input)
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1502, in _wrapped_call_impl
return self._call_impl(args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _call_impl
return forward_call(*args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Engineering/Engines/AuxiliaryEngines/ReconEngine.py", line 177, in forward
return self.net(x)
^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1502, in _wrapped_call_impl
return self._call_impl(*args, *kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1548, in _call_impl
result = forward_call(args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 286, in _fn
return fn(*args, kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1502, in _wrapped_call_impl
return self._call_impl(*args, *kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _call_impl
return forward_call(args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Engineering/Engines/WarpDrives/pythaeDrive/pythaeStation.py", line 191, in forward
def forward(self, x, return_only_recon=True):
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1502, in _wrapped_call_impl
return self._call_impl(*args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1511, in _call_impl
return forward_call(args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Engineering/Engines/WarpDrives/pythaeDrive/wrappers/wrapped_models.py", line 34, in forward
def forward(self, inputs: BaseDataset, kwargs) -> ModelOutput:
File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Engineering/Engines/WarpDrives/pythaeDrive/wrappers/wrapped_models.py", line 93, in
recon_loss, autoencoder_loss, discriminator_loss = self.loss_function(
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 439, in catch_errors
return callback(frame, cache_size, hooks, frame_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py", line 522, in _convert_frame
result = inner_convert(frame, cache_size, hooks, frame_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py", line 125, in _fn
return fn( args, kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py", line 358, in _convert_frame_assert
return _compile(
^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/utils.py", line 177, in time_wrapper
r = func(*args, kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py", line 428, in _compile
out_code = transform_code_object(code, transform)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/bytecode_transformation.py", line 1000, in transform_code_object
transformations(instructions, code_options)
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/convert_frame.py", line 413, in transform
tracer.run()
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py", line 2009, in run
super().run()
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py", line 703, in run
and self.step()
^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py", line 663, in step
getattr(self, inst.opname)(inst)
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/symbolic_convert.py", line 2097, in RETURN_VALUE
self.output.compile_subgraph(
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/output_graph.py", line 752, in compile_subgraph
self.compile_and_call_fx_graph(tx, pass2.graph_output_vars(), root)
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/contextlib.py", line 81, in inner
return func(*args, *kwds)
^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/output_graph.py", line 829, in compile_and_call_fx_graph
compiled_fn = self.call_user_compiler(gm)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/utils.py", line 177, in time_wrapper
r = func(args, kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/output_graph.py", line 888, in call_user_compiler
raise BackendCompilerFailed(self.compiler_fn, e).with_traceback(
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/output_graph.py", line 884, in call_user_compiler
compiled_fn = compiler_fn(gm, self.example_inputs())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/repro/after_dynamo.py", line 117, in debug_wrapper
compiled_gm = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/init.py", line 1538, in call
return compilefx(model, inputs_, config_patches=self.config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 610, in compile_fx
return compile_fx(
^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 720, in compile_fx
return aot_autograd(
^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/backends/common.py", line 55, in compiler_fn
cg = aot_module_simplified(gm, example_inputs, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 3686, in aot_module_simplified
compiled_fn = create_aot_dispatcher_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/utils.py", line 177, in time_wrapper
r = func(*args, *kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 3225, in create_aot_dispatcher_function
compiled_fn = compiler_fn(flat_fn, fake_flat_args, aot_config, fw_metadata=fw_metadata)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 2090, in aot_wrapper_dedupe
return compiler_fn(flat_fn, leaf_flat_args, aot_config, fw_metadata=fw_metadata)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 2270, in aot_wrapper_synthetic_base
return compiler_fn(flat_fn, flat_args, aot_config, fw_metadata=fw_metadata)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1532, in aot_dispatch_base
compiled_fw = compiler(fw_module, adjusted_flat_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/utils.py", line 177, in time_wrapper
r = func(args, kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 676, in fw_compiler_base
return inner_compile(
^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/contextlib.py", line 81, in inner
return func(*args, kwds)
^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/repro/after_aot.py", line 80, in debug_wrapper
inner_compiled_fn = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/debug.py", line 220, in inner
return fn(*args, *kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/contextlib.py", line 81, in inner
return func(args, kwds)
^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 45, in newFunction
return old_func(*args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/compile_fx.py", line 279, in compile_fx_inner
graph.run(example_inputs)
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_dynamo/utils.py", line 177, in time_wrapper
r = func(args, kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/graph.py", line 268, in run
return super().run(args)
^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/fx/interpreter.py", line 138, in run
self.env[node] = self.run_node(node)
^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/graph.py", line 509, in run_node
result = super().run_node(n)
^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/fx/interpreter.py", line 195, in run_node
return getattr(self, n.op)(n.target, args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/graph.py", line 412, in call_function
raise LoweringException(e, target, args, kwargs).with_traceback(
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/graph.py", line 409, in call_function
out = lowerings[target](args, *kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/lowering.py", line 227, in wrapped
out = decomp_fn(args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/lowering.py", line 3635, in mean
denom = sympy_product(size[i] for i in axis)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/utils.py", line 105, in sympy_product
return functools.reduce(operator.mul, it, sympy.Integer(1))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/lowering.py", line 3635, in
denom = sympy_product(size[i] for i in axis)
torch.set_float32_matmul_precision('medium' | 'high')
which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision /home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:622: UserWarning: Checkpoint directory /project/ukbblatent/Out/toysets/Results/provaTime2Ch_provaV2_fp32fold0_prec32_pythaemodel-factor_vae/Checkpoints exists and is not empty. rank_zero_warn(f"Checkpoint directory {dirpath} exists and is not empty.") LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] [2023-05-30 10:06:03,192] torch._inductor.utils: [WARNING] using triton random, expect difference from eager [2023-05-30 10:06:03,894] torch._inductor.utils: [WARNING] using triton random, expect difference from eager /home/soumick.chatterjee/anaconda3/envs/torchHTBeta2V2/lib/python3.11/site-packages/torch/_inductor/compile_fx.py:123: UserWarning: TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled. Consider settingtorch.set_float32_matmul_precision('high')
for better performance. warnings.warn( [2023-05-30 10:06:07,591] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,595] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,600] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,605] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,609] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,614] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,618] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,622] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,626] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,631] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,635] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,639] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,643] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,648] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,652] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,656] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,660] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,664] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,668] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,672] torch._inductor.utils: [WARNING] DeviceCopy in input program [2023-05-30 10:06:07,803] torch._inductor.utils: [WARNING] skipping cudagraphs due to multiple devices Traceback (most recent call last): File "/home/soumick.chatterjee/Codes/GitLab/UKBBLatent/Bridge/prova_recon.py", line 147, in