When training the Fantasia3D texture phase with multiple GPUs, self.light.diffuse and self.light.specular constructed in funcion build_mips() are always on cuda:0, causing the RuntimeError: texture_fwd_mip(): Inputs tex, uv must reside on the same GPU device.
I attempted to manually move them to their respective devices, but doing so resulted in a failure to initialize RasterizeCudaContext, with the following RuntimeError: Cuda error: 700[cudaFuncGetAttributes(&attr, (void*)fineRasterKernel);]"
Traceback (most recent call last):
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/launch.py", line 237, in
main(args, extras)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/launch.py", line 180, in main
trainer.fit(system, datamodule=dm, ckpt_path=cfg.resume)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 546, in fit
call._call_and_handle_interrupt(
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 43, in _call_and_handle_interrupt
return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 102, in launch
return function(*args, *kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 582, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 991, in _run
results = self._run_stage()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1037, in _run_stage
self.fit_loop.run()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 202, in run
self.advance()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 359, in advance
self.epoch_loop.run(self._data_fetcher)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 136, in run
self.advance(data_fetcher)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 240, in advance
batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 187, in run
self._optimizer_step(batch_idx, closure)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 265, in _optimizer_step
call._call_lightning_module_hook(
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 157, in _call_lightning_module_hook
output = fn(args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1282, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 151, in step
step_output = self._strategy.optimizer_step(self._optimizer, closure, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 263, in optimizer_step
optimizer_output = super().optimizer_step(optimizer, closure, model, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 230, in optimizer_step
return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 117, in optimizer_step
return optimizer.step(closure=closure, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/optim/optimizer.py", line 140, in wrapper
out = func(*args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, *kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/optim/adamw.py", line 120, in step
loss = closure()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 104, in _wrap_closure
closure_result = closure()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in call
self._result = self.closure(args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 126, in closure
step_output = self._step_fn()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 315, in _training_step
training_step_output = call._call_strategy_hook(trainer, "training_step", kwargs.values())
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 309, in _call_strategy_hook
output = fn(args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 381, in training_step
return self._forward_redirection(self.model, self.lightning_module, "training_step", *args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 628, in call
wrapper_output = wrapper_module(*args, *kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1040, in forward
output = self._run_ddp_forward(*inputs, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1000, in _run_ddp_forward
return module_to_run(*inputs[0], *kwargs[0])
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 621, in wrapped_forward
out = method(*_args, _kwargs)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/threestudio/systems/fantasia3d.py", line 47, in training_step
out = self(batch)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, kwargs)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/threestudio/systems/fantasia3d.py", line 26, in forward
render_out = self.renderer(batch, render_rgb=self.cfg.texture)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, kwargs)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/threestudio/models/renderers/nvdiff_rasterizer.py", line 95, in forward
rgb_fg = self.material(
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(*input, **kwargs)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/threestudio/models/materials/pbr_material.py", line 103, in forward
diffuse_light = self.light(shading_normal)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/envlight/light.py", line 87, in call
light = dr.texture(self.diffuse[None, ...], l, filter_mode='linear', boundary_mode='cube')
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/nvdiffrast/torch/ops.py", line 615, in texture
return _texture_func.apply(filter_mode, tex, uv, filter_mode_enum, boundary_mode_enum)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/nvdiffrast/torch/ops.py", line 504, in forward
out = _get_plugin().texture_fwd(tex, uv, filter_mode_enum, boundary_mode_enum)
RuntimeError: texture_fwd_mip(): Inputs tex, uv must reside on the same GPU device
When training the Fantasia3D texture phase with multiple GPUs, self.light.diffuse and self.light.specular constructed in funcion build_mips() are always on cuda:0, causing the RuntimeError: texture_fwd_mip(): Inputs tex, uv must reside on the same GPU device.
I attempted to manually move them to their respective devices, but doing so resulted in a failure to initialize RasterizeCudaContext, with the following RuntimeError: Cuda error: 700[cudaFuncGetAttributes(&attr, (void*)fineRasterKernel);]"
Traceback (most recent call last): File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/launch.py", line 237, in
main(args, extras)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/launch.py", line 180, in main
trainer.fit(system, datamodule=dm, ckpt_path=cfg.resume)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 546, in fit
call._call_and_handle_interrupt(
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 43, in _call_and_handle_interrupt
return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 102, in launch
return function(*args, *kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 582, in _fit_impl
self._run(model, ckpt_path=ckpt_path)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 991, in _run
results = self._run_stage()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/trainer.py", line 1037, in _run_stage
self.fit_loop.run()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 202, in run
self.advance()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/fit_loop.py", line 359, in advance
self.epoch_loop.run(self._data_fetcher)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 136, in run
self.advance(data_fetcher)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/training_epoch_loop.py", line 240, in advance
batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 187, in run
self._optimizer_step(batch_idx, closure)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 265, in _optimizer_step
call._call_lightning_module_hook(
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 157, in _call_lightning_module_hook
output = fn(args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/core/module.py", line 1282, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/core/optimizer.py", line 151, in step
step_output = self._strategy.optimizer_step(self._optimizer, closure, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/ddp.py", line 263, in optimizer_step
optimizer_output = super().optimizer_step(optimizer, closure, model, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 230, in optimizer_step
return self.precision_plugin.optimizer_step(optimizer, model=model, closure=closure, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 117, in optimizer_step
return optimizer.step(closure=closure, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/optim/optimizer.py", line 140, in wrapper
out = func(*args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, *kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/optim/adamw.py", line 120, in step
loss = closure()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py", line 104, in _wrap_closure
closure_result = closure()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 140, in call
self._result = self.closure(args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 126, in closure
step_output = self._step_fn()
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/loops/optimization/automatic.py", line 315, in _training_step
training_step_output = call._call_strategy_hook(trainer, "training_step", kwargs.values())
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/trainer/call.py", line 309, in _call_strategy_hook
output = fn(args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 381, in training_step
return self._forward_redirection(self.model, self.lightning_module, "training_step", *args, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 628, in call
wrapper_output = wrapper_module(*args, *kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1040, in forward
output = self._run_ddp_forward(*inputs, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1000, in _run_ddp_forward
return module_to_run(*inputs[0], *kwargs[0])
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, kwargs)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/pytorch_lightning/strategies/strategy.py", line 621, in wrapped_forward
out = method(*_args, _kwargs)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/threestudio/systems/fantasia3d.py", line 47, in training_step
out = self(batch)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, kwargs)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/threestudio/systems/fantasia3d.py", line 26, in forward
render_out = self.renderer(batch, render_rgb=self.cfg.texture)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(input, kwargs)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/threestudio/models/renderers/nvdiff_rasterizer.py", line 95, in forward
rgb_fg = self.material(
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(*input, **kwargs)
File "/home/zyq/Disk2T/users/zyq/refcode/threestudio/threestudio/models/materials/pbr_material.py", line 103, in forward
diffuse_light = self.light(shading_normal)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/envlight/light.py", line 87, in call
light = dr.texture(self.diffuse[None, ...], l, filter_mode='linear', boundary_mode='cube')
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/nvdiffrast/torch/ops.py", line 615, in texture
return _texture_func.apply(filter_mode, tex, uv, filter_mode_enum, boundary_mode_enum)
File "/home/zyq/anaconda3/envs/texture/lib/python3.10/site-packages/nvdiffrast/torch/ops.py", line 504, in forward
out = _get_plugin().texture_fwd(tex, uv, filter_mode_enum, boundary_mode_enum)
RuntimeError: texture_fwd_mip(): Inputs tex, uv must reside on the same GPU device