Open nangongliufeng opened 1 year ago
Fixed in the latest commit. Could you try again?
Oh wow, it is the other way out for my scenario, need to adjust the variables to double from float. Haven’t checked in details for such error, anyone have insight?
Fixed in the latest commit. Could you try again? Thank you, it has been solved!
Thanks for this very good work, how can this problem be solved? File "launch.py", line 125, in
main()
File "launch.py", line 114, in main
trainer.fit(system, datamodule=dm)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 608, in fit
call._call_and_handle_interrupt(
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/trainer/call.py", line 36, in _call_and_handle_interrupt
return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 88, in launch
return function(*args, *kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 650, in _fit_impl
self._run(model, ckpt_path=self.ckpt_path)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1112, in _run
results = self._run_stage()
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1191, in _run_stage
self._run_train()
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1214, in _run_train
self.fit_loop.run()
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(args, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/fit_loop.py", line 267, in advance
self._outputs = self.epoch_loop.run(self._data_fetcher)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(*args, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 213, in advance
batch_output = self.batch_loop.run(kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(*args, *kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
outputs = self.optimizer_loop.run(optimizers, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 199, in run
self.advance(args, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 202, in advance
result = self._run_optimization(kwargs, self._optimizers[self.optim_progress.optimizer_position])
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 249, in _run_optimization
self._optimizer_step(optimizer, opt_idx, kwargs.get("batch_idx", 0), closure)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 370, in _optimizer_step
self.trainer._call_lightning_module_hook(
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1356, in _call_lightning_module_hook
output = fn(*args, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/core/module.py", line 1754, in optimizer_step
optimizer.step(closure=optimizer_closure)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py", line 169, in step
step_output = self._strategy.optimizer_step(self._optimizer, self._optimizer_idx, closure, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/strategies/ddp.py", line 280, in optimizer_step
optimizer_output = super().optimizer_step(optimizer, opt_idx, closure, model, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/strategies/strategy.py", line 234, in optimizer_step
return self.precision_plugin.optimizer_step(
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/plugins/precision/native_amp.py", line 75, in optimizer_step
closure_result = closure()
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 149, in call
self._result = self.closure(*args, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 135, in closure
step_output = self._step_fn()
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 419, in _training_step
training_step_output = self.trainer._call_strategy_hook("training_step", kwargs.values())
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1494, in _call_strategy_hook
output = fn(args, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/strategies/ddp.py", line 351, in training_step
return self.model(*args, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(*input, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 1008, in forward
output = self._run_ddp_forward(*inputs, *kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/torch/nn/parallel/distributed.py", line 969, in _run_ddp_forward
return module_to_run(inputs[0], kwargs[0])
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(*input, kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/pytorch_lightning/overrides/base.py", line 98, in forward
output = self._forward_module.training_step(*inputs, *kwargs)
File "/home/vge/code/instant-nsr-pl/systems/neus.py", line 89, in training_step
out = self(batch)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forward_call(input, kwargs)
File "/home/vge/code/instant-nsr-pl/systems/neus.py", line 32, in forward
return self.model(batch['rays'])
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1130, in _call_impl
return forwardcall(*input, **kwargs)
File "/home/vge/code/instant-nsr-pl/models/neus.py", line 284, in forward
out = self.forward(rays)
File "/home/vge/code/instant-nsr-pl/models/neus.py", line 262, in forward_
out_bg = self.forwardbg(rays)
File "/home/vge/code/instant-nsr-pl/models/neus.py", line 159, in forwardbg
ray_indices, t_starts, t_ends = ray_marching(
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, *kwargs)
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/nerfacc/ray_marching.py", line 177, in ray_marching
packed_info, ray_indices, t_starts, t_ends = _C.ray_marching(
File "/home/vge/anaconda3/envs/nsr/lib/python3.8/site-packages/nerfacc/cuda/init.py", line 13, in call_cuda
return getattr(_C, name)(args, **kwargs)
RuntimeError: expected scalar type Float but found Double
Epoch 0: : 0it [00:02, ?it/s]