494 K Trainable params
1.6 K Non-trainable params
496 K Total params
1.985 Total estimated model params size (MB)
Sanity Checking: 0it [00:00, ?it/s]Traceback (most recent call last):
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/script_train_vectornet.py", line 86, in
run(config_parser.GlobalConfig.load(steps.get_config_path()))
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/script_train_vectornet.py", line 77, in run
trainer.fit(
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 700, in fit
self._call_and_handle_interrupt(
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 654, in _call_and_handle_interrupt
return trainer_fn(*args, kwargs)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 741, in _fit_impl
results = self._run(model, ckpt_path=self.ckpt_path)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1166, in _run
results = self._run_stage()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1252, in _run_stage
return self._run_train()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1274, in _run_train
self._run_sanity_check()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1342, in _run_sanity_check
val_loop.run()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
self.advance(*args, *kwargs)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 155, in advance
dl_outputs = self.epoch_loop.run(self._data_fetcher, dl_max_batches, kwargs)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
self.advance(args, kwargs)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 127, in advance
batch = next(data_fetcher)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 185, in next
return self.fetching_function()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 264, in fetching_function
self._fetch_next_batch(self.dataloader_iter)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 278, in _fetch_next_batch
batch = next(iterator)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 530, in next
data = self._next_data()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1224, in _next_data
return self._process_data(data)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1250, in _process_data
data.reraise()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/_utils.py", line 457, in reraise
raise exception
FileNotFoundError: Caught FileNotFoundError in DataLoader worker process 0.
Original Traceback (most recent call last):
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
data = fetcher.fetch(index)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/datasets/vectornet_dataset.py", line 50, in getitem
data = self._dataset[index]
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/datasets/vectornet_dataset.py", line 32, in getitem
return GraphScenarioData.load(self.scenario_paths[index])
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/datasets/graph_scenario.py", line 93, in load
catalog[object_name] = np.load(object_path)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/numpy/lib/npyio.py", line 407, in load
fid = stack.enter_context(open(os_fspath(file), "rb"))
FileNotFoundError: [Errno 2] No such file or directory: '/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/internal_graph/val/MIA_13379/anchors.npy'
/home/sti/anaconda3/envs/TnH/bin/python /home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/script_train_vectornet.py --cfg configs/vectornet.yaml 2023-09-24 21:07:54 [TrainVectornet]: GraphTrainConfig(train_input_path='internal_graph/train', val_input_path='internal_graph/val', model_name='vectornet-v2-50', parameters=GraphTrainConfigParameters(epochs=50, batch_size=64, n_targets=6, n_trajectories=6, tg_lr=0.0001, tg_sched_step=15, tg_sched_gamma=0.1, tf_lr=0.0001, tf_sched_step=15, tf_sched_gamma=0.1, tfs_lr=0.0001, tfs_sched_step=15, tfs_sched_gamma=0.1, huber_delta=0.2, use_traj_scoring=False), visualize=False, n_workers=8, resume=True, starting_checkpoint_name='last.ckpt') 2023-09-24 21:07:54 [TrainVectornet]: Global path: "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/" 2023-09-24 21:07:54 [TrainVectornet]: Train dataset path: "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/internal_graph/train". 2023-09-24 21:07:54 [TrainVectornet]: Validation dataset path: "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/internal_graph/val". 2023-09-24 21:07:54 [TrainVectornet]: Model storage path: "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/model_storage/vectornet/vectornet-v2-50". 2023-09-24 21:07:54 [torch.distributed.nn.jit.instantiator]: Created a temporary directory at /tmp/tmpzouo5zoy 2023-09-24 21:07:54 [torch.distributed.nn.jit.instantiator]: Writing /tmp/tmpzouo5zoy/_remote_module_non_sriptable.py /home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:445: LightningDeprecationWarning: Setting
Trainer(gpus=1)
is deprecated in v1.7 and will be removed in v2.0. Please useTrainer(accelerator='gpu', devices=1)
instead. rank_zero_deprecation( GPU available: True (cuda), used: True TPU available: False, using: 0 TPU cores IPU available: False, using: 0 IPUs HPU available: False, using: 0 HPUs /home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:616: UserWarning: Checkpoint directory /home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/model_storage/vectornet/vectornet-v2-50 exists and is not empty. rank_zero_warn(f"Checkpoint directory {dirpath} exists and is not empty.") LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]| Name | Type | Params
0 | _target_generator | TargetGenerator | 413 K 1 | _trajectory_forecaster | TrajectoryForecaster | 41.3 K 2 | _trajectory_scorer | TrajectoryScorer | 41.0 K 3 | _loss | LiteTNTLoss | 0
494 K Trainable params 1.6 K Non-trainable params 496 K Total params 1.985 Total estimated model params size (MB) Sanity Checking: 0it [00:00, ?it/s]Traceback (most recent call last): File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/script_train_vectornet.py", line 86, in
run(config_parser.GlobalConfig.load(steps.get_config_path()))
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/script_train_vectornet.py", line 77, in run
trainer.fit(
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 700, in fit
self._call_and_handle_interrupt(
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 654, in _call_and_handle_interrupt
return trainer_fn(*args, kwargs)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 741, in _fit_impl
results = self._run(model, ckpt_path=self.ckpt_path)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1166, in _run
results = self._run_stage()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1252, in _run_stage
return self._run_train()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1274, in _run_train
self._run_sanity_check()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1342, in _run_sanity_check
val_loop.run()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
self.advance(*args, *kwargs)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py", line 155, in advance
dl_outputs = self.epoch_loop.run(self._data_fetcher, dl_max_batches, kwargs)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py", line 200, in run
self.advance(args, kwargs)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/evaluation_epoch_loop.py", line 127, in advance
batch = next(data_fetcher)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 185, in next
return self.fetching_function()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 264, in fetching_function
self._fetch_next_batch(self.dataloader_iter)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/pytorch_lightning/utilities/fetching.py", line 278, in _fetch_next_batch
batch = next(iterator)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 530, in next
data = self._next_data()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1224, in _next_data
return self._process_data(data)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1250, in _process_data
data.reraise()
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/_utils.py", line 457, in reraise
raise exception
FileNotFoundError: Caught FileNotFoundError in DataLoader worker process 0.
Original Traceback (most recent call last):
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
data = fetcher.fetch(index)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 49, in
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/datasets/vectornet_dataset.py", line 50, in getitem
data = self._dataset[index]
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/datasets/vectornet_dataset.py", line 32, in getitem
return GraphScenarioData.load(self.scenario_paths[index])
File "/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/src/vectornet/datasets/graph_scenario.py", line 93, in load
catalog[object_name] = np.load(object_path)
File "/home/sti/anaconda3/envs/TnH/lib/python3.8/site-packages/numpy/lib/npyio.py", line 407, in load
fid = stack.enter_context(open(os_fspath(file), "rb"))
FileNotFoundError: [Errno 2] No such file or directory: '/home/sti/hdd_ldz/py_proj/TNT-VectorNet-and-HOME-Trajectory-Forecasting/internal_graph/val/MIA_13379/anchors.npy'
Process finished with exit code 1