When looping through Wanda_sweeps there seems to be an error related to the validation loader.
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.6/dist-packages/wandb/wandb_agent.py", line 64, in _start
function()
File "<ipython-input-14-43e09d654e98>", line 7, in <lambda>
wandb.agent(sweep_id, lambda: train_function("PyTorch", make_config_file(file_path2, len(region), weight_path=None)))
File "/content/github_aistream-peelout_flow-forecast/flood_forecast/trainer.py", line 33, in train_function
train_transformer_style(trained_model, params["training_params"], params["forward_params"])
File "/content/github_aistream-peelout_flow-forecast/flood_forecast/pytorch_training.py", line 65, in train_transformer_style
test = compute_validation(test_data_loader, model.model, epoch, model.params["dataset_params"]["forecast_length"], criterion, model.device, decoder_structure=True, use_wandb=use_wandb, val_or_test="test_loss")
File "/content/github_aistream-peelout_flow-forecast/flood_forecast/pytorch_training.py", line 108, in compute_validation
for src, targ in validation_loader:
File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 345, in __next__
data = self._next_data()
File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 384, in _next_data
index = self._next_index() # may raise StopIteration
File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/dataloader.py", line 339, in _next_index
return next(self._sampler_iter) # may raise StopIteration
File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/sampler.py", line 200, in __iter__
for idx in self.sampler:
File "/usr/local/lib/python3.6/dist-packages/torch/utils/data/sampler.py", line 62, in __iter__
return iter(range(len(self.data_source)))
ValueError: __len__() should return >= 0
When looping through Wanda_sweeps there seems to be an error related to the validation loader.