[04/20 12:23:58 d2.utils.events]: eta: 2:29:30 iter: 19 total_loss: 3.171 loss_ins: 2.299 loss_cate: 0.9552 time: 0.1545 data_time: 0.0114 lr: 9.9905e-06 max_mem: 1153M
[04/20 12:24:01 d2.utils.events]: eta: 2:31:05 iter: 39 total_loss: 2.703 loss_ins: 2.117 loss_cate: 0.5777 time: 0.1474 data_time: 0.0018 lr: 1.998e-05 max_mem: 1153M
[04/20 12:24:04 d2.utils.events]: eta: 2:31:02 iter: 59 total_loss: 2.485 loss_ins: 2.017 loss_cate: 0.4502 time: 0.1449 data_time: 0.0019 lr: 2.997e-05 max_mem: 1153M
[04/20 12:24:07 d2.utils.events]: eta: 2:30:47 iter: 79 total_loss: 2.38 loss_ins: 2.045 loss_cate: 0.3513 time: 0.1429 data_time: 0.0016 lr: 3.9961e-05 max_mem: 1156M
Traceback (most recent call last):
File "/data/project_all/AdelaiDet/tools/train_net.py", line 339, in
launch(
File "/data/pytorchDetect/detectron2/detectron2/engine/launch.py", line 62, in launch
main_func(args)
File "/data/project_all/AdelaiDet/tools/train_net.py", line 333, in main
return trainer.train()
File "/data/project_all/AdelaiDet/tools/train_net.py", line 164, in train
self.train_loop(self.start_iter, self.max_iter)
File "/data/project_all/AdelaiDet/tools/train_net.py", line 153, in train_loop
self.run_step()
File "/data/pytorchDetect/detectron2/detectron2/engine/defaults.py", line 423, in run_step
self._trainer.run_step()
File "/data/pytorchDetect/detectron2/detectron2/engine/train_loop.py", line 222, in run_step
data = next(self._data_loader_iter)
File "/data/pytorchDetect/detectron2/detectron2/data/common.py", line 179, in iter
for d in self.dataset:
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 435, in next
data = self._next_data()
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1085, in _next_data
return self._process_data(data)
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1111, in _process_data
data.reraise()
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/_utils.py", line 428, in reraise
raise self.exc_type(msg)
AssertionError: Caught AssertionError in DataLoader worker process 2.
Original Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 198, in _worker_loop
data = fetcher.fetch(index)
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 44, in
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/data/pytorchDetect/detectron2/detectron2/data/common.py", line 43, in getitem
data = self._map_func(self._dataset[cur_idx])
File "/data/pytorchDetect/detectron2/detectron2/utils/serialize.py", line 23, in call
return self._obj(args, *kwargs)
File "/data/project_all/AdelaiDet/adet/data/dataset_mapper.py", line 182, in call
instances = annotations_to_instances(
File "/data/project_all/AdelaiDet/adet/data/detection_utils.py", line 53, in annotations_to_instances
instance = d2_anno_to_inst(annos, image_size, mask_format)
File "/data/pytorchDetect/detectron2/detectron2/data/detection_utils.py", line 395, in annotations_to_instances
masks.append(polygons_to_bitmask(segm, image_size))
File "/data/pytorchDetect/detectron2/detectron2/structures/masks.py", line 29, in polygons_to_bitmask
assert len(polygons) > 0, "COCOAPI does not support empty polygons"
AssertionError: COCOAPI does not support empty polygons
[04/20 12:23:58 d2.utils.events]: eta: 2:29:30 iter: 19 total_loss: 3.171 loss_ins: 2.299 loss_cate: 0.9552 time: 0.1545 data_time: 0.0114 lr: 9.9905e-06 max_mem: 1153M [04/20 12:24:01 d2.utils.events]: eta: 2:31:05 iter: 39 total_loss: 2.703 loss_ins: 2.117 loss_cate: 0.5777 time: 0.1474 data_time: 0.0018 lr: 1.998e-05 max_mem: 1153M [04/20 12:24:04 d2.utils.events]: eta: 2:31:02 iter: 59 total_loss: 2.485 loss_ins: 2.017 loss_cate: 0.4502 time: 0.1449 data_time: 0.0019 lr: 2.997e-05 max_mem: 1153M [04/20 12:24:07 d2.utils.events]: eta: 2:30:47 iter: 79 total_loss: 2.38 loss_ins: 2.045 loss_cate: 0.3513 time: 0.1429 data_time: 0.0016 lr: 3.9961e-05 max_mem: 1156M Traceback (most recent call last): File "/data/project_all/AdelaiDet/tools/train_net.py", line 339, in
launch(
File "/data/pytorchDetect/detectron2/detectron2/engine/launch.py", line 62, in launch
main_func(args)
File "/data/project_all/AdelaiDet/tools/train_net.py", line 333, in main
return trainer.train()
File "/data/project_all/AdelaiDet/tools/train_net.py", line 164, in train
self.train_loop(self.start_iter, self.max_iter)
File "/data/project_all/AdelaiDet/tools/train_net.py", line 153, in train_loop
self.run_step()
File "/data/pytorchDetect/detectron2/detectron2/engine/defaults.py", line 423, in run_step
self._trainer.run_step()
File "/data/pytorchDetect/detectron2/detectron2/engine/train_loop.py", line 222, in run_step
data = next(self._data_loader_iter)
File "/data/pytorchDetect/detectron2/detectron2/data/common.py", line 179, in iter
for d in self.dataset:
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 435, in next
data = self._next_data()
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1085, in _next_data
return self._process_data(data)
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1111, in _process_data
data.reraise()
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/_utils.py", line 428, in reraise
raise self.exc_type(msg)
AssertionError: Caught AssertionError in DataLoader worker process 2.
Original Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 198, in _worker_loop
data = fetcher.fetch(index)
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/ubuntu/anaconda3/envs/detectron2/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 44, in
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/data/pytorchDetect/detectron2/detectron2/data/common.py", line 43, in getitem
data = self._map_func(self._dataset[cur_idx])
File "/data/pytorchDetect/detectron2/detectron2/utils/serialize.py", line 23, in call
return self._obj( args, *kwargs)
File "/data/project_all/AdelaiDet/adet/data/dataset_mapper.py", line 182, in call
instances = annotations_to_instances(
File "/data/project_all/AdelaiDet/adet/data/detection_utils.py", line 53, in annotations_to_instances
instance = d2_anno_to_inst(annos, image_size, mask_format)
File "/data/pytorchDetect/detectron2/detectron2/data/detection_utils.py", line 395, in annotations_to_instances
masks.append(polygons_to_bitmask(segm, image_size))
File "/data/pytorchDetect/detectron2/detectron2/structures/masks.py", line 29, in polygons_to_bitmask
assert len(polygons) > 0, "COCOAPI does not support empty polygons"
AssertionError: COCOAPI does not support empty polygons
Process finished with exit code 1