Broken Pipe Error during tests

After having trouble with recognizing the contents of the 'dataset' folder, I modified the yaml file as such: volumes:

.:/work
/mnt/c/Users/gunku/OneDrive/Belgeler/GitHub/CLRerNet/dataset:/work/dataset

Which temporarily appeared to have resolved the issue, however the test slows down heavily over time and I received errors regarding 'Broken Pipe'. The error is shown below and the contents of the dataset folder are attached, I would sincerely appreciate the help.

Screenshot 2024-05-19 170918

docker@0d896633412d:/work$ python tools/test.py configs/clrernet/culane/clrernet_culane_dla34_ema.py clrernet_culane_dla34_ema.pth Config: model = dict( type='CLRerNet', backbone=dict(type='DLANet', dla='dla34', pretrained=True), neck=dict( type='CLRerNetFPN', in_channels=[128, 256, 512], out_channels=64, num_outs=3), bbox_head=dict( type='CLRerHead', anchor_generator=dict( type='CLRerNetAnchorGenerator', num_priors=192, num_points=72), img_w=800, img_h=320, prior_feat_channels=64, fc_hidden_dim=64, num_fc=2, refine_layers=3, sample_points=36, attention=dict(type='ROIGather'), loss_cls=dict( type='KorniaFocalLoss', alpha=0.25, gamma=2, loss_weight=2.0), loss_bbox=dict(type='SmoothL1Loss', reduction='none', loss_weight=0.2), loss_iou=dict( type='LaneIoULoss', lane_width=0.009375, loss_weight=4.0), loss_seg=dict( type='CLRNetSegLoss', loss_weight=1.0, num_classes=5, ignore_label=255, bg_weight=0.4)), train_cfg=dict( assigner=dict( type='DynamicTopkAssigner', max_topk=4, min_topk=1, cost_combination=1, cls_cost=dict(type='FocalCost', weight=1.0), reg_cost=dict(type='DistanceCost', weight=0.0), iou_dynamick=dict( type='LaneIoUCost', lane_width=0.009375, use_pred_start_end=False, use_giou=True), iou_cost=dict( type='LaneIoUCost', lane_width=0.0375, use_pred_start_end=True, use_giou=True))), test_cfg=dict( conf_threshold=0.43, use_nms=True, as_lanes=True, nms_thres=50, nms_topk=4, ori_img_w=1640, ori_img_h=590, cut_height=270), pretrained=None) dataset_type = 'CulaneDataset' data_root = 'dataset/culane' crop_bbox = [0, 270, 1640, 590] img_scale = (800, 320) img_norm_cfg = dict( mean=[0.0, 0.0, 0.0], std=[255.0, 255.0, 255.0], to_rgb=False) compose_cfg = dict(bboxes=False, keypoints=True, masks=True) train_al_pipeline = [ dict( type='Compose', params=dict(bboxes=False, keypoints=True, masks=True)), dict(type='Crop', x_min=0, x_max=1640, y_min=270, y_max=590, p=1), dict(type='Resize', height=320, width=800, p=1), dict(type='HorizontalFlip', p=0.5), dict(type='ChannelShuffle', p=0.1), dict( type='RandomBrightnessContrast', brightness_limit=0.04, contrast_limit=0.15, p=0.6), dict( type='HueSaturationValue', hue_shift_limit=(-10, 10), sat_shift_limit=(-10, 10), val_shift_limit=(-10, 10), p=0.7), dict( type='OneOf', transforms=[ dict(type='MotionBlur', blur_limit=5, p=1.0), dict(type='MedianBlur', blur_limit=5, p=1.0) ], p=0.2), dict( type='IAAAffine', scale=(0.8, 1.2), rotate=(-10.0, 10.0), translate_percent=0.1, p=0.7), dict(type='Resize', height=320, width=800, p=1) ] val_al_pipeline = [ dict( type='Compose', params=dict(bboxes=False, keypoints=True, masks=True)), dict(type='Crop', x_min=0, x_max=1640, y_min=270, y_max=590, p=1), dict(type='Resize', height=320, width=800, p=1) ] train_pipeline = [ dict( type='albumentation', pipelines=[ dict( type='Compose', params=dict(bboxes=False, keypoints=True, masks=True)), dict(type='Crop', x_min=0, x_max=1640, y_min=270, y_max=590, p=1), dict(type='Resize', height=320, width=800, p=1), dict(type='HorizontalFlip', p=0.5), dict(type='ChannelShuffle', p=0.1), dict( type='RandomBrightnessContrast', brightness_limit=0.04, contrast_limit=0.15, p=0.6), dict( type='HueSaturationValue', hue_shift_limit=(-10, 10), sat_shift_limit=(-10, 10), val_shift_limit=(-10, 10), p=0.7), dict( type='OneOf', transforms=[ dict(type='MotionBlur', blur_limit=5, p=1.0), dict(type='MedianBlur', blur_limit=5, p=1.0) ], p=0.2), dict( type='IAAAffine', scale=(0.8, 1.2), rotate=(-10.0, 10.0), translate_percent=0.1, p=0.7), dict(type='Resize', height=320, width=800, p=1) ]), dict( type='Normalize', mean=[0.0, 0.0, 0.0], std=[255.0, 255.0, 255.0], to_rgb=False), dict(type='DefaultFormatBundle'), dict( type='CollectCLRNet', keys=['img'], meta_keys=[ 'filename', 'sub_img_name', 'ori_shape', 'img_shape', 'img_norm_cfg', 'ori_shape', 'img_shape', 'gt_points', 'gt_masks', 'lanes' ]) ] val_pipeline = [ dict( type='albumentation', pipelines=[ dict( type='Compose', params=dict(bboxes=False, keypoints=True, masks=True)), dict(type='Crop', x_min=0, x_max=1640, y_min=270, y_max=590, p=1), dict(type='Resize', height=320, width=800, p=1) ]), dict( type='Normalize', mean=[0.0, 0.0, 0.0], std=[255.0, 255.0, 255.0], to_rgb=False), dict(type='DefaultFormatBundle'), dict( type='CollectCLRNet', keys=['img'], meta_keys=[ 'filename', 'sub_img_name', 'ori_shape', 'img_shape', 'img_norm_cfg' ]) ] data = dict( samples_per_gpu=24, workers_per_gpu=8, train=dict( type='CulaneDataset', data_root='dataset/culane', data_list='dataset/culane/list/train_gt.txt', diff_file='dataset/culane/list/train_diffs.npz', diff_thr=15, pipeline=[ dict( type='albumentation', pipelines=[ dict( type='Compose', params=dict(bboxes=False, keypoints=True, masks=True)), dict( type='Crop', x_min=0, x_max=1640, y_min=270, y_max=590, p=1), dict(type='Resize', height=320, width=800, p=1), dict(type='HorizontalFlip', p=0.5), dict(type='ChannelShuffle', p=0.1), dict( type='RandomBrightnessContrast', brightness_limit=0.04, contrast_limit=0.15, p=0.6), dict( type='HueSaturationValue', hue_shift_limit=(-10, 10), sat_shift_limit=(-10, 10), val_shift_limit=(-10, 10), p=0.7), dict( type='OneOf', transforms=[ dict(type='MotionBlur', blur_limit=5, p=1.0), dict(type='MedianBlur', blur_limit=5, p=1.0) ], p=0.2), dict( type='IAAAffine', scale=(0.8, 1.2), rotate=(-10.0, 10.0), translate_percent=0.1, p=0.7), dict(type='Resize', height=320, width=800, p=1) ]), dict( type='Normalize', mean=[0.0, 0.0, 0.0], std=[255.0, 255.0, 255.0], to_rgb=False), dict(type='DefaultFormatBundle'), dict( type='CollectCLRNet', keys=['img'], meta_keys=[ 'filename', 'sub_img_name', 'ori_shape', 'img_shape', 'img_norm_cfg', 'ori_shape', 'img_shape', 'gt_points', 'gt_masks', 'lanes' ]) ], test_mode=False), val=dict( type='CulaneDataset', data_root='dataset/culane', data_list='dataset/culane/list/test.txt', pipeline=[ dict( type='albumentation', pipelines=[ dict( type='Compose', params=dict(bboxes=False, keypoints=True, masks=True)), dict( type='Crop', x_min=0, x_max=1640, y_min=270, y_max=590, p=1), dict(type='Resize', height=320, width=800, p=1) ]), dict( type='Normalize', mean=[0.0, 0.0, 0.0], std=[255.0, 255.0, 255.0], to_rgb=False), dict(type='DefaultFormatBundle'), dict( type='CollectCLRNet', keys=['img'], meta_keys=[ 'filename', 'sub_img_name', 'ori_shape', 'img_shape', 'img_norm_cfg' ]) ], test_mode=True), test=dict( type='CulaneDataset', data_root='dataset/culane', data_list='dataset/culane/list/test.txt', pipeline=[ dict( type='albumentation', pipelines=[ dict( type='Compose', params=dict(bboxes=False, keypoints=True, masks=True)), dict( type='Crop', x_min=0, x_max=1640, y_min=270, y_max=590, p=1), dict(type='Resize', height=320, width=800, p=1) ]), dict( type='Normalize', mean=[0.0, 0.0, 0.0], std=[255.0, 255.0, 255.0], to_rgb=False), dict(type='DefaultFormatBundle'), dict( type='CollectCLRNet', keys=['img'], meta_keys=[ 'filename', 'sub_img_name', 'ori_shape', 'img_shape', 'img_norm_cfg' ]) ], test_mode=True)) checkpoint_config = dict(interval=50) log_config = dict( interval=100, hooks=[ dict(type='TextLoggerHook'), dict(type='TensorboardLoggerHookEpoch') ]) device_ids = '0' dist_params = dict(backend='nccl') log_level = 'INFO' load_from = None resume_from = None workflow = [('train', 1)] evaluation = dict(interval=50, metric='F1') custom_imports = dict( imports=[ 'libs.models', 'libs.datasets', 'libs.core.bbox', 'libs.core.anchor', 'libs.core.hook' ], allow_failed_imports=False) cfg_name = 'clrernet_culane_dla34_ema.py' custom_hooks = [dict(type='ExpMomentumEMAHook', momentum=0.0001, priority=49)] total_epochs = 50 optimizer = dict(type='AdamW', lr=0.0006) optimizer_config = dict(grad_clip=None) lr_config = dict(policy='CosineAnnealing', min_lr=0.0006, by_epoch=False)

34680 data are loaded load checkpoint from local path: clrernet_culane_dla34_ema.pth The model and loaded state dict do not match exactly

missing keys in source state_dict: bbox_head.loss_seg.criterion.weight, bbox_head.seg_decoder.conv.weight, bbox_head.seg_decoder.conv.bias

100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 34680/34680 [03:08<00:00, 184.06it/s] 2024-05-18 16:34:51,748 - mmdet - INFO - List: dataset/culane/list/test.txt 2024-05-18 16:34:51,908 - mmdet - INFO - Loading prediction data... 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 34680/34680 [01:15<00:00, 461.85it/s] 2024-05-18 16:36:07,105 - mmdet - INFO - Loading annotation data... 100%|████████████████████████████████████████████████████████████████████████████████████████████████████| 34680/34680 [03:07<00:00, 184.71it/s] 2024-05-18 16:39:14,954 - mmdet - INFO - Calculating metric in parallel... 89%|█████████████████████████████████████████████████████████████████████████████████████▌ | 30913/34680 [5:23:12<59:21:31, 56.73s/it]Killed Process ForkPoolWorker-6: Process ForkPoolWorker-12: Process ForkPoolWorker-3: Process ForkPoolWorker-10: Process ForkPoolWorker-5: Process ForkPoolWorker-15: Process ForkPoolWorker-11: Process ForkPoolWorker-2: Process ForkPoolWorker-14: Process ForkPoolWorker-4: Process ForkPoolWorker-7: Process ForkPoolWorker-9: Process ForkPoolWorker-13: Process ForkPoolWorker-8: docker@0d896633412d:/work$ Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/pool.py", line 131, in worker put((job, i, result)) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/queues.py", line 371, in put self._writer.send_bytes(obj) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 203, in send_bytes self._send_bytes(m[offset:offset + size]) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 414, in _send_bytes self._send(header + buf) BrokenPipeError: [Errno 32] Broken pipe File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) BrokenPipeError: [Errno 32] Broken pipe BrokenPipeError: [Errno 32] Broken pipe File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf) BrokenPipeError: [Errno 32] Broken pipe BrokenPipeError: [Errno 32] Broken pipe BrokenPipeError: [Errno 32] Broken pipe BrokenPipeError: [Errno 32] Broken pipe BrokenPipeError: [Errno 32] Broken pipe BrokenPipeError: [Errno 32] Broken pipe BrokenPipeError: [Errno 32] Broken pipe BrokenPipeError: [Errno 32] Broken pipe File "/home/docker/.pyenv/versions/3.8.4/lib/python3.8/site-packages/multiprocess/connection.py", line 371, in _send n = write(self._handle, buf)