Czm369
commented
1 year ago Maybe you can adjust the parameters of MeanTeacherHook , or make sure supervised learning normal firstly.
Open janglinko-dac opened 1 year ago
Czm369
commented
1 year ago Maybe you can adjust the parameters of MeanTeacherHook , or make sure supervised learning normal firstly.
janglinko-dac
commented
1 year ago I'm sure that supervised learning is normal, I've tried mask rcnn.
JHT458
commented
1 year ago default_scope = 'mmdet' default_hooks = dict( timer=dict(type='IterTimerHook'), logger=dict(type='LoggerHook', interval=50), param_scheduler=dict(type='ParamSchedulerHook'), checkpoint=dict( type='CheckpointHook', interval=500, by_epoch=False, max_keep_ckpts=2), sampler_seed=dict(type='DistSamplerSeedHook'), visualization=dict(type='DetVisualizationHook')) env_cfg = dict( cudnn_benchmark=False, mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), dist_cfg=dict(backend='nccl')) vis_backends = [dict(type='LocalVisBackend')] visualizer = dict( type='DetLocalVisualizer', vis_backends=[dict(type='LocalVisBackend')], name='visualizer') log_processor = dict(type='LogProcessor', window_size=50, by_epoch=False) log_level = 'INFO' load_from = None resume = False dataset_type = 'CocoDataset' data_root = 'D:\\jht_datasets\\tea\\1_2_10_semi\\' metainfo = dict(classes=('Sick', ), palette=[(220, 20, 60)]) batch_size = 5 num_workers = 5 backend_args = None color_space = [[{ 'type': 'ColorTransform' }], [{ 'type': 'AutoContrast' }], [{ 'type': 'Equalize' }], [{ 'type': 'Sharpness' }], [{ 'type': 'Posterize' }], [{ 'type': 'Solarize' }], [{ 'type': 'Color' }], [{ 'type': 'Contrast' }], [{ 'type': 'Brightness' }]] geometric = [[{ 'type': 'Rotate' }], [{ 'type': 'ShearX' }], [{ 'type': 'ShearY' }], [{ 'type': 'TranslateX' }], [{ 'type': 'TranslateY' }]] scale = [(1333, 400), (1333, 1200)] branch_field = ['sup', 'unsup_teacher', 'unsup_student'] sup_pipeline = [ dict(type='LoadImageFromFile', backend_args=None), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='RandAugment', aug_space=[[{ 'type': 'ColorTransform' }], [{ 'type': 'AutoContrast' }], [{ 'type': 'Equalize' }], [{ 'type': 'Sharpness' }], [{ 'type': 'Posterize' }], [{ 'type': 'Solarize' }], [{ 'type': 'Color' }], [{ 'type': 'Contrast' }], [{ 'type': 'Brightness' }]], aug_num=1), dict(type='FilterAnnotations', min_gt_bbox_wh=(0.01, 0.01)), dict( type='MultiBranch', branch_field=['sup', 'unsup_teacher', 'unsup_student'], sup=dict(type='PackDetInputs')) ] weak_pipeline = [ dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')) ] strong_pipeline = [ dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='RandomOrder', transforms=[ dict( type='RandAugment', aug_space=[[{ 'type': 'ColorTransform' }], [{ 'type': 'AutoContrast' }], [{ 'type': 'Equalize' }], [{ 'type': 'Sharpness' }], [{ 'type': 'Posterize' }], [{ 'type': 'Solarize' }], [{ 'type': 'Color' }], [{ 'type': 'Contrast' }], [{ 'type': 'Brightness' }]], aug_num=1), dict( type='RandAugment', aug_space=[[{ 'type': 'Rotate' }], [{ 'type': 'ShearX' }], [{ 'type': 'ShearY' }], [{ 'type': 'TranslateX' }], [{ 'type': 'TranslateY' }]], aug_num=1) ]), dict(type='RandomErasing', n_patches=(1, 5), ratio=(0, 0.2)), dict(type='FilterAnnotations', min_gt_bbox_wh=(0.01, 0.01)), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')) ] unsup_pipeline = [ dict(type='LoadImageFromFile', backend_args=None), dict(type='LoadEmptyAnnotations'), dict( type='MultiBranch', branch_field=['sup', 'unsup_teacher', 'unsup_student'], unsup_teacher=[ dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')) ], unsup_student=[ dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='RandomOrder', transforms=[ dict( type='RandAugment', aug_space=[[{ 'type': 'ColorTransform' }], [{ 'type': 'AutoContrast' }], [{ 'type': 'Equalize' }], [{ 'type': 'Sharpness' }], [{ 'type': 'Posterize' }], [{ 'type': 'Solarize' }], [{ 'type': 'Color' }], [{ 'type': 'Contrast' }], [{ 'type': 'Brightness' }]], aug_num=1), dict( type='RandAugment', aug_space=[[{ 'type': 'Rotate' }], [{ 'type': 'ShearX' }], [{ 'type': 'ShearY' }], [{ 'type': 'TranslateX' }], [{ 'type': 'TranslateY' }]], aug_num=1) ]), dict(type='RandomErasing', n_patches=(1, 5), ratio=(0, 0.2)), dict(type='FilterAnnotations', min_gt_bbox_wh=(0.01, 0.01)), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')) ]) ] test_pipeline = [ dict(type='LoadImageFromFile', backend_args=None), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] labeled_dataset = dict( type='CocoDataset', data_root='D:\\jht_datasets\\tea\\1_2_10_semi\\', metainfo=dict(classes=('Sick', ), palette=[(220, 20, 60)]), ann_file='annotations\\instance_train.json', data_prefix=dict(img='train\\'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[ dict(type='LoadImageFromFile', backend_args=None), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='RandAugment', aug_space=[[{ 'type': 'ColorTransform' }], [{ 'type': 'AutoContrast' }], [{ 'type': 'Equalize' }], [{ 'type': 'Sharpness' }], [{ 'type': 'Posterize' }], [{ 'type': 'Solarize' }], [{ 'type': 'Color' }], [{ 'type': 'Contrast' }], [{ 'type': 'Brightness' }]], aug_num=1), dict(type='FilterAnnotations', min_gt_bbox_wh=(0.01, 0.01)), dict( type='MultiBranch', branch_field=['sup', 'unsup_teacher', 'unsup_student'], sup=dict(type='PackDetInputs')) ], backend_args=None) unlabeled_dataset = dict( type='CocoDataset', data_root='D:\\jht_datasets\\tea\\1_2_10_semi\\', metainfo=dict(classes=('Sick', ), palette=[(220, 20, 60)]), ann_file='annotations\\instance_unlabeled.json', data_prefix=dict(img='unlabeled\\'), filter_cfg=dict(filter_empty_gt=False), pipeline=[ dict(type='LoadImageFromFile', backend_args=None), dict(type='LoadEmptyAnnotations'), dict( type='MultiBranch', branch_field=['sup', 'unsup_teacher', 'unsup_student'], unsup_teacher=[ dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')) ], unsup_student=[ dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='RandomOrder', transforms=[ dict( type='RandAugment', aug_space=[[{ 'type': 'ColorTransform' }], [{ 'type': 'AutoContrast' }], [{ 'type': 'Equalize' }], [{ 'type': 'Sharpness' }], [{ 'type': 'Posterize' }], [{ 'type': 'Solarize' }], [{ 'type': 'Color' }], [{ 'type': 'Contrast' }], [{ 'type': 'Brightness' }]], aug_num=1), dict( type='RandAugment', aug_space=[[{ 'type': 'Rotate' }], [{ 'type': 'ShearX' }], [{ 'type': 'ShearY' }], [{ 'type': 'TranslateX' }], [{ 'type': 'TranslateY' }]], aug_num=1) ]), dict(type='RandomErasing', n_patches=(1, 5), ratio=(0, 0.2)), dict(type='FilterAnnotations', min_gt_bbox_wh=(0.01, 0.01)), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')) ]) ], backend_args=None) train_dataloader = dict( batch_size=5, num_workers=5, persistent_workers=True, sampler=dict( type='GroupMultiSourceSampler', batch_size=5, source_ratio=[1, 4]), dataset=dict( type='ConcatDataset', datasets=[ dict( type='CocoDataset', data_root='D:\\jht_datasets\\tea\\1_2_10_semi\\', metainfo=dict(classes=('Sick', ), palette=[(220, 20, 60)]), ann_file='annotations\\instance_train.json', data_prefix=dict(img='train\\'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[ dict(type='LoadImageFromFile', backend_args=None), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='RandAugment', aug_space=[[{ 'type': 'ColorTransform' }], [{ 'type': 'AutoContrast' }], [{ 'type': 'Equalize' }], [{ 'type': 'Sharpness' }], [{ 'type': 'Posterize' }], [{ 'type': 'Solarize' }], [{ 'type': 'Color' }], [{ 'type': 'Contrast' }], [{ 'type': 'Brightness' }]], aug_num=1), dict( type='FilterAnnotations', min_gt_bbox_wh=(0.01, 0.01)), dict( type='MultiBranch', branch_field=['sup', 'unsup_teacher', 'unsup_student'], sup=dict(type='PackDetInputs')) ], backend_args=None), dict( type='CocoDataset', data_root='D:\\jht_datasets\\tea\\1_2_10_semi\\', metainfo=dict(classes=('Sick', ), palette=[(220, 20, 60)]), ann_file='annotations\\instance_unlabeled.json', data_prefix=dict(img='unlabeled\\'), filter_cfg=dict(filter_empty_gt=False), pipeline=[ dict(type='LoadImageFromFile', backend_args=None), dict(type='LoadEmptyAnnotations'), dict( type='MultiBranch', branch_field=['sup', 'unsup_teacher', 'unsup_student'], unsup_teacher=[ dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')) ], unsup_student=[ dict( type='RandomResize', scale=[(1333, 400), (1333, 1200)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='RandomOrder', transforms=[ dict( type='RandAugment', aug_space=[[{ 'type': 'ColorTransform' }], [{ 'type': 'AutoContrast' }], [{ 'type': 'Equalize' }], [{ 'type': 'Sharpness' }], [{ 'type': 'Posterize' }], [{ 'type': 'Solarize' }], [{ 'type': 'Color' }], [{ 'type': 'Contrast' }], [{ 'type': 'Brightness' }]], aug_num=1), dict( type='RandAugment', aug_space=[[{ 'type': 'Rotate' }], [{ 'type': 'ShearX' }], [{ 'type': 'ShearY' }], [{ 'type': 'TranslateX' }], [{ 'type': 'TranslateY' }]], aug_num=1) ]), dict( type='RandomErasing', n_patches=(1, 5), ratio=(0, 0.2)), dict( type='FilterAnnotations', min_gt_bbox_wh=(0.01, 0.01)), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')) ]) ], backend_args=None) ])) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type='CocoDataset', data_root='D:\\jht_datasets\\tea\\1_2_10_semi\\', metainfo=dict(classes=('Sick', ), palette=[(220, 20, 60)]), ann_file='annotations\\instance_val.json', data_prefix=dict(img='val\\'), test_mode=True, pipeline=[ dict(type='LoadImageFromFile', backend_args=None), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ], backend_args=None)) test_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type='CocoDataset', data_root='D:\\jht_datasets\\tea\\1_2_10_semi\\', metainfo=dict(classes=('Sick', ), palette=[(220, 20, 60)]), ann_file='annotations\\instance_test.json', data_prefix=dict(img='test\\'), test_mode=True, pipeline=[ dict(type='LoadImageFromFile', backend_args=None), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ], backend_args=None)) val_evaluator = dict( type='CocoMetric', ann_file= 'D:\\jht_datasets\\tea\\1_2_10_semi\\annotations\\instance_val.json', metric='bbox', format_only=False, backend_args=None) test_evaluator = dict( type='CocoMetric', ann_file= 'D:\\jht_datasets\\tea\\1_2_10_semi\\annotations\\instance_test.json', metric='bbox', format_only=False, backend_args=None) detector_ = dict( type='FasterRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), roi_head=dict( type='StandardRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=1, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0))), train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=2000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=1000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))) model = dict( type='SoftTeacher', detector=dict( type='FasterRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), roi_head=dict( type='StandardRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=1, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0))), train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=2000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=1000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))), data_preprocessor=dict( type='MultiBranchDataPreprocessor', data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32)), semi_train_cfg=dict( freeze_teacher=True, sup_weight=1.0, unsup_weight=4.0, pseudo_label_initial_score_thr=0.5, rpn_pseudo_thr=0.9, cls_pseudo_thr=0.9, reg_pseudo_thr=0.02, jitter_times=10, jitter_scale=0.06, min_pseudo_bbox_wh=(0.01, 0.01)), semi_test_cfg=dict(predict_on='teacher')) custom_hooks = [dict(type='MeanTeacherHook')] train_cfg = dict(type='IterBasedTrainLoop', max_iters=5000, val_interval=500) val_cfg = dict(type='TeacherStudentValLoop') test_cfg = dict(type='TestLoop') param_scheduler = [ dict( type='LinearLR', start_factor=1e-05, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=5001, by_epoch=False, milestones=[3000, 4200], gamma=0.1) ] optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.002, momentum=0.9, weight_decay=0.0001)) launcher = 'none' work_dir = 'D:\\jht_code\\mmdetection_git\\mmdetection\\semi_supervised\\faster-rcnn-r50_fpn_semi'
My problem is that epoch has not been able to move on to the next one, so after I modified my configuration file against your configuration file, it was normal, and the top is my configuration file, I hope it will help you. @janglinko-dac
waterfflo
commented
2 months ago Could you solved this propblem?I saw the official network released the log to traning process,which recorded the loss of unsup pipeline. The unsup_loss_cls was not equal to 0 until iter was about 2000. As follows:
{"lr": 0.009999999999999976, "data_time": 0.02199803352355957, "loss": 0.927637699842453, "sup_loss_rpn_cls": 0.09896203741431236, "sup_loss_rpn_bbox": 0.07710891971364617, "sup_loss_cls": 0.39088220164179804, "sup_acc": 76.171875, "sup_loss_bbox": 0.30077028153813445, "unsup_loss_rpn_cls": 0.022934514712542296, "unsup_loss_rpn_bbox": 0.0, "unsup_loss_cls": 0.033303560058120636, "unsup_acc": 100.0, "unsup_loss_bbox": 0.003676195340231061, "time": 0.7283811378479004, "step": 2000} {"lr": 0.009999999999999976, "data_time": 0.02259686470031738, "loss": 0.9643001854419708, "sup_loss_rpn_cls": 0.0845933137089014, "sup_loss_rpn_bbox": 0.07065149092581123, "sup_loss_cls": 0.44346953749656676, "sup_acc": 97.265625, "sup_loss_bbox": 0.2899776066839695, "unsup_loss_rpn_cls": 0.028726631170138718, "unsup_loss_rpn_bbox": 0.0, "unsup_loss_cls": 0.0435255526273977, "unsup_acc": 99.609375, "unsup_loss_bbox": 0.0033560442179441452, "time": 0.7373209142684937, "step": 2050}
But I met the problem that my traning process often stop in one iter, GPU's Utilization rate becomes low even close to 0%,and no error or warnings.
waterfflo
commented
2 months ago Could you solved this propblem?I saw the official network released the log to traning process,which recorded the loss of unsup pipeline. The unsup_loss_cls was not equal to 0 until iter was about 2000. As follows:
{"lr": 0.009999999999999976, "data_time": 0.02199803352355957, "loss": 0.927637699842453, "sup_loss_rpn_cls": 0.09896203741431236, "sup_loss_rpn_bbox": 0.07710891971364617, "sup_loss_cls": 0.39088220164179804, "sup_acc": 76.171875, "sup_loss_bbox": 0.30077028153813445, "unsup_loss_rpn_cls": 0.022934514712542296, "unsup_loss_rpn_bbox": 0.0, "unsup_loss_cls": 0.033303560058120636, "unsup_acc": 100.0, "unsup_loss_bbox": 0.003676195340231061, "time": 0.7283811378479004, "step": 2000} {"lr": 0.009999999999999976, "data_time": 0.02259686470031738, "loss": 0.9643001854419708, "sup_loss_rpn_cls": 0.0845933137089014, "sup_loss_rpn_bbox": 0.07065149092581123, "sup_loss_cls": 0.44346953749656676, "sup_acc": 97.265625, "sup_loss_bbox": 0.2899776066839695, "unsup_loss_rpn_cls": 0.028726631170138718, "unsup_loss_rpn_bbox": 0.0, "unsup_loss_cls": 0.0435255526273977, "unsup_acc": 99.609375, "unsup_loss_bbox": 0.0033560442179441452, "time": 0.7373209142684937, "step": 2050}
But I met the problem that my traning process often stop in one iter, GPU's Utilization rate becomes low even close to 0%,and no error or warnings.
waterfflo
commented
2 months ago Could you solved this propblem?I saw the official network released the log to traning process,which recorded the loss of unsup pipeline. The unsup_loss_cls was not equal to 0 until iter was about 2000. As follows:
{"lr": 0.009999999999999976, "data_time": 0.02199803352355957, "loss": 0.927637699842453, "sup_loss_rpn_cls": 0.09896203741431236, "sup_loss_rpn_bbox": 0.07710891971364617, "sup_loss_cls": 0.39088220164179804, "sup_acc": 76.171875, "sup_loss_bbox": 0.30077028153813445, "unsup_loss_rpn_cls": 0.022934514712542296, "unsup_loss_rpn_bbox": 0.0, "unsup_loss_cls": 0.033303560058120636, "unsup_acc": 100.0, "unsup_loss_bbox": 0.003676195340231061, "time": 0.7283811378479004, "step": 2000} {"lr": 0.009999999999999976, "data_time": 0.02259686470031738, "loss": 0.9643001854419708, "sup_loss_rpn_cls": 0.0845933137089014, "sup_loss_rpn_bbox": 0.07065149092581123, "sup_loss_cls": 0.44346953749656676, "sup_acc": 97.265625, "sup_loss_bbox": 0.2899776066839695, "unsup_loss_rpn_cls": 0.028726631170138718, "unsup_loss_rpn_bbox": 0.0, "unsup_loss_cls": 0.0435255526273977, "unsup_acc": 99.609375, "unsup_loss_bbox": 0.0033560442179441452, "time": 0.7373209142684937, "step": 2050}
But I met the problem that my traning process often stop in one iter, GPU's Utilization rate becomes low even close to 0%,and no error or warnings.
Hi, im trying to train SoftTeacher Fast R-CNN, but the network is not learning at all. It's related to #10223 but it doesn't resolve my problems.
Unuspervised losses are very close to 0
06/27 08:12:13 - mmengine - INFO - Epoch(train) [1][1000/1407] lr: 1.0000e-04 eta: 1:22:00 time: 1.2690 data_time: 0.0195 memory: 10704 loss: 0.8256 sup_loss_rpn_cls: 0.2903 sup_loss_rpn_bbox: 0.1076 sup_loss_cls: 0.2240 sup_acc: 96.4844 sup_loss_bbox: 0.1864 unsup_loss_rpn_cls: 0.0102 unsup_loss_rpn_bbox: 0.0005 unsup_loss_cls: 0.0067 unsup_acc: 100.0000 unsup_loss_bbox: 0.0000Results are very bad
What should I change?
Edit: What does it mean
freeze_teacherin thesemi_train_cfg?