MendelXu
commented
3 years ago Could you post the full log here?
Closed winnerziqi closed 3 years ago
MendelXu
commented
3 years ago Could you post the full log here?
winnerziqi
commented
3 years ago thanks, here is my full log.
fatal: Not a git repository (or any parent up to mount point /data6) Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set). 2021-10-18 13:36:46,968 - mmdet.ssod - INFO - [<StreamHandler
(INFO)>, <FileHandler /data6/ziqiwen/code/softteacher/work_dirs/soft_teacher_faster_rcnn_r50_caffe_fpn_coco_180k/10/1/20211018_133646.log (INFO)>] 2021-10-18 13:36:46,968 - mmdet.ssod - INFO - Environment info: sys.platform: linux Python: 3.7.11 (default, Jul 27 2021, 14:32:16) [GCC 7.5.0] CUDA available: True GPU 0,1,2,3: TITAN Xp CUDA_HOME: /usr/local/cuda NVCC: Cuda compilation tools, release 9.0, V9.0.176 GCC: gcc (Ubuntu 5.4.0-6ubuntu1~16.04.12) 5.4.0 20160609 PyTorch: 1.6.0 PyTorch compiling details: PyTorch built with:
- GCC 7.3
- C++ Version: 201402
- Intel(R) oneAPI Math Kernel Library Version 2021.3-Product Build 20210617 for Intel(R) 64 architecture applications
- Intel(R) MKL-DNN v1.5.0 (Git Hash e2ac1fac44c5078ca927cb9b90e1b3066a0b2ed0)
- OpenMP 201511 (a.k.a. OpenMP 4.5)
- NNPACK is enabled
- CPU capability usage: AVX2
- CUDA Runtime 10.1
- NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_37,code=compute_37
- CuDNN 7.6.3
- Magma 2.5.2
- Build settings: BLAS=MKL, BUILD_TYPE=Release, CXX_FLAGS= -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -fopenmp -DNDEBUG -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DUSE_VULKAN_WRAPPER -O2 -fPIC -Wno-narrowing -Wall -Wextra -Werror=return-type -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wno-sign-compare -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function -Wno-unused-result -Wno-unused-local-typedefs -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Wno-stringop-overflow, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, USE_CUDA=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=ON, USE_NNPACK=ON, USE_OPENMP=ON, USE_STATIC_DISPATCH=OFF,
2021-10-18 13:36:49,973 - mmdet.ssod - INFO - Distributed training: True 2021-10-18 13:36:53,132 - mmdet.ssod - INFO - Config: model = dict( type='SoftTeacher', model=dict( type='FasterRCNN', backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), roi_head=dict( type='StandardRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0))), train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=2000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=1000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))), train_cfg=dict( use_teacher_proposal=False, pseudo_label_initial_score_thr=0.5, rpn_pseudo_threshold=0.9, cls_pseudo_threshold=0.9, reg_pseudo_threshold=0.01, jitter_times=10, jitter_scale=0.06, min_pseduo_box_size=0, unsup_weight=4.0), test_cfg=dict(inference_on='student')) dataset_type = 'CocoDataset' img_norm_cfg = dict( mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False) train_pipeline = [ dict(type='LoadImageFromFile'), dict(type='LoadAnnotations', with_bbox=True), dict( type='Sequential', transforms=[ dict( type='RandResize', img_scale=[(1333, 400), (1333, 1200)], multiscale_mode='range', keep_ratio=True), dict(type='RandFlip', flip_ratio=0.5), dict( type='OneOf', transforms=[ dict(type='Identity'), dict(type='AutoContrast'), dict(type='RandEqualize'), dict(type='RandSolarize'), dict(type='RandColor'), dict(type='RandContrast'), dict(type='RandBrightness'), dict(type='RandSharpness'), dict(type='RandPosterize') ]) ], record=True), dict(type='Pad', size_divisor=32), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='ExtraAttrs', tag='sup'), dict(type='DefaultFormatBundle'), dict( type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'], meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg', 'pad_shape', 'scale_factor', 'tag')) ] test_pipeline = [ dict(type='LoadImageFromFile'), dict( type='MultiScaleFlipAug', img_scale=(1333, 800), flip=False, transforms=[ dict(type='Resize', keep_ratio=True), dict(type='RandomFlip'), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='Pad', size_divisor=32), dict(type='ImageToTensor', keys=['img']), dict(type='Collect', keys=['img']) ]) ] data = dict( samples_per_gpu=4, workers_per_gpu=4, train=dict( type='SemiDataset', sup=dict( type='CocoDataset', ann_file= '/data6/ziqiwen/code/unbiased-teacher/datasets/coco/annotations/semi_supervised/instances_train2017.1@10.json', img_prefix= '/data6/ziqiwen/code/unbiased-teacher/datasets/coco/train2017/', pipeline=[ dict(type='LoadImageFromFile'), dict(type='LoadAnnotations', with_bbox=True), dict( type='Sequential', transforms=[ dict( type='RandResize', img_scale=[(1333, 400), (1333, 1200)], multiscale_mode='range', keep_ratio=True), dict(type='RandFlip', flip_ratio=0.5), dict( type='OneOf', transforms=[ dict(type='Identity'), dict(type='AutoContrast'), dict(type='RandEqualize'), dict(type='RandSolarize'), dict(type='RandColor'), dict(type='RandContrast'), dict(type='RandBrightness'), dict(type='RandSharpness'), dict(type='RandPosterize') ]) ], record=True), dict(type='Pad', size_divisor=32), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='ExtraAttrs', tag='sup'), dict(type='DefaultFormatBundle'), dict( type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'], meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg', 'pad_shape', 'scale_factor', 'tag')) ]), unsup=dict( type='CocoDataset', ann_file= '/data6/ziqiwen/code/unbiased-teacher/datasets/coco/annotations/semi_supervised/instances_train2017.1@10-unlabeled.json', img_prefix= '/data6/ziqiwen/code/unbiased-teacher/datasets/coco/train2017/', pipeline=[ dict(type='LoadImageFromFile'), dict(type='PseudoSamples', with_bbox=True), dict( type='MultiBranch', unsup_teacher=[ dict( type='Sequential', transforms=[ dict( type='RandResize', img_scale=[(1333, 400), (1333, 1200)], multiscale_mode='range', keep_ratio=True), dict(type='RandFlip', flip_ratio=0.5), dict( type='ShuffledSequential', transforms=[ dict( type='OneOf', transforms=[ dict(type='Identity'), dict(type='AutoContrast'), dict(type='RandEqualize'), dict(type='RandSolarize'), dict(type='RandColor'), dict(type='RandContrast'), dict(type='RandBrightness'), dict(type='RandSharpness'), dict(type='RandPosterize') ]), dict( type='OneOf', transforms=[{ 'type': 'RandTranslate', 'x': (-0.1, 0.1) }, { 'type': 'RandTranslate', 'y': (-0.1, 0.1) }, { 'type': 'RandRotate', 'angle': (-30, 30) }, [{ 'type': 'RandShear', 'x': (-30, 30) }, { 'type': 'RandShear', 'y': (-30, 30) }]]) ]), dict( type='RandErase', n_iterations=(1, 5), size=[0, 0.2], squared=True) ], record=True), dict(type='Pad', size_divisor=32), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='ExtraAttrs', tag='unsup_student'), dict(type='DefaultFormatBundle'), dict( type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'], meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg', 'pad_shape', 'scale_factor', 'tag', 'transform_matrix')) ], unsup_student=[ dict( type='Sequential', transforms=[ dict( type='RandResize', img_scale=[(1333, 400), (1333, 1200)], multiscale_mode='range', keep_ratio=True), dict(type='RandFlip', flip_ratio=0.5) ], record=True), dict(type='Pad', size_divisor=32), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='ExtraAttrs', tag='unsup_teacher'), dict(type='DefaultFormatBundle'), dict( type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'], meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg', 'pad_shape', 'scale_factor', 'tag', 'transform_matrix')) ]) ], filter_empty_gt=False)), val=dict( type='CocoDataset', ann_file= '/data6/ziqiwen/code/unbiased-teacher/datasets/coco/annotations/semi_supervised/instances_train2017.1@10.json', img_prefix= '/data6/ziqiwen/code/unbiased-teacher/datasets/coco/train2017/', pipeline=[ dict(type='LoadImageFromFile'), dict( type='MultiScaleFlipAug', img_scale=(1333, 800), flip=False, transforms=[ dict(type='Resize', keep_ratio=True), dict(type='RandomFlip'), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='Pad', size_divisor=32), dict(type='ImageToTensor', keys=['img']), dict(type='Collect', keys=['img']) ]) ]), test=dict( type='CocoDataset', ann_file= '/data6/ziqiwen/code/unbiased-teacher/datasets/coco/annotations/semi_supervised/instances_train2017.1@10.json', img_prefix= '/data6/ziqiwen/code/unbiased-teacher/datasets/coco/train2017/', pipeline=[ dict(type='LoadImageFromFile'), dict( type='MultiScaleFlipAug', img_scale=(1333, 800), flip=False, transforms=[ dict(type='Resize', keep_ratio=True), dict(type='RandomFlip'), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='Pad', size_divisor=32), dict(type='ImageToTensor', keys=['img']), dict(type='Collect', keys=['img']) ]) ]), sampler=dict( train=dict( type='SemiBalanceSampler', sample_ratio=[1, 4], by_prob=True, epoch_length=7330))) evaluation = dict(interval=4000, metric='bbox', type='SubModulesDistEvalHook') optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001) optimizer_config = dict(grad_clip=None) lr_config = dict( policy='step', warmup='linear', warmup_iters=500, warmup_ratio=0.001, step=[120000, 160000]) runner = dict(type='IterBasedRunner', max_iters=180000) checkpoint_config = dict(interval=4000, by_epoch=False, max_keep_ckpts=20) log_config = dict(interval=50, hooks=[dict(type='TextLoggerHook')]) custom_hooks = [ dict(type='NumClassCheckHook'), dict(type='WeightSummary'), dict(type='MeanTeacher', momentum=0.999, interval=1, warm_up=0) ] dist_params = dict(backend='nccl') log_level = 'INFO' load_from = None resume_from = None workflow = [('train', 1)] mmdet_base = '../base' strong_pipeline = [ dict( type='Sequential', transforms=[ dict( type='RandResize', img_scale=[(1333, 400), (1333, 1200)], multiscale_mode='range', keep_ratio=True), dict(type='RandFlip', flip_ratio=0.5), dict( type='ShuffledSequential', transforms=[ dict( type='OneOf', transforms=[ dict(type='Identity'), dict(type='AutoContrast'), dict(type='RandEqualize'), dict(type='RandSolarize'), dict(type='RandColor'), dict(type='RandContrast'), dict(type='RandBrightness'), dict(type='RandSharpness'), dict(type='RandPosterize') ]), dict( type='OneOf', transforms=[{ 'type': 'RandTranslate', 'x': (-0.1, 0.1) }, { 'type': 'RandTranslate', 'y': (-0.1, 0.1) }, { 'type': 'RandRotate', 'angle': (-30, 30) }, [{ 'type': 'RandShear', 'x': (-30, 30) }, { 'type': 'RandShear', 'y': (-30, 30) }]]) ]), dict( type='RandErase', n_iterations=(1, 5), size=[0, 0.2], squared=True) ], record=True), dict(type='Pad', size_divisor=32), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='ExtraAttrs', tag='unsup_student'), dict(type='DefaultFormatBundle'), dict( type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'], meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg', 'pad_shape', 'scale_factor', 'tag', 'transform_matrix')) ] weak_pipeline = [ dict( type='Sequential', transforms=[ dict( type='RandResize', img_scale=[(1333, 400), (1333, 1200)], multiscale_mode='range', keep_ratio=True), dict(type='RandFlip', flip_ratio=0.5) ], record=True), dict(type='Pad', size_divisor=32), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='ExtraAttrs', tag='unsup_teacher'), dict(type='DefaultFormatBundle'), dict( type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'], meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg', 'pad_shape', 'scale_factor', 'tag', 'transform_matrix')) ] unsup_pipeline = [ dict(type='LoadImageFromFile'), dict(type='PseudoSamples', with_bbox=True), dict( type='MultiBranch', unsup_teacher=[ dict( type='Sequential', transforms=[ dict( type='RandResize', img_scale=[(1333, 400), (1333, 1200)], multiscale_mode='range', keep_ratio=True), dict(type='RandFlip', flip_ratio=0.5), dict( type='ShuffledSequential', transforms=[ dict( type='OneOf', transforms=[ dict(type='Identity'), dict(type='AutoContrast'), dict(type='RandEqualize'), dict(type='RandSolarize'), dict(type='RandColor'), dict(type='RandContrast'), dict(type='RandBrightness'), dict(type='RandSharpness'), dict(type='RandPosterize') ]), dict( type='OneOf', transforms=[{ 'type': 'RandTranslate', 'x': (-0.1, 0.1) }, { 'type': 'RandTranslate', 'y': (-0.1, 0.1) }, { 'type': 'RandRotate', 'angle': (-30, 30) }, [{ 'type': 'RandShear', 'x': (-30, 30) }, { 'type': 'RandShear', 'y': (-30, 30) }]]) ]), dict( type='RandErase', n_iterations=(1, 5), size=[0, 0.2], squared=True) ], record=True), dict(type='Pad', size_divisor=32), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='ExtraAttrs', tag='unsup_student'), dict(type='DefaultFormatBundle'), dict( type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'], meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg', 'pad_shape', 'scale_factor', 'tag', 'transform_matrix')) ], unsup_student=[ dict( type='Sequential', transforms=[ dict( type='RandResize', img_scale=[(1333, 400), (1333, 1200)], multiscale_mode='range', keep_ratio=True), dict(type='RandFlip', flip_ratio=0.5) ], record=True), dict(type='Pad', size_divisor=32), dict( type='Normalize', mean=[103.53, 116.28, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False), dict(type='ExtraAttrs', tag='unsup_teacher'), dict(type='DefaultFormatBundle'), dict( type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'], meta_keys=('filename', 'ori_shape', 'img_shape', 'img_norm_cfg', 'pad_shape', 'scale_factor', 'tag', 'transform_matrix')) ]) ] fp16 = dict(loss_scale='dynamic') fold = 1 percent = 10 work_dir = 'work_dirs/soft_teacher_faster_rcnn_r50_caffe_fpn_coco_180k/10/1' cfg_name = 'soft_teacher_faster_rcnn_r50_caffe_fpn_coco_180k' gpu_ids = range(0, 1)
/home/ziqiwen/code/mmdetection/mmdet/core/anchor/builder.py:17: UserWarning: build_anchor_generator would be deprecated soon, please use build_prior_generator
'build_anchor_generator would be deprecated soon, please use '
2021-10-18 13:36:54,143 - mmdet.ssod - INFO - initialize ResNet with init_cfg {'type': 'Pretrained', 'checkpoint': 'open-mmlab://detectron2/resnet50_caffe'}
2021-10-18 13:36:54,144 - mmcv - INFO - load model from: open-mmlab://detectron2/resnet50_caffe
2021-10-18 13:36:54,144 - mmcv - INFO - Use load_from_openmmlab loader
2021-10-18 13:36:54,265 - mmcv - WARNING - The model and loaded state dict do not match exactly
unexpected key in source state_dict: conv1.bias
2021-10-18 13:36:54,295 - mmdet.ssod - INFO - initialize FPN with init_cfg {'type': 'Xavier', 'layer': 'Conv2d', 'distribution': 'uniform'} 2021-10-18 13:36:54,328 - mmdet.ssod - INFO - initialize RPNHead with init_cfg {'type': 'Normal', 'layer': 'Conv2d', 'std': 0.01} 2021-10-18 13:36:54,337 - mmdet.ssod - INFO - initialize Shared2FCBBoxHead with init_cfg [{'type': 'Normal', 'std': 0.01, 'override': {'name': 'fc_cls'}}, {'type': 'Normal', 'std': 0.001, 'override': {'name': 'fc_reg'}}, {'type': 'Xavier', 'layer': 'Linear', 'override': [{'name': 'shared_fcs'}, {'name': 'cls_fcs'}, {'name': 'reg_fcs'}]}] 2021-10-18 13:36:54,773 - mmdet.ssod - INFO - initialize ResNet with init_cfg {'type': 'Pretrained', 'checkpoint': 'open-mmlab://detectron2/resnet50_caffe'} 2021-10-18 13:36:54,774 - mmcv - INFO - load model from: open-mmlab://detectron2/resnet50_caffe 2021-10-18 13:36:54,774 - mmcv - INFO - Use load_from_openmmlab loader 2021-10-18 13:36:54,883 - mmcv - WARNING - The model and loaded state dict do not match exactly
unexpected key in source state_dict: conv1.bias
2021-10-18 13:36:54,912 - mmdet.ssod - INFO - initialize FPN with init_cfg {'type': 'Xavier', 'layer': 'Conv2d', 'distribution': 'uniform'}
2021-10-18 13:36:54,943 - mmdet.ssod - INFO - initialize RPNHead with init_cfg {'type': 'Normal', 'layer': 'Conv2d', 'std': 0.01}
2021-10-18 13:36:54,953 - mmdet.ssod - INFO - initialize Shared2FCBBoxHead with init_cfg [{'type': 'Normal', 'std': 0.01, 'override': {'name': 'fc_cls'}}, {'type': 'Normal', 'std': 0.001, 'override': {'name': 'fc_reg'}}, {'type': 'Xavier', 'layer': 'Linear', 'override': [{'name': 'shared_fcs'}, {'name': 'cls_fcs'}, {'name': 'reg_fcs'}]}]
loading annotations into memory...
Done (t=1.48s)
creating index...
index created!
loading annotations into memory...
Done (t=14.08s)
creating index...
index created!
fatal: Not a git repository (or any parent up to mount point /data6)
Stopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).
loading annotations into memory...
Done (t=1.17s)
creating index...
index created!
2021-10-18 13:37:18,183 - mmdet.ssod - INFO - Start running, host: ziqiwen@ISIP-IW4200-4G-3, work_dir: /data6/ziqiwen/code/softteacher/work_dirs/soft_teacher_faster_rcnn_r50_caffe_fpn_coco_180k/10/1
2021-10-18 13:37:18,184 - mmdet.ssod - INFO - Hooks will be executed in the following order:
before_run:
(VERY_HIGH ) StepLrUpdaterHook
(ABOVE_NORMAL) Fp16OptimizerHook
(NORMAL ) CheckpointHook
(NORMAL ) WeightSummary
(NORMAL ) MeanTeacher
(80 ) SubModulesDistEvalHook
(VERY_LOW ) TextLoggerHook
before_train_epoch:
(VERY_HIGH ) StepLrUpdaterHook
(NORMAL ) NumClassCheckHook
(LOW ) IterTimerHook
(80 ) SubModulesDistEvalHook
(VERY_LOW ) TextLoggerHook
before_train_iter:
(VERY_HIGH ) StepLrUpdaterHook
(NORMAL ) MeanTeacher
(LOW ) IterTimerHook
(80 ) SubModulesDistEvalHook
after_train_iter:
(ABOVE_NORMAL) Fp16OptimizerHook
(NORMAL ) CheckpointHook
(NORMAL ) MeanTeacher
(LOW ) IterTimerHook
(80 ) SubModulesDistEvalHook
(VERY_LOW ) TextLoggerHook
after_train_epoch:
(NORMAL ) CheckpointHook
(80 ) SubModulesDistEvalHook
(VERY_LOW ) TextLoggerHook
before_val_epoch:
(NORMAL ) NumClassCheckHook
(LOW ) IterTimerHook
(VERY_LOW ) TextLoggerHook
before_val_iter: (LOW ) IterTimerHook
after_val_iter: (LOW ) IterTimerHook
after_val_epoch: (VERY_LOW ) TextLoggerHook
len(self) 29320
len(indices) 20488
Traceback (most recent call last):
File "tools/train.py", line 198, in
MendelXu
commented
3 years ago I am not sure what the problem is. I have tried to change the samplers_per_gpu to 4 like you, but it works well. I will take a deeper look later.
winnerziqi
commented
3 years ago thanks a lot!!!
MendelXu
commented
3 years ago Could you have a look at the json file and check whether correct number of instances is loaded? I have tried to build a similar python environment and similar config but it seems ok.
jackhu-bme
commented
3 years ago I guess the parametyer epoch_length is too small for your dataset since I encountered the same problem in my medical daataset weeks ago and solved this simply by turning it bigger. I haven't carefully look at the sampler code so it's just a simple and maybe unreasonable guess.
jessicametzger
commented
3 years ago I am having this same issue. Regardless of what I set epoch_length to, len(indices) always ends up slightly smaller than len(self)=sum(epoch_length)*samples_per_gpu. Here's my config:
_base_ = [parent_dir+'SoftTeacher/configs/soft_teacher/base.py']
data = dict(
samples_per_gpu=2,
workers_per_gpu=2,
train = dict(
sup = dict(
ann_file=parent_dir+sup_data_path+'/train_data/annotations.json',
img_prefix=parent_dir+sup_data_path+'/train_data/',
classes=classes
),
unsup = dict(
ann_file=parent_dir+unsup_data_path+'/train_data/annotations.json',
img_prefix=parent_dir+unsup_data_path+'/train_data/',
classes=classes
),
),
val = dict(
ann_file=parent_dir+sup_data_path+'/val_data/annotations.json',
img_prefix=parent_dir+sup_data_path+'/val_data/',
classes=classes
),
test = dict(
ann_file=parent_dir+sup_data_path+'/val_data/annotations.json',
img_prefix=parent_dir+sup_data_path+'/val_data/',
classes=classes
),
sampler=dict(
train=dict(
type="SemiBalanceSampler",
sample_ratio=[1, 4],
by_prob=True,
# at_least_one=True,
epoch_length=1000,
)
),
)
evaluation = dict(interval=1000, metric='bbox', type='SubModulesDistEvalHook')
optimizer = dict(type='SGD', lr=0.001, momentum=0.9, weight_decay=0.0001)
optimizer_config = dict(grad_clip=None)
evaluation = dict(type="SubModulesDistEvalHook", interval=4000)
optimizer = dict(type="SGD", lr=0.01, momentum=0.9, weight_decay=0.0001)
lr_config = dict(step=[3000, 4000])
runner = dict(_delete_=True, type="IterBasedRunner", max_iters=5000)
checkpoint_config = dict(by_epoch=False, interval=1000, max_keep_ckpts=2)
fp16 = dict(loss_scale="dynamic")
log_config = dict(
interval=49,
hooks=[
dict(type="TextLoggerHook", by_epoch=False),
dict(
type="WandbLoggerHook",
init_kwargs=dict(
project="pre_release",
name="${cfg_name}",
config=dict(
work_dirs="${work_dir}",
total_step="${runner.max_iters}",
),
),
by_epoch=False,
),
],
)
watermellon2018
commented
3 years ago @jessicametzger I have same problem. Did you solve this?
jessicametzger
commented
3 years ago @watermellon2018 I was able to to fix it by setting by_prob=False in the sampler config. So the bug is somewhere in here.
tahirashehzadi
commented
2 years ago @winnerziqi how you solved this issue? assert len(indices) == len(self) I am getting the same error
alaa-shubbak
commented
2 years ago where did you set self.by_prob =false exactly .. in the code its a part of if loop can you please explain it more ?
alaa-shubbak
commented
2 years ago i got it. in my config, it already set to False . and i still have same problem .. what shall i do ?
xiangtaowong
commented
2 years ago i got it. in my config, it already set to False . and i still have same problem .. what shall i do ?
how did u solve the problem?
alaa-shubbak
commented
2 years ago d u solve the proble
my problem was in the present of mask annotaion, while my dataset dose not have any mask information and annotaion. I removed any part related to mask , and try to train the model gain . for example , i removed ("gt_masks") in this
i hope this will help you.
xiangtaowong
commented
2 years ago thank u, I'll have a try.
hello, When I use it, raise error: "assert len(indices) == len(self), f"{indices} not equal {len(self)} while offset is: {offset}"" then I print the length info, =====len of indices is 26865 - offset: 0 - len self 36650 below is the detail error info, Please help me.
Traceback (most recent call last): File "tools/train.py", line 198, in <module> main() File "tools/train.py", line 193, in main meta=meta, File "/data6/ziqiwen/code/softteacher/ssod/apis/train.py", line 206, in train_detector runner.run(data_loaders, cfg.workflow) File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/mmcv/runner/iter_based_runner.py", line 117, in run iter_loaders = [IterLoader(x) for x in data_loaders] File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/mmcv/runner/iter_based_runner.py", line 117, in <listcomp> iter_loaders = [IterLoader(x) for x in data_loaders] File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/mmcv/runner/iter_based_runner.py", line 23, in __init__ self.iter_loader = iter(self._dataloader) File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 291, in __iter__ return _MultiProcessingDataLoaderIter(self) File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 764, in __init__ self._try_put_index() File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 994, in _try_put_index index = self._next_index() File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 357, in _next_index return next(self._sampler_iter) # may raise StopIteration File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/torch/utils/data/sampler.py", line 208, in __iter__ for idx in self.sampler: File "/data6/ziqiwen/code/softteacher/ssod/datasets/samplers/semi_sampler.py", line 189, in __iter__ assert len(indices) == len(self) AssertionError Traceback (most recent call last): File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/runpy.py", line 193, in _run_module_as_main "__main__", mod_spec) File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/runpy.py", line 85, in _run_code exec(code, run_globals) File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/torch/distributed/launch.py", line 261, in <module> main() File "/home/ziqiwen/anaconda3/envs/mm/lib/python3.7/site-packages/torch/distributed/launch.py", line 257, in main cmd=cmd)