search

open-mmlab / mmdetection

OpenMMLab Detection Toolbox and Benchmark
https://mmdetection.readthedocs.io
Apache License 2.0
28.95k stars 9.36k forks source link

WandbVisBackend creates experiment but logs nothing. #10226

Open ihoholko opened 1 year ago

ihoholko commented 1 year ago

Tensorboard logs metrics. Wandb creates experiment and does nothing. Config:

04/26 11:54:39 - mmengine - INFO - Config:
default_scope = 'mmdet'
default_hooks = dict(
    timer=dict(type='IterTimerHook'),
    logger=dict(type='LoggerHook', interval=50),
    param_scheduler=dict(type='ParamSchedulerHook'),
    checkpoint=dict(type='CheckpointHook', interval=10, max_keep_ckpts=3),
    sampler_seed=dict(type='DistSamplerSeedHook'),
    visualization=dict(type='DetVisualizationHook'))
env_cfg = dict(
    cudnn_benchmark=False,
    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0),
    dist_cfg=dict(backend='nccl'))
vis_backends = [
    dict(type='LocalVisBackend'),
    dict(
        type='WandbVisBackend',
        save_dir=
        '/home/oem/Projects/aimosaic-roi/logs/Mosaic-roi-temp/train/runs/2023-04-26_11-54-37',
        init_kwargs=dict(
            project='Mosaic-roi-temp',
            name='rtmdet_m_8xb32-300e_coco',),
        define_metric_cfg=None,
        commit=True,
        log_code_name=None,
        watch_kwargs=None),
    dict(type='TensorboardVisBackend')
]
visualizer = dict(
    type='DetLocalVisualizer',
    vis_backends=[
        dict(type='LocalVisBackend'),
        dict(
            type='WandbVisBackend',
            save_dir=
            '/home/oem/Projects/aimosaic-roi/logs/Mosaic-roi-temp/train/runs/2023-04-26_11-54-37',
            init_kwargs=dict(
                project='Mosaic-roi-temp',
                name='rtmdet_m_8xb32-300e_coco',
                interval=1),
            define_metric_cfg=None,
            commit=True,
            log_code_name=None,
            watch_kwargs=None),
        dict(type='TensorboardVisBackend')
    ],
    name='visualizer')
log_processor = dict(type='LogProcessor', window_size=50, by_epoch=True)
log_level = 'INFO'
load_from = None
resume = False
train_cfg = dict(
    type='EpochBasedTrainLoop',
    max_epochs=50,
    val_interval=1,
    dynamic_intervals=[(280, 1)])
val_cfg = dict(type='ValLoop')
test_cfg = dict(type='TestLoop')
param_scheduler = [
    dict(
        type='LinearLR', start_factor=1e-05, by_epoch=False, begin=0,
        end=1000),
    dict(
        type='CosineAnnealingLR',
        eta_min=0.0002,
        begin=150,
        end=300,
        T_max=150,
        by_epoch=True,
        convert_to_iter_based=True)
]
optim_wrapper = dict(
    type='OptimWrapper',
    optimizer=dict(type='AdamW', lr=0.004, weight_decay=0.05),
    paramwise_cfg=dict(
        norm_decay_mult=0, bias_decay_mult=0, bypass_duplicate=True))
auto_scale_lr = dict(enable=False, base_batch_size=16)
dataset_type = 'CocoDataset'
data_root = '/home/oem/Projects/aimosaic-roi/data/aimosaic-seg'
backend_args = None
train_pipeline = [
    dict(type='LoadImageFromFile', backend_args=None),
    dict(type='LoadAnnotations', with_bbox=True),
    dict(type='CachedMosaic', img_scale=(640, 640), pad_val=114.0),
    dict(
        type='RandomResize',
        scale=(1280, 1280),
        ratio_range=(0.1, 2.0),
        keep_ratio=True),
    dict(type='RandomCrop', crop_size=(640, 640)),
    dict(type='YOLOXHSVRandomAug'),
    dict(type='RandomFlip', prob=0.5),
    dict(type='Pad', size=(640, 640), pad_val=dict(img=(114, 114, 114))),
    dict(
        type='CachedMixUp',
        img_scale=(640, 640),
        ratio_range=(1.0, 1.0),
        max_cached_images=20,
        pad_val=(114, 114, 114)),
    dict(type='PackDetInputs')
]
test_pipeline = [
    dict(type='LoadImageFromFile', backend_args=None),
    dict(type='Resize', scale=(640, 640), keep_ratio=True),
    dict(type='Pad', size=(640, 640), pad_val=dict(img=(114, 114, 114))),
    dict(
        type='PackDetInputs',
        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
                   'scale_factor'))
]
train_dataloader = dict(
    batch_size=20,
    num_workers=10,
    persistent_workers=True,
    sampler=dict(type='DefaultSampler', shuffle=True),
    batch_sampler=None,
    dataset=dict(
        type='CocoDataset',
        data_root='/home/oem/Projects/aimosaic-roi/data/aimosaic-seg',
        ann_file='equifish_annotations/traindataset_toy.json',
        data_prefix=dict(img='equifish/'),
        filter_cfg=dict(filter_empty_gt=True, min_size=32),
        pipeline=[
            dict(type='LoadImageFromFile', backend_args=None),
            dict(type='LoadAnnotations', with_bbox=True),
            dict(type='CachedMosaic', img_scale=(640, 640), pad_val=114.0),
            dict(
                type='RandomResize',
                scale=(1280, 1280),
                ratio_range=(0.1, 2.0),
                keep_ratio=True),
            dict(type='RandomCrop', crop_size=(640, 640)),
            dict(type='YOLOXHSVRandomAug'),
            dict(type='RandomFlip', prob=0.5),
            dict(
                type='Pad', size=(640, 640),
                pad_val=dict(img=(114, 114, 114))),
            dict(
                type='CachedMixUp',
                img_scale=(640, 640),
                ratio_range=(1.0, 1.0),
                max_cached_images=20,
                pad_val=(114, 114, 114)),
            dict(type='PackDetInputs')
        ],
        backend_args=None,
        metainfo=dict(classes=['Anal', 'Censor'])),
    pin_memory=True)
val_dataloader = dict(
    batch_size=5,
    num_workers=10,
    persistent_workers=True,
    drop_last=False,
    sampler=dict(type='DefaultSampler', shuffle=False),
    dataset=dict(
        type='CocoDataset',
        data_root='/home/oem/Projects/aimosaic-roi/data/aimosaic-seg',
        ann_file='equifish_annotations/valdataset_toy.json',
        data_prefix=dict(img='equifish/'),
        test_mode=True,
        pipeline=[
            dict(type='LoadImageFromFile', backend_args=None),
            dict(type='Resize', scale=(640, 640), keep_ratio=True),
            dict(
                type='Pad', size=(640, 640),
                pad_val=dict(img=(114, 114, 114))),
            dict(
                type='PackDetInputs',
                meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
                           'scale_factor'))
        ],
        backend_args=None,
        metainfo=dict(classes=['Anal', 'Censor'])))
test_dataloader = dict(
    batch_size=5,
    num_workers=10,
    persistent_workers=True,
    drop_last=False,
    sampler=dict(type='DefaultSampler', shuffle=False),
    dataset=dict(
        type='CocoDataset',
        data_root='/home/oem/Projects/aimosaic-roi/data/aimosaic-seg',
        ann_file='equifish_annotations/testdataset_toy.json',
        data_prefix=dict(img='equifish/'),
        test_mode=True,
        pipeline=[
            dict(type='LoadImageFromFile', backend_args=None),
            dict(type='Resize', scale=(640, 640), keep_ratio=True),
            dict(
                type='Pad', size=(640, 640),
                pad_val=dict(img=(114, 114, 114))),
            dict(
                type='PackDetInputs',
                meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
                           'scale_factor'))
        ],
        backend_args=None,
        metainfo=dict(classes=['Anal', 'Censor'])))
val_evaluator = dict(
    type='CocoMetric',
    ann_file=
    '/home/oem/Projects/aimosaic-roi/data/aimosaic-seg/equifish_annotations/valdataset_toy.json',
    metric=['bbox', 'proposal'],
    format_only=False,
    backend_args=None,
    proposal_nums=(100, 1, 10),
    iou_thrs=[0.2])
test_evaluator = dict(
    type='CocoMetric',
    ann_file=
    '/home/oem/Projects/aimosaic-roi/data/aimosaic-seg/equifish_annotations/testdataset_toy.json',
    metric=['bbox', 'proposal'],
    format_only=False,
    backend_args=None,
    proposal_nums=(100, 1, 10),
    iou_thrs=[0.2])
tta_model = dict(
    type='DetTTAModel',
    tta_cfg=dict(nms=dict(type='nms', iou_threshold=0.6), max_per_img=100))
img_scales = [(640, 640), (320, 320), (960, 960)]
tta_pipeline = [
    dict(type='LoadImageFromFile', backend_args=None),
    dict(
        type='TestTimeAug',
        transforms=[[{
            'type': 'Resize',
            'scale': (640, 640),
            'keep_ratio': True
        }, {
            'type': 'Resize',
            'scale': (320, 320),
            'keep_ratio': True
        }, {
            'type': 'Resize',
            'scale': (960, 960),
            'keep_ratio': True
        }],
                    [{
                        'type': 'RandomFlip',
                        'prob': 1.0
                    }, {
                        'type': 'RandomFlip',
                        'prob': 0.0
                    }],
                    [{
                        'type': 'Pad',
                        'size': (960, 960),
                        'pad_val': {
                            'img': (114, 114, 114)
                        }
                    }],
                    [{
                        'type':
                        'PackDetInputs',
                        'meta_keys':
                        ('img_id', 'img_path', 'ori_shape', 'img_shape',
                         'scale_factor', 'flip', 'flip_direction')
                    }]])
]
model = dict(
    type='RTMDet',
    data_preprocessor=dict(
        type='DetDataPreprocessor',
        mean=[103.53, 116.28, 123.675],
        std=[57.375, 57.12, 58.395],
        bgr_to_rgb=False,
        batch_augments=None),
    backbone=dict(
        type='CSPNeXt',
        arch='P5',
        expand_ratio=0.5,
        deepen_factor=0.67,
        widen_factor=0.75,
        channel_attention=True,
        norm_cfg=dict(type='SyncBN'),
        act_cfg=dict(type='SiLU', inplace=True)),
    neck=dict(
        type='CSPNeXtPAFPN',
        in_channels=[192, 384, 768],
        out_channels=192,
        num_csp_blocks=2,
        expand_ratio=0.5,
        norm_cfg=dict(type='SyncBN'),
        act_cfg=dict(type='SiLU', inplace=True)),
    bbox_head=dict(
        type='RTMDetSepBNHead',
        num_classes=2,
        in_channels=192,
        stacked_convs=2,
        feat_channels=192,
        anchor_generator=dict(
            type='MlvlPointGenerator', offset=0, strides=[8, 16, 32]),
        bbox_coder=dict(type='DistancePointBBoxCoder'),
        loss_cls=dict(
            type='QualityFocalLoss',
            use_sigmoid=True,
            beta=2.0,
            loss_weight=1.0),
        loss_bbox=dict(type='GIoULoss', loss_weight=2.0),
        with_objectness=False,
        exp_on_reg=True,
        share_conv=True,
        pred_kernel_size=1,
        norm_cfg=dict(type='SyncBN'),
        act_cfg=dict(type='SiLU', inplace=True)),
    train_cfg=dict(
        assigner=dict(type='DynamicSoftLabelAssigner', topk=13),
        allowed_border=-1,
        pos_weight=-1,
        debug=False),
    test_cfg=dict(
        nms_pre=30000,
        min_bbox_size=0,
        score_thr=0.001,
        nms=dict(type='nms', iou_threshold=0.65),
        max_per_img=300))
train_pipeline_stage2 = [
    dict(type='LoadImageFromFile', backend_args=None),
    dict(type='LoadAnnotations', with_bbox=True),
    dict(
        type='RandomResize',
        scale=(640, 640),
        ratio_range=(0.1, 2.0),
        keep_ratio=True),
    dict(type='RandomCrop', crop_size=(640, 640)),
    dict(type='YOLOXHSVRandomAug'),
    dict(type='RandomFlip', prob=0.5),
    dict(type='Pad', size=(640, 640), pad_val=dict(img=(114, 114, 114))),
    dict(type='PackDetInputs')
]
max_epochs = 300
stage2_num_epochs = 20
base_lr = 0.004
interval = 10
custom_hooks = None
optimize_metric = 'coco/bbox_mAP'
optimize_direction = 'greater'
_base_ = '/home/oem/Projects/aimosaic-roi/3rd/mmdetection/configs/rtmdet/rtmdet_m_8xb32-300e_coco.py'
name = 'Mosaic-roi-temp'
task_name = 'train'
run_name = 'rtmdet_m_8xb32-300e_coco'
tags = ['ai-mosaic', 'rtmdet']
train = True
test = True
compile = False
seed = 12345
mode = 'mmdet'
metainfo = dict(classes=['Anal', 'Censor'])
modules = dict(
    LocalVisBackend=dict(type='LocalVisBackend'),
    TensorboardVisBackend=dict(type='TensorboardVisBackend'),
    MLflowVisBackend=dict(
        type='MLflowVisBackend',
        save_dir=None,
        exp_name='Mosaic-roi-temp',
        run_name='rtmdet_m_8xb32-300e_coco',
        tags=['ai-mosaic', 'rtmdet'],
        params=None,
        tracking_uri='http://mlflow:mm',
        artifact_suffix=['.json', '.log', '.py', 'yaml']),
    WandbVisBackend=dict(
        type='WandbVisBackend',
        save_dir=
        '/home/oem/Projects/aimosaic-roi/logs/Mosaic-roi-temp/train/runs/2023-04-26_11-54-37',
        init_kwargs=dict(
            project='Mosaic-roi-temp',
            name='rtmdet_m_8xb32-300e_coco',
            interval=1),
        define_metric_cfg=None,
        commit=True,
        log_code_name=None,
        watch_kwargs=None),
    EarlyStoppingHook=dict(
        type='EarlyStoppingHook',
        monitor='coco/bbox_mAP',
        rule='greater',
        min_delta=0.01,
        strict=False,
        check_finite=True,
        patience=1,
        stopping_threshold=None),
    CheckpointHook=dict(
        type='CheckpointHook',
        interval=1,
        by_epoch=True,
        save_optimizer=True,
        save_param_scheduler=True,
        out_dir=None,
        max_keep_ckpts=4,
        save_last=True,
        save_best='coco/bbox_mAP',
        rule='greater',
        greater_keys=None,
        less_keys=None,
        file_client_args=None,
        filename_tmpl=None,
        backend_args=None,
        published_keys=None),
    EMAHook=dict(
        type='EMAHook',
        ema_type='ExponentialMovingAverage',
        strict_load=False,
        begin_iter=0,
        begin_epoch=0),
    NaiveVisualizationHook=dict(
        type='NaiveVisualizationHook',
        interval=20,
        draw_gt=True,
        draw_pred=True),
    YOLOXModeSwitchHook=dict(
        type='YOLOXModeSwitchHook',
        num_last_epochs=15,
        skip_type_keys=['Mosaic', 'RandomAffine', 'MixUp']),
    DetVisualizationHook=dict(
        type='DetVisualizationHook', interval=1, show=False))
paths = dict(
    root_dir='/home/oem/Projects/aimosaic-roi',
    data_dir='/home/oem/Projects/aimosaic-roi/data',
    log_dir='/home/oem/Projects/aimosaic-roi/logs',
    output_dir=
    '/home/oem/Projects/aimosaic-roi/logs/Mosaic-roi-temp/train/runs/2023-04-26_11-54-37',
    work_dir='/home/oem/Projects/aimosaic-roi',
    mmdet_dir='/home/oem/Projects/aimosaic-roi/3rd/mmdetection')
extras = dict(ignore_warnings=False, enforce_tags=True, print_config=True)
work_dir = '/home/oem/Projects/aimosaic-roi/logs/Mosaic-roi-temp/train/runs/2023-04-26_11-54-37'
hhaAndroid commented 1 year ago

@ihoholko Thank you for your feedback, I have run it and found that it exists, can you double-check?

Roger-F commented 1 year ago

Same problem here with TensorBoard and Wandb backends. They don't create the log files.

tall-josh commented 1 year ago

~I'm having the same issue here too. Have even tried running with wandb in offline mode then syncing but the sync just hangs.~

Scratch that. Mine is now working now that I set the save_dir arg. Looks like @ihoholko has already done that though.