How to get train augmentation output images

[28djs]

I am trying to get the input to the model, the augmented train images. Is there a way to do it? I have followed this - https://github.com/open-mmlab/mmsegmentation/issues/1710 but get this error "AttributeError: 'MMSegVisualizationHook' object has no attribute 'every_n_iters'"

[28djs]

Config - TorchVision: 0.12.0 OpenCV: 4.7.0 MMEngine: 0.7.3

Runtime environment: cudnn_benchmark: True mp_cfg: {'mp_start_method': 'fork', 'opencv_num_threads': 0} dist_cfg: {'backend': 'nccl'} seed: None Distributed launcher: none Distributed training: False GPU number: 1

06/06 11:06:20 - mmengine - INFO - Config: norm_cfg = dict(type='SyncBN', requires_grad=True) data_preprocessor = dict( type='SegDataPreProcessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_val=0, seg_pad_val=255, size=(224, 224)) model = dict( type='EncoderDecoder', data_preprocessor=dict( type='SegDataPreProcessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_val=0, seg_pad_val=255, size=(224, 224)), pretrained=None, backbone=dict( type='UNet', in_channels=3, base_channels=64, num_stages=5, strides=(1, 1, 1, 1, 1), enc_num_convs=(2, 2, 2, 2, 2), dec_num_convs=(2, 2, 2, 2), downsamples=(True, True, True, True), enc_dilations=(1, 1, 1, 1, 1), dec_dilations=(1, 1, 1, 1), with_cp=False, conv_cfg=None, norm_cfg=dict(type='SyncBN', requires_grad=True), act_cfg=dict(type='ReLU'), upsample_cfg=dict(type='InterpConv'), norm_eval=False), decode_head=dict( type='FCNHead', in_channels=64, in_index=4, channels=64, num_convs=1, concat_input=False, dropout_ratio=0.1, num_classes=2, norm_cfg=dict(type='SyncBN', requires_grad=True), align_corners=False, loss_decode=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), auxiliary_head=dict( type='FCNHead', in_channels=128, in_index=3, channels=64, num_convs=1, concat_input=False, dropout_ratio=0.1, num_classes=2, norm_cfg=dict(type='SyncBN', requires_grad=True), align_corners=False, loss_decode=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), train_cfg=dict(), test_cfg=dict(mode='slide', crop_size=(224, 224), stride=(85, 85))) dataset_type = 'EBHIDataset' data_root = 'data/EBHI_Adenocarcinoma' crop_size = (224, 224) custom_imports = dict(imports='mmcls.models', allow_failed_imports=True) train_pipeline = [ dict(type='LoadImageFromFile'), dict(type='LoadAnnotations', reduce_zero_label=False), dict( type='RandomResize', scale=(448, 448), ratio_range=(0.5, 2.0), keep_ratio=True), dict(type='RandomCrop', crop_size=(224, 224), cat_max_ratio=0.75), dict(type='RandomFlip', prob=0.5), dict(type='PhotoMetricDistortion'), dict(type='PackSegInputs') ] test_pipeline = [ dict(type='LoadImageFromFile'), dict(type='Resize', scale=(448, 448), keep_ratio=True), dict(type='LoadAnnotations', reduce_zero_label=False), dict(type='PackSegInputs') ] img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75] tta_pipeline = [ dict(type='LoadImageFromFile', backend_args=None), dict( type='TestTimeAug', transforms=[[{ 'type': 'Resize', 'scale_factor': 0.5, 'keep_ratio': True }, { 'type': 'Resize', 'scale_factor': 0.75, 'keep_ratio': True }, { 'type': 'Resize', 'scale_factor': 1.0, 'keep_ratio': True }, { 'type': 'Resize', 'scale_factor': 1.25, 'keep_ratio': True }, { 'type': 'Resize', 'scale_factor': 1.5, 'keep_ratio': True }, { 'type': 'Resize', 'scale_factor': 1.75, 'keep_ratio': True }], [{ 'type': 'RandomFlip', 'prob': 0.0, 'direction': 'horizontal' }, { 'type': 'RandomFlip', 'prob': 1.0, 'direction': 'horizontal' }], [{ 'type': 'LoadAnnotations' }], [{ 'type': 'PackSegInputs' }]]) ] train_dataloader = dict( batch_size=16, num_workers=4, persistent_workers=True, sampler=dict(type='InfiniteSampler', shuffle=True), dataset=dict( type='EBHIDataset', data_root='data/EBHI_Adenocarcinoma', data_prefix=dict( img_path='images/training', seg_map_path='annotations/training'), pipeline=[ dict(type='LoadImageFromFile'), dict(type='LoadAnnotations', reduce_zero_label=False), dict( type='RandomResize', scale=(448, 448), ratio_range=(0.5, 2.0), keep_ratio=True), dict(type='RandomCrop', crop_size=(224, 224), cat_max_ratio=0.75), dict(type='RandomFlip', prob=0.5), dict(type='PhotoMetricDistortion'), dict(type='PackSegInputs') ])) val_dataloader = dict( batch_size=1, num_workers=4, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type='EBHIDataset', data_root='data/EBHI_Adenocarcinoma', data_prefix=dict( img_path='images/validation', seg_map_path='annotations/validation'), pipeline=[ dict(type='LoadImageFromFile'), dict(type='Resize', scale=(448, 448), keep_ratio=True), dict(type='LoadAnnotations', reduce_zero_label=False), dict(type='PackSegInputs') ])) test_dataloader = dict( batch_size=1, num_workers=4, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type='EBHIDataset', data_root='data/EBHI_Adenocarcinoma', data_prefix=dict( img_path='images/test', seg_map_path='annotations/test'), pipeline=[ dict(type='LoadImageFromFile'), dict(type='Resize', scale=(448, 448), keep_ratio=True), dict(type='LoadAnnotations', reduce_zero_label=False), dict(type='PackSegInputs') ])) val_evaluator = dict( type='IoUMetric', iou_metrics=['mIoU', 'mDice', 'mFscore'], output_dir= '/host/Upgrad/Thesis/EBHI/MMSeg_output/VIT_MLN_Sun_Run_data_train_vis/EBHI_Adeno_with_test_dataloader_run_1_val' ) test_evaluator = dict( type='IoUMetric', iou_metrics=['mIoU', 'mDice', 'mFscore'], output_dir= '/host/Upgrad/Thesis/EBHI/MMSeg_output/VIT_MLN_Sun_Run_data_train_vis/EBHI_Adeno_with_test_dataloader_run_1_test' ) default_scope = 'mmseg' env_cfg = dict( cudnn_benchmark=True, mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), dist_cfg=dict(backend='nccl')) vis_backends = [dict(type='LocalVisBackend')] visualizer = dict( type='SegLocalVisualizer', vis_backends=[dict(type='LocalVisBackend')], name='visualizer') log_processor = dict(by_epoch=False) log_level = 'INFO' load_from = None resume = False tta_model = dict(type='SegTTAModel') optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005), clip_grad=None) param_scheduler = [ dict( type='PolyLR', eta_min=0.0001, power=0.9, begin=0, end=20000, by_epoch=False) ] train_cfg = dict(type='IterBasedTrainLoop', max_iters=20000, val_interval=2000) val_cfg = dict(type='ValLoop') test_cfg = dict(type='TestLoop') default_hooks = dict( timer=dict(type='IterTimerHook'), logger=dict(type='LoggerHook', interval=50, log_metric_by_epoch=False), param_scheduler=dict(type='ParamSchedulerHook'), checkpoint=dict(type='CheckpointHook', by_epoch=False, interval=2000), sampler_seed=dict(type='DistSamplerSeedHook'), visualization=dict(type='SegVisualizationHook')) custom_hooks = [ dict( type='MMSegVisualizationHook', output_dir= '/host/Upgrad/Thesis/EBHI/MMSeg_output/VIT_MLN_Sun_Run_data_train_vis/training_samples', bgr2rgb=True, interval=50) ] launcher = 'none' work_dir = '/host/Upgrad/Thesis/EBHI/MMSeg_Models_Sun_run/unet-s5-d16_fcn_4xb4-40k_chase-db1-128x128-EBHI_Adeno_Data_Augment_save/'

[28djs]

I have added a new hook file in - "mmsegmentation/mmseg/engine/hooks/visualize_training_samples.py"

Code in the file -

Copyright (c) OpenMMLab. All rights reserved.

import os.path as osp import warnings from typing import Optional, Sequence

import mmcv import mmengine.fileio as fileio from mmengine.hooks import Hook from mmengine.runner import Runner

from mmseg.registry import HOOKS from mmseg.structures import SegDataSample from mmseg.visualization import SegLocalVisualizer from torchvision.utils import save_image

@HOOKS.register_module() class MMSegVisualizationHook(Hook): """Visualization hook for unconditional GANs.

In this hook, we use the official api `save_image` in torchvision to save
the visualization results.

Args:
    output_dir (str): The file path to store visualizations.
    fixed_noise (bool, optional): Whether to use fixed noises in sampling.
        Defaults to True.
    num_samples (int, optional): The number of samples to show in
        visualization. Defaults to 16.
    interval (int): The interval of calling this hook. If set to -1,
        the visualization hook will not be called. Default: -1.
    filename_tmpl (str): Format string used to save images. The output file
        name will be formatted as this args. Default: 'iter_{}.png'.
    rerange (bool): Whether to rerange the output value from [-1, 1] to
        [0, 1]. We highly recommend users should preprocess the
        visualization results on their own. Here, we just provide a simple
        interface. Default: True.
    bgr2rgb (bool): Whether to reformat the channel dimension from BGR to
        RGB. The final image we will save is following RGB style.
        Default: True.
    nrow (int): The number of samples in a row. Default: 1.
    padding (int): The number of padding pixels between each samples.
        Default: 4.
    kwargs (dict | None, optional): Key-word arguments for sampling
        function. Defaults to None.
"""

def __init__(self,
             output_dir,
             fixed_noise=True,
             num_samples=16,
             interval=-1,
             filename_tmpl='iter_{}.png',
             rerange=True,
             bgr2rgb=True,
             nrow=4,
             padding=0,
             kwargs=None):
    self.output_dir = output_dir
    self.fixed_noise = fixed_noise
    self.num_samples = num_samples
    self.interval = interval
    self.filename_tmpl = filename_tmpl
    self.bgr2rgb = bgr2rgb
    self.rerange = rerange
    self.nrow = nrow
    self.padding = padding

    # the sampling noise will be initialized by the first sampling.
    self.sampling_noise = None

    self.kwargs = kwargs if kwargs is not None else dict()

def _after_iter(self,
                runner: Runner,
                batch_idx: int,
                data_batch: dict,
                outputs: Sequence[SegDataSample],
                mode: str = 'val') -> None:
    """Run after every ``self.interval`` validation iterations.

    Args:
        runner (:obj:`Runner`): The runner of the validation process.
        batch_idx (int): The index of the current batch in the val loop.
        data_batch (dict): Data from dataloader.
        outputs (Sequence[:obj:`SegDataSample`]): Outputs from model.
        mode (str): mode (str): Current mode of runner. Defaults to 'val'.
    """
    if self.draw is False or mode == 'train':
        return

    if self.every_n_inner_iters(batch_idx, self.interval):
        for output in outputs:
            img_path = output.img_path
            img_bytes = fileio.get(
                img_path, backend_args=self.backend_args)
            img = mmcv.imfrombytes(img_bytes, channel_order='rgb')
            window_name = f'{mode}_{osp.basename(img_path)}'

            self._visualizer.add_datasample(
                window_name,
                img,
                data_sample=output,
                show=self.show,
                wait_time=self.wait_time,
                step=runner.iter)

# Modify after_train_iter to before_train_iter
#@master_only
def before_train_iter(self, runner: Runner, batch_idx: int,data_batch: dict):
    """The behavior before each train iteration.

    Args:
        runner (object): The runner.
    """
    if not self.every_n_iters(runner, self.interval):
        return
    # get images from data container
    imgs = runner.data_batch['img'].data
    img_metas = runner.data_batch['img_metas'].data[0]

    filename = self.filename_tmpl.format(runner.iter + 1)

    # img_list = [x for k, x in results.items() if k in self.res_name_list]
    img_list = [img for img in imgs]
    for img in img_list:
        print(img.shape)
    img_cat = torch.cat(img_list, dim=3).detach()
    if self.rerange:
        img_cat = ((img_cat + 1) / 2)
    if self.bgr2rgb:
        img_cat = img_cat[:, [2, 1, 0], ...]
    img_cat = img_cat.clamp_(0, 1)

    if not hasattr(self, '_out_dir'):
        self._out_dir = osp.join(runner.work_dir, self.output_dir)
    mmcv.mkdir_or_exist(self._out_dir)
    save_image(
        img_cat,
        osp.join(self._out_dir, filename),
        nrow=self.nrow,
        padding=self.padding)
    filename = filename.replace('.png', '_meta_infos.json')
    mmcv.dump(img_metas, osp.join(self._out_dir, filename))

[28djs]

@xiexinch can you please help on this.

[msp-96]

Try this tool: https://github.com/open-mmlab/mmsegmentation/blob/main/tools/misc/browse_dataset.py