how to generate blank_code.pkl

[Zhangjyhhh]

In Eval step 0: I change the parameter "self.data_infos = random.sample(self.data_infos, 50)" in the 62 line of UltraLiDAR_nusc_waymo/plugin/datasets/nusc_dataset.py. And run the command, i find there are no blank_code.pkl file

(ultralidar) jyzhang@sumig-System-Product-Name:~/mmdetection3d/UltraLiDAR_nusc_waymo$ python -m torch.distributed.launch --nproc_per_node=1 --master_port=29501 ./tools/test.py ./configs/ultralidar_nusc_static_blank_code.py ./configs/epoch_160.pth --launcher pytorch  --eval mIoU
/home/jyzhang/anaconda3/envs/ultralidar/lib/python3.8/site-packages/torch/distributed/launch.py:178: FutureWarning: The module torch.distributed.launch is deprecated
and will be removed in future. Use torchrun.
Note that --use_env is set by default in torchrun.
If your script expects `--local_rank` argument to be set, please
change it to read from `os.environ['LOCAL_RANK']` instead. See 
https://pytorch.org/docs/stable/distributed.html#launch-utility for 
further instructions

  warnings.warn(
plugin
work_dir:  /media/jyzhang/C0FED904FED8F39E/wok_dir_zjy/nusc_stage2
collecting samples...
collected 81 samples in 0.00s
collecting samples...
collected 81 samples in 0.00s
/home/jyzhang/anaconda3/envs/ultralidar/lib/python3.8/site-packages/torch/functional.py:445: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at  ../aten/src/ATen/native/TensorShape.cpp:2157.)
  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
load checkpoint from local path: ./configs/epoch_160.pth
[>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>] 50/50, 2.5 task/s, elapsed: 20s, ETA:     0sstart evaluation!
7.152557373046875e-06

[myc634]

Hi, as I didn't provide config ultralidar_nusc_static_blank_code.py in the repo, can you please provide the ultralidar_nusc_static_blank_code.py config to help me debug?

[Zhangjyhhh]

Hi, as I didn't provide config ultralidar_nusc_static_blank_code.py in the repo, can you please provide the ultralidar_nusc_static_blank_code.py config to help me debug?

Sure,why not, this file is created according to 'ultralidar_kitti360_static_blank_code.py', but some parameters is changged as same as 'ultralidar_nusc_gene.py' and 'ultralidar_nusc.py'.

model_type = 'static_blank_code'
_base_ = ['./_base_/default_runtime.py']
batch_size = 3
# Global
# If point cloud range is changed, the models should also change their point
# cloud range accordingly
#point_cloud_range = [-50.0, -50.0, -3.73, 50.0, 50.0, 2.27]
point_cloud_range = [-50.0, -50.0, -5, 50.0, 50.0, 3]
voxel_size = [0.15625, 0.15625, 0.15]

# For nuScenes we usually do 10-class detection
class_names = [
    "car",
    "truck",
    "construction_vehicle",
    "bus",
    "trailer",
    "barrier",
    "motorcycle",
    "bicycle",
    "pedestrian",
    "traffic_cone",
]

dataset_type = 'NuscDataset'
data_root = "/home/jyzhang/datasets/nuScenes/"

file_client_args = dict(backend='disk')

plugin = True
plugin_dir = 'plugin/'

model = dict(
    type='UltraLiDAR',
    model_type=model_type,
    dataset_type=dataset_type,
    pts_bbox_head=dict(
        type='CenterHead',
        in_channels=256,
        tasks=[
            dict(num_class=1, class_names=['car']),
            dict(num_class=2, class_names=['truck', 'construction_vehicle']),
            dict(num_class=2, class_names=['bus', 'trailer']),
            dict(num_class=1, class_names=['barrier']),
            dict(num_class=2, class_names=['motorcycle', 'bicycle']),
            dict(num_class=2, class_names=['pedestrian', 'traffic_cone']),
        ],
        common_heads=dict(
            reg=(2, 2), height=(1, 2), dim=(3, 2), rot=(2, 2), vel=(2, 2)),
        share_conv_channel=64,
        bbox_coder=dict(
            type='CenterPointBBoxCoder',
            pc_range=point_cloud_range[:2],
            post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0],
            max_num=500,
            score_threshold=0.1,
            out_size_factor=8,
            voxel_size=voxel_size[:2],
            code_size=9),
        separate_head=dict(
            type='SeparateHead', init_bias=-2.19, final_kernel=3),
        loss_cls=dict(type='GaussianFocalLoss', reduction='mean'),
        loss_bbox=dict(type='L1Loss', reduction='mean', loss_weight=0.25),
        norm_bbox=True),
    voxelizer=dict(
        type='Voxelizer',
        x_min=point_cloud_range[0],
        x_max=point_cloud_range[3],
        y_min=point_cloud_range[1],
        y_max=point_cloud_range[4],
        z_min=point_cloud_range[2],
        z_max=point_cloud_range[5],
        step=voxel_size[0],
        z_step=voxel_size[2],
    ),
    vector_quantizer=dict(
        type='VectorQuantizer',
        n_e=1024,
        e_dim=1024,
        beta=0.25,
        cosine_similarity=False,
    ),
    maskgit_transformer=dict(
        type='BidirectionalTransformer',
        n_e=1024, 
        e_dim=1024, 
        img_size=80,
    ),
    lidar_encoder=dict(
        type='VQEncoder',
        img_size=640,
        codebook_dim=1024,
    ),
    lidar_decoder=dict(
        type='VQDecoder',
        img_size=(640, 640),
        num_patches=6400,
        codebook_dim=1024,
    ),
    # model training and testing settings
    train_cfg=dict(
        pts=dict(
            point_cloud_range=point_cloud_range,
            grid_size=[1024, 1024, 40],
            voxel_size=voxel_size,
            out_size_factor=8,
            dense_reg=1,
            gaussian_overlap=0.1,
            max_objs=500,
            min_radius=2,
            code_weights=[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])),
    test_cfg=dict(
        pts=dict(
            pc_range=point_cloud_range[:2],
            post_center_limit_range=point_cloud_range,
            max_per_img=500,
            max_pool_nms=False,
            min_radius=[4, 12, 10, 1, 0.85, 0.175],
            score_threshold=0.1,
            out_size_factor=8,
            voxel_size=voxel_size[:2],
            pre_max_size=1000,
            post_max_size=83,

            # Scale-NMS
            nms_type=[
                'rotate', 'rotate', 'rotate', 'circle', 'rotate', 'rotate'
            ],
            nms_thr=[0.2, 0.2, 0.2, 0.2, 0.2, 0.5],
            nms_rescale_factor=[
                1.0, [0.7, 0.7], [0.4, 0.55], 1.1, [1.0, 1.0], [4.5, 9.0]
            ])))

# Data

bda_aug_conf = dict(
    rot_lim=(-22.5, 22.5),
    scale_lim=(0.95, 1.05),
    flip_dx_ratio=0.5,
    flip_dy_ratio=0.5)

train_pipeline = [
    dict(
        type='LoadPointsFromFile',
        coord_type='LIDAR',
        load_dim=5,
        use_dim=5,
        #load_dim=4,
        #use_dim=4,
        file_client_args=file_client_args),

    dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True),
    dict(type='PointsRangeFilter', point_cloud_range=point_cloud_range),
    dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
    dict(type='ObjectNameFilter', classes=class_names),
    dict(type='DefaultFormatBundle3D', class_names=class_names),
    dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d'])
]

input_modality = dict(use_lidar=True, use_camera=False, use_radar=False, use_map=False, use_external=False)

share_data_config = dict(
    type=dataset_type,
    classes=class_names,
    modality=input_modality,
)

data = dict(
    samples_per_gpu=batch_size,
    workers_per_gpu=8,
    train=dict(
        type=dataset_type,
        data_root=data_root,
        ann_file=data_root + 'nuscenes_infos_train.pkl',
        pipeline=train_pipeline,
        classes=class_names,
        modality=input_modality,
        test_mode=False,
        # we use box_type_3d='LiDAR' in kitti and nuscenes dataset
        # and box_type_3d='Depth' in sunrgbd and scannet dataset.
        box_type_3d='LiDAR'),
    val=dict(
        type=dataset_type,
        data_root=data_root,
        ann_file=data_root + 'nuscenes_infos_val.pkl',
        pipeline=train_pipeline,
        classes=class_names,
        modality=input_modality,
        test_mode=True,
        box_type_3d='LiDAR'),
    test=dict(
        type=dataset_type,
        data_root=data_root,
        ann_file=data_root + 'nuscenes_infos_val.pkl', #'nuscenes_mapandbbox_infos_val.pkl
        pipeline=train_pipeline,
        classes=class_names,
        modality=input_modality,
        test_mode=True,
        box_type_3d='LiDAR'))

optimizer = dict(
    type='AdamW',
    lr=0.00024,
    betas=(0.9, 0.95),  # the momentum is change during training
    paramwise_cfg=dict(
        custom_keys={'absolute_pos_embed': dict(decay_mult=0.),
                     'relative_position_bias_table': dict(decay_mult=0.),
                     'norm': dict(decay_mult=0.),
                     'embedding': dict(decay_mult=0.),
                     'img_backbone': dict(lr_mult=0.1, decay_mult=0.001),
                    }),
        # {
        #     ''
        #     # 'img_backbone': dict(lr_mult=0.1),
        # }),
    weight_decay=0.0001)
optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
# optimizer_config = dict(type='GradientCumulativeOptimizerHook', cumulative_iters=16, grad_clip=dict(max_norm=500, norm_type=2))
lr_config = dict(policy="CosineAnnealing", warmup="linear", warmup_iters=2000, warmup_ratio=1.0 / 3, min_lr_ratio=1e-3)
runner = dict(type='EpochBasedRunner', max_epochs=160)

find_unused_parameters=True
checkpoint = "/media/jyzhang/C0FED904FED8F39E/wok_dir_zjy/epoch_80.pth"

work_dir='/media/jyzhang/C0FED904FED8F39E/wok_dir_zjy/nusc_stage2'
checkpoint_config = dict(interval=2)

[myc634]

Sorry, I can't reproduce this bug in my server, I guess adding import time;time.sleep(10) after https://github.com/myc634/UltraLiDAR_nusc_waymo/blob/9bff2fc9d9d67538dbb3f7517efc5ea61b960923/plugin/models/detectors/ultralidar.py#L424 might work.

[Zhangjyhhh]

Sorry, I can't reproduce this bug in my server, I guess adding import time;time.sleep(10) after https://github.com/myc634/UltraLiDAR_nusc_waymo/blob/9bff2fc9d9d67538dbb3f7517efc5ea61b960923/plugin/models/detectors/ultralidar.py#L424 might work.

Thanks for your reply, i will try it later