About the new version of the code.

[zyrant]

First of all, thank you for your work. it seems that you have update the version of pcdet from 0.3 to 0.5, and data_processor.py gets a new version. such as

        # if data_dict is None:
        #     try:
        #         from spconv.utils import VoxelGeneratorV2 as VoxelGenerator
        #     except:
        #         from spconv.utils import VoxelGenerator

        #     voxel_generator = VoxelGenerator(
        #         voxel_size=config.VOXEL_SIZE,
        #         point_cloud_range=self.point_cloud_range,
        #         max_num_points=config.MAX_POINTS_PER_VOXEL,
        #         max_voxels=config.MAX_NUMBER_OF_VOXELS[self.mode]
        #     )
        if data_dict is None:
            grid_size = (self.point_cloud_range[3:6] - self.point_cloud_range[0:3]) / np.array(config.VOXEL_SIZE)
            self.grid_size = np.round(grid_size).astype(np.int64)
            self.voxel_size = config.VOXEL_SIZE
            self.max_num_points = config.MAX_POINTS_PER_VOXEL

            return partial(
                self.transform_points_to_voxels, 
                dynamic = getattr(config, 'DYNAMIC', False)
            )

        points = data_dict['points']
        if dynamic:
            pass
        else:
            voxel_output = spconv.utils.VoxelGenerator(
                voxel_size= self.voxel_size, 
                point_cloud_range=self.point_cloud_range,
                max_num_points=self.max_num_points).generate(points)

            voxels, coordinates, num_points = voxel_output

            if not data_dict['use_lead_xyz']:
                voxels = voxels[..., 3:]  # remove xyz in voxels(N, 3)

            data_dict['voxels'] = voxels
            data_dict['voxel_coords'] = coordinates
            data_dict['voxel_num_points'] = num_points

        return data_dict

to

    def transform_points_to_voxels(self, data_dict=None, config=None):
        if data_dict is None:
            grid_size = (self.point_cloud_range[3:6] - self.point_cloud_range[0:3]) / np.array(config.VOXEL_SIZE)
            self.grid_size = np.round(grid_size).astype(np.int64)
            self.voxel_size = config.VOXEL_SIZE
            # just bind the config, we will create the VoxelGeneratorWrapper later,
            # to avoid pickling issues in multiprocess spawn
            return partial(self.transform_points_to_voxels, config=config)

        if self.voxel_generator is None:
            self.voxel_generator = VoxelGeneratorWrapper(
                vsize_xyz=config.VOXEL_SIZE,
                coors_range_xyz=self.point_cloud_range,
                num_point_features=self.num_point_features,
                max_num_points_per_voxel=config.MAX_POINTS_PER_VOXEL,
                max_num_voxels=config.MAX_NUMBER_OF_VOXELS[self.mode],
            )

        points = data_dict['points']

        if config.MAX_POINTS_PER_VOXEL == -1:
            return data_dict

        voxel_output = self.voxel_generator.generate(points)
        voxels, coordinates, num_points = voxel_output

        if not data_dict['use_lead_xyz']:
            voxels = voxels[..., 3:]  # remove xyz in voxels(N, 3)

        data_dict['voxels'] = voxels
        data_dict['voxel_coords'] = coordinates
        data_dict['voxel_num_points'] = num_points

        mask = np.tile(np.arange(config.MAX_POINTS_PER_VOXEL).reshape(1, -1), [len(num_points), 1]) < num_points.reshape(-1, 1)
        data_dict['points'] = voxels[mask, ...]

        return data_dict

when I try the latest version, I get a problem， so I change the code to

if config.MAX_POINTS_PER_VOXEL == -1:
            pass
        else:
            if self.voxel_generator is None:
                self.voxel_generator = VoxelGeneratorWrapper(
                    vsize_xyz=config.VOXEL_SIZE,
                    coors_range_xyz=self.point_cloud_range,
                    num_point_features=self.num_point_features,
                    max_num_points_per_voxel=config.MAX_POINTS_PER_VOXEL,
                    max_num_voxels=config.MAX_NUMBER_OF_VOXELS[self.mode],
                )
            voxel_output = self.voxel_generator.generate(points)
            voxels, coordinates, num_points = voxel_output

and it works. Then I got a new bug again.

/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:97: operator(): block: [510,0,0], thread: [127,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
epochs:   0%|                                                                                                                                       | 0/100 [00:03<?, ?it/s]
Traceback (most recent call last):
  File "./tools/train.py", line 202, in <module>
    main()
  File "./tools/train.py", line 154, in main
    train_model(
  File "/opt/data/private/OpenPCDet/tools/train_utils/train_utils.py", line 111, in train_model
    accumulated_iter = train_one_epoch(
  File "/opt/data/private/OpenPCDet/tools/train_utils/train_utils.py", line 47, in train_one_epoch
    loss, tb_dict, disp_dict = model_func(model, batch)
  File "/opt/data/private/OpenPCDet/pcdet/models/__init__.py", line 42, in model_func
    ret_dict, tb_dict, disp_dict = model(batch_dict)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "/opt/data/private/OpenPCDet/pcdet/models/detectors/pointpillar.py", line 11, in forward
    batch_dict = cur_module(batch_dict)
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "/opt/data/private/OpenPCDet/pcdet/models/backbones_3d/vfe/voxset.py", line 117, in forward
    src = self.mlp_vsa_layer_0(src, inverse01x, coords01x, self.grid_size)  # VSA
  File "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py", line 1051, in _call_impl
    return forward_call(*input, **kwargs)
  File "/opt/data/private/OpenPCDet/pcdet/models/backbones_3d/vfe/voxset.py", line 196, in forward
    h = spconv.SparseConvTensor(F.relu(x_), coords.int(), bev_shape, batch_size).dense().squeeze(-1) # hr~
  File "/usr/local/lib/python3.8/dist-packages/spconv/pytorch/core.py", line 222, in dense
    res = scatter_nd(
  File "/usr/local/lib/python3.8/dist-packages/spconv/pytorch/core.py", line 118, in scatter_nd
    ret[slices] = updates.view(*output_shape)
RuntimeError: CUDA error: device-side assert triggered

I have tried my best to solve this problem, but get noting, can you give me some advice? Actually, when i use your code with pcdet 0.3，everything is okay.

[skyhehe123]

@zyrant This error usually implies the coordinates (coords01x) being out of the point cloud range. You can print the coords[:, 0].max(), coords[:, 1].max(), coords[:, 2].max() in the debug console.

[zyrant]

thank you for your reply. I print the coords use

print( '0max:',coords.int()[:,0].max(), '1max: ',coords.int()[:,1].max(),  '2max: ', coords.int()[:,2].max(),'3max: ', coords.int()[:,3].max())
print('bev_shape : ',bev_shape)

and get the results

0max: tensor(1, device='cuda:0', dtype=torch.int32) 1max:  tensor(215, device='cuda:0', dtype=torch.int32) 2max:  tensor(232, device='cuda:0', dtype=torch.int32) 3max:  tensor(1, device='cuda:0', dtype=torch.int32)
bev_shape :  [216, 248, 1]

[skyhehe123]

thank you for your reply. I print the coords use

print( '0max:',coords.int()[:,0].max(), '1max: ',coords.int()[:,1].max(),  '2max: ', coords.int()[:,2].max(),'3max: ', coords.int()[:,3].max())
print('bev_shape : ',bev_shape)

and get the results

0max: tensor(1, device='cuda:0', dtype=torch.int32) 1max:  tensor(215, device='cuda:0', dtype=torch.int32) 2max:  tensor(232, device='cuda:0', dtype=torch.int32) 3max:  tensor(1, device='cuda:0', dtype=torch.int32)
bev_shape :  [216, 248, 1]

It seems some points fall out of the range on Z-axis(height). The coords.int()[:, 3].max() should be 0 since we got only one voxel (acutually a pillar) for each XY position.

[zyrant]

Yes， you are right. I use the official code of openPCDet-master. i find the _mask_points_byrange function in common_utils.py is

def mask_points_by_range(points, limit_range):
    mask = (points[:, 0] >= limit_range[0]) & (points[:, 0] <= limit_range[3]) \
           & (points[:, 1] >= limit_range[1]) & (points[:, 1] <= limit_range[4])
    return mask

It's strange that it doesn't limit the range of the point cloud on the Z axis. So I added a z-axis constraint based on your code.

def mask_points_by_range(points, limit_range):
    # mask = (points[:, 0] >= limit_range[0]) & (points[:, 0] <= limit_range[3]) \
    #        & (points[:, 1] >= limit_range[1]) & (points[:, 1] <= limit_range[4])
    mask = (points[:, 0] > limit_range[0] + 1e-4) & (points[:, 0] < limit_range[3] - 1e-4) \
           & (points[:, 1] > limit_range[1] + 1e-4) & (points[:, 1] < limit_range[4] - 1e-4) \
           & (points[:, 2] > limit_range[2] + 1e-4) & (points[:, 2] < limit_range[5] - 1e-4)
    return mask

Last, everything is okay, and thank you for your answer.