Closed robin2002 closed 5 years ago
Could you provide the log.txt in model_dir? I can't test multi-gpu code, so I need your help to fix this kind of bug.
(base) robin2002@robin2002-desktop:/media/ssd_2tb/myGit/second.pytorch/second$ CUDA_VISIBLE_DEVICES=0,1 python ./pytorch/train.py train --config_path=./configs/all.lite.nu.anchor_revised.config --model_dir=/media/ssd_2tb/myGit/second.pytorch/second/model/20190412_nu_all --multi_gpu=True [ 41 2000 2000] num_trainable parameters: 39 False _amp_stash MULTI-GPU: use 2 gpu feature_map_size [1, 250, 250] feature_map_size [1, 250, 250] WORKER 0 seed: 1555059006 WORKER 1 seed: 1555059007 WORKER 2 seed: 1555059008 model: { second: { voxel_generator { point_cloud_range : [-50, -50.0, -4, 50, 50, 2] voxel_size : [0.05, 0.05, 0.15] max_number_of_points_per_voxel : 1 }
voxel_feature_extractor: {
module_class_name: "SimpleVoxelRadius"
num_filters: [16]
with_distance: false
num_input_features: 3
}
middle_feature_extractor: {
module_class_name: "SpMiddleFHDLite"
# num_filters_down1: [] # protobuf don't support empty list.
# num_filters_down2: []
downsample_factor: 8
num_input_features: 2
}
rpn: {
module_class_name: "RPNV2"
layer_nums: [5]
layer_strides: [1]
num_filters: [128]
upsample_strides: [1]
num_upsample_filters: [128]
use_groupnorm: false
num_groups: 32
num_input_features: 128
}
loss: {
classification_loss: {
weighted_sigmoid_focal: {
alpha: 0.25
gamma: 2.0
anchorwise_output: true
}
}
localization_loss: {
weighted_smooth_l1: {
sigma: 3.0
code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
}
}
classification_weight: 1.0
localization_weight: 2.0
}
num_point_features: 3 # model's num point feature should be independent of dataset
# Outputs
use_sigmoid_score: true
encode_background_as_zeros: true
encode_rad_error_by_sin: true
use_direction_classifier: true # this can help for orientation benchmark
direction_loss_weight: 0.2 # enough.
# Loss
pos_class_weight: 1.0
neg_class_weight: 1.0
loss_norm_type: NormByNumPositives
# Postprocess
post_center_limit_range: [-50, -50, -4.0, 50, 50, 1.0]
use_rotate_nms: true
use_multi_class_nms: false
nms_pre_max_size: 1000
nms_post_max_size: 100
nms_score_threshold: 0.05 # 0.4 in submit, but 0.3 can get better hard performance
nms_iou_threshold: 0.5
box_coder: {
ground_box3d_coder: {
linear_dim: false
encode_angle_vector: false
}
}
target_assigner: {
anchor_generators: {
anchor_generator_range: {
sizes: [1.95017717, 4.60718145, 1.72270761] # wlh
anchor_ranges: [-50, -50.0, -0.93897414, 50, 50, -0.93897414]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "car"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.66344886, 0.7256437, 1.75748069] # wlh
anchor_ranges: [-50, -50.0, -0.73911038, 50, 50, -0.73911038]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "pedestrian"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.60058911, 1.68452161, 1.27192197] # wlh
anchor_ranges: [-50, -50.0, -1.03743013, 50, 50, -1.03743013]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "bicycle"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.76279481, 2.09973778, 1.44403034] # wlh
anchor_ranges: [-50, -50.0, -0.99194854, 50, 50, -0.99194854]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "motorcycle"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.4560939, 6.73778078, 2.73004906] # wlh
anchor_ranges: [-50, -50.0, -0.37937912, 50, 50, -0.37937912]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "truck"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.94046906, 11.1885991, 3.47030982] # wlh
anchor_ranges: [-50, -50.0, -0.0715754, 50, 50, -0.0715754]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "bus"
}
}
sample_positive_fraction : -1
sample_size : 512
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
} }
train_input_reader: { dataset: { dataset_class_name: "NuScenesDataset" kitti_info_path: "/media/ssd_2tb/dataset/nuscene/infos_train.pkl" kitti_root_path: "/media/ssd_2tb/dataset/nuscene"
# kitti_root_path: "/media/yy/960evo/datasets/nuscene/v1.0-mini"
}
batch_size: 6 preprocess: { max_number_of_voxels: 63000 shuffle_points: false num_workers: 3 groundtruth_localization_noise_std: [0, 0, 0] groundtruth_rotation_uniform_noise: [0, 0]
# groundtruth_localization_noise_std: [0.25, 0.25, 0.25]
# groundtruth_rotation_uniform_noise: [-0.3141592654, 0.3141592654]
# groundtruth_rotation_uniform_noise: [-0.78539816, 0.78539816]
global_rotation_uniform_noise: [-1.57, 1.57]
global_scaling_uniform_noise: [0.95, 1.05]
global_random_rotation_range_per_object: [0, 0] # pi/4 ~ 3pi/4
global_translate_noise_std: [0.2, 0.2, 0.2]
anchor_area_threshold: -1
remove_points_after_sample: true
groundtruth_points_drop_percentage: 0.0
groundtruth_drop_max_keep_points: 15
remove_unknown_examples: false
remove_environment: false
database_sampler {
# leave this empty to disable database_sampler, nuscenes don't need sample
# because 1. the number of ground-truth is enough. 2. sweeps don't support
# sample.
}
} }
train_config: { optimizer: { adam_optimizer: { learning_rate: { one_cycle: { lr_max: 3e-3 moms: [0.95, 0.85] div_factor: 10.0 pct_start: 0.4 } } weight_decay: 0.01 } fixed_weight_decay: true use_moving_average: false } steps: 234450 # 4689 50 (28130 // 6 + 1) steps_per_eval: 9378 # 4689 2
save_checkpoints_secs : 1800 # half hour save_summary_steps : 10 enable_mixed_precision: false loss_scale_factor: 8.0 clear_metrics_every_epoch: true }
eval_input_reader: { batch_size: 6 dataset: { dataset_class_name: "NuScenesDataset" kitti_info_path: "/media/ssd_2tb/dataset/nuscene/infos_val.pkl" kitti_root_path: "/media/ssd_2tb/dataset/nuscene" } preprocess: { max_number_of_voxels: 80000 shuffle_points: false num_workers: 3 anchor_area_threshold: -1 remove_environment: false } }
Traceback (most recent call last):
File "./pytorch/train.py", line 505, in
try the newest code, I don't know if this bug is fixed.
(base) robin2002@robin2002-desktop:/media/ssd_2tb/myGit/second.pytorch/second$ CUDA_VISIBLE_DEVICES=0,1 python ./pytorch/train.py train --config_path=./configs/all.lite.nu.anchor_revised.config --model_dir=/media/ssd_2tb/myGit/second.pytorch/second/model/20190412_nu_all --multi_gpu=True [ 41 2000 2000] num_trainable parameters: 39 False _amp_stash MULTI-GPU: use 2 gpu feature_map_size [1, 250, 250] feature_map_size [1, 250, 250] WORKER 0 seed: 1555069348 WORKER 1 seed: 1555069350 WORKER 2 seed: 1555069351 model: { second: { voxel_generator { point_cloud_range : [-50, -50.0, -4, 50, 50, 2] voxel_size : [0.05, 0.05, 0.15] max_number_of_points_per_voxel : 1 }
voxel_feature_extractor: {
module_class_name: "SimpleVoxelRadius"
num_filters: [16]
with_distance: false
num_input_features: 3
}
middle_feature_extractor: {
module_class_name: "SpMiddleFHDLite"
# num_filters_down1: [] # protobuf don't support empty list.
# num_filters_down2: []
downsample_factor: 8
num_input_features: 2
}
rpn: {
module_class_name: "RPNV2"
layer_nums: [5]
layer_strides: [1]
num_filters: [128]
upsample_strides: [1]
num_upsample_filters: [128]
use_groupnorm: false
num_groups: 32
num_input_features: 128
}
loss: {
classification_loss: {
weighted_sigmoid_focal: {
alpha: 0.25
gamma: 2.0
anchorwise_output: true
}
}
localization_loss: {
weighted_smooth_l1: {
sigma: 3.0
code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
}
}
classification_weight: 1.0
localization_weight: 2.0
}
num_point_features: 3 # model's num point feature should be independent of dataset
# Outputs
use_sigmoid_score: true
encode_background_as_zeros: true
encode_rad_error_by_sin: true
use_direction_classifier: true # this can help for orientation benchmark
direction_loss_weight: 0.2 # enough.
# Loss
pos_class_weight: 1.0
neg_class_weight: 1.0
loss_norm_type: NormByNumPositives
# Postprocess
post_center_limit_range: [-50, -50, -4.0, 50, 50, 1.0]
use_rotate_nms: true
use_multi_class_nms: false
nms_pre_max_size: 1000
nms_post_max_size: 100
nms_score_threshold: 0.05 # 0.4 in submit, but 0.3 can get better hard performance
nms_iou_threshold: 0.5
box_coder: {
ground_box3d_coder: {
linear_dim: false
encode_angle_vector: false
}
}
target_assigner: {
anchor_generators: {
anchor_generator_range: {
sizes: [1.95017717, 4.60718145, 1.72270761] # wlh
anchor_ranges: [-50, -50.0, -0.93897414, 50, 50, -0.93897414]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "car"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.66344886, 0.7256437, 1.75748069] # wlh
anchor_ranges: [-50, -50.0, -0.73911038, 50, 50, -0.73911038]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "pedestrian"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.60058911, 1.68452161, 1.27192197] # wlh
anchor_ranges: [-50, -50.0, -1.03743013, 50, 50, -1.03743013]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "bicycle"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.76279481, 2.09973778, 1.44403034] # wlh
anchor_ranges: [-50, -50.0, -0.99194854, 50, 50, -0.99194854]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "motorcycle"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.4560939, 6.73778078, 2.73004906] # wlh
anchor_ranges: [-50, -50.0, -0.37937912, 50, 50, -0.37937912]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "truck"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.94046906, 11.1885991, 3.47030982] # wlh
anchor_ranges: [-50, -50.0, -0.0715754, 50, 50, -0.0715754]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "bus"
}
}
sample_positive_fraction : -1
sample_size : 512
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
} }
train_input_reader: { dataset: { dataset_class_name: "NuScenesDataset" kitti_info_path: "/media/ssd_2tb/dataset/nuscene/infos_train.pkl" kitti_root_path: "/media/ssd_2tb/dataset/nuscene"
# kitti_root_path: "/media/yy/960evo/datasets/nuscene/v1.0-mini"
}
batch_size: 6 preprocess: { max_number_of_voxels: 63000 shuffle_points: false num_workers: 3 groundtruth_localization_noise_std: [0, 0, 0] groundtruth_rotation_uniform_noise: [0, 0]
# groundtruth_localization_noise_std: [0.25, 0.25, 0.25]
# groundtruth_rotation_uniform_noise: [-0.3141592654, 0.3141592654]
# groundtruth_rotation_uniform_noise: [-0.78539816, 0.78539816]
global_rotation_uniform_noise: [-1.57, 1.57]
global_scaling_uniform_noise: [0.95, 1.05]
global_random_rotation_range_per_object: [0, 0] # pi/4 ~ 3pi/4
global_translate_noise_std: [0.2, 0.2, 0.2]
anchor_area_threshold: -1
remove_points_after_sample: true
groundtruth_points_drop_percentage: 0.0
groundtruth_drop_max_keep_points: 15
remove_unknown_examples: false
remove_environment: false
database_sampler {
# leave this empty to disable database_sampler, nuscenes don't need sample
# because 1. the number of ground-truth is enough. 2. sweeps don't support
# sample.
}
} }
train_config: { optimizer: { adam_optimizer: { learning_rate: { one_cycle: { lr_max: 3e-3 moms: [0.95, 0.85] div_factor: 10.0 pct_start: 0.4 } } weight_decay: 0.01 } fixed_weight_decay: true use_moving_average: false } steps: 234450 # 4689 50 (28130 // 6 + 1) steps_per_eval: 9378 # 4689 2
save_checkpoints_secs : 1800 # half hour save_summary_steps : 10 enable_mixed_precision: false loss_scale_factor: 8.0 clear_metrics_every_epoch: true }
eval_input_reader: { batch_size: 6 dataset: { dataset_class_name: "NuScenesDataset" kitti_info_path: "/media/ssd_2tb/dataset/nuscene/infos_val.pkl" kitti_root_path: "/media/ssd_2tb/dataset/nuscene" } preprocess: { max_number_of_voxels: 80000 shuffle_points: false num_workers: 3 anchor_area_threshold: -1 remove_environment: false } }
THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch_1550802451070/work/aten/src/THC/generic/THCTensorMath.cu line=14 error=77 : an illegal memory access was encountered Traceback (most recent call last): File "./pytorch/train.py", line 259, in train ret_dict = net_parallel(example_torch) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call result = self.forward(*input, kwargs) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 143, in forward outputs = self.parallel_apply(replicas, inputs, kwargs) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 153, in parallel_apply return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)]) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 83, in parallel_apply raise output File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 59, in _worker output = module(*input, *kwargs) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call result = self.forward(input, kwargs) File "/media/ssd_2tb/myGit/second.pytorch/second/pytorch/models/voxelnet.py", line 274, in forward voxel_features, coors, batch_size_dev) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call result = self.forward(*input, kwargs) File "/media/ssd_2tb/myGit/second.pytorch/second/pytorch/models/middle.py", line 1021, in forward ret = self.middle_conv(ret) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call result = self.forward(*input, *kwargs) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/spconv/modules.py", line 123, in forward input = module(input) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call result = self.forward(input, kwargs) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/spconv/conv.py", line 155, in forward self.stride, self.padding, self.dilation, self.output_padding, self.subm, self.transposed, grid=input.grid) File "/home/robin2002/anaconda3/lib/python3.6/site-packages/spconv/ops.py", line 89, in get_indice_pairs stride, padding, dilation, out_padding, int(subm), int(transpose)) RuntimeError: cuda runtime error (77) : an illegal memory access was encountered at /opt/conda/conda-bld/pytorch_1550802451070/work/aten/src/THC/generic/THCTensorMath.cu:14
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "./pytorch/train.py", line 507, in
Could you change some code in test_conv.py in spconv and run it?
import sparseconvnet as scn
unittest.main()
to main()
(base) robin2002@robin2002-desktop:/media/ssd_2tb/myGit/second.pytorch/second$ CUDA_VISIBLE_DEVICES=0,1 python ./pytorch/train.py train --config_path=./configs/all.lite.nu.anchor_revised.config --model_dir=/media/ssd_2tb/myGit/second.pytorch/second/model/20190412_nu_all --multi_gpu=True [ 41 2000 2000] num_trainable parameters: 39 False _amp_stash MULTI-GPU: use 2 gpu feature_map_size [1, 250, 250] feature_map_size [1, 250, 250] WORKER 0 seed: 1555119356 WORKER 1 seed: 1555119357 WORKER 2 seed: 1555119358 model: { second: { voxel_generator { point_cloud_range : [-50, -50.0, -4, 50, 50, 2] voxel_size : [0.05, 0.05, 0.15] max_number_of_points_per_voxel : 1 }
voxel_feature_extractor: {
module_class_name: "SimpleVoxelRadius"
num_filters: [16]
with_distance: false
num_input_features: 3
}
middle_feature_extractor: {
module_class_name: "SpMiddleFHDLite"
# num_filters_down1: [] # protobuf don't support empty list.
# num_filters_down2: []
downsample_factor: 8
num_input_features: 2
}
rpn: {
module_class_name: "RPNV2"
layer_nums: [5]
layer_strides: [1]
num_filters: [128]
upsample_strides: [1]
num_upsample_filters: [128]
use_groupnorm: false
num_groups: 32
num_input_features: 128
}
loss: {
classification_loss: {
weighted_sigmoid_focal: {
alpha: 0.25
gamma: 2.0
anchorwise_output: true
}
}
localization_loss: {
weighted_smooth_l1: {
sigma: 3.0
code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
}
}
classification_weight: 1.0
localization_weight: 2.0
}
num_point_features: 3 # model's num point feature should be independent of dataset
# Outputs
use_sigmoid_score: true
encode_background_as_zeros: true
encode_rad_error_by_sin: true
use_direction_classifier: true # this can help for orientation benchmark
direction_loss_weight: 0.2 # enough.
# Loss
pos_class_weight: 1.0
neg_class_weight: 1.0
loss_norm_type: NormByNumPositives
# Postprocess
post_center_limit_range: [-50, -50, -4.0, 50, 50, 1.0]
use_rotate_nms: true
use_multi_class_nms: false
nms_pre_max_size: 1000
nms_post_max_size: 100
nms_score_threshold: 0.05 # 0.4 in submit, but 0.3 can get better hard performance
nms_iou_threshold: 0.5
box_coder: {
ground_box3d_coder: {
linear_dim: false
encode_angle_vector: false
}
}
target_assigner: {
anchor_generators: {
anchor_generator_range: {
sizes: [1.95017717, 4.60718145, 1.72270761] # wlh
anchor_ranges: [-50, -50.0, -0.93897414, 50, 50, -0.93897414]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "car"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.66344886, 0.7256437, 1.75748069] # wlh
anchor_ranges: [-50, -50.0, -0.73911038, 50, 50, -0.73911038]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "pedestrian"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.60058911, 1.68452161, 1.27192197] # wlh
anchor_ranges: [-50, -50.0, -1.03743013, 50, 50, -1.03743013]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "bicycle"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.76279481, 2.09973778, 1.44403034] # wlh
anchor_ranges: [-50, -50.0, -0.99194854, 50, 50, -0.99194854]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.35
unmatched_threshold : 0.2
class_name: "motorcycle"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.4560939, 6.73778078, 2.73004906] # wlh
anchor_ranges: [-50, -50.0, -0.37937912, 50, 50, -0.37937912]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "truck"
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.94046906, 11.1885991, 3.47030982] # wlh
anchor_ranges: [-50, -50.0, -0.0715754, 50, 50, -0.0715754]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "bus"
}
}
sample_positive_fraction : -1
sample_size : 512
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
} }
train_input_reader: { dataset: { dataset_class_name: "NuScenesDataset" kitti_info_path: "/media/ssd_2tb/dataset/nuscene/infos_train.pkl" kitti_root_path: "/media/ssd_2tb/dataset/nuscene"
# kitti_root_path: "/media/yy/960evo/datasets/nuscene/v1.0-mini"
}
batch_size: 6 preprocess: { max_number_of_voxels: 63000 shuffle_points: false num_workers: 3 groundtruth_localization_noise_std: [0, 0, 0] groundtruth_rotation_uniform_noise: [0, 0]
# groundtruth_localization_noise_std: [0.25, 0.25, 0.25]
# groundtruth_rotation_uniform_noise: [-0.3141592654, 0.3141592654]
# groundtruth_rotation_uniform_noise: [-0.78539816, 0.78539816]
global_rotation_uniform_noise: [-1.57, 1.57]
global_scaling_uniform_noise: [0.95, 1.05]
global_random_rotation_range_per_object: [0, 0] # pi/4 ~ 3pi/4
global_translate_noise_std: [0.2, 0.2, 0.2]
anchor_area_threshold: -1
remove_points_after_sample: true
groundtruth_points_drop_percentage: 0.0
groundtruth_drop_max_keep_points: 15
remove_unknown_examples: false
remove_environment: false
database_sampler {
# leave this empty to disable database_sampler, nuscenes don't need sample
# because 1. the number of ground-truth is enough. 2. sweeps don't support
# sample.
}
} }
train_config: { optimizer: { adam_optimizer: { learning_rate: { one_cycle: { lr_max: 3e-3 moms: [0.95, 0.85] div_factor: 10.0 pct_start: 0.4 } } weight_decay: 0.01 } fixed_weight_decay: true use_moving_average: false } steps: 234450 # 4689 50 (28130 // 6 + 1) steps_per_eval: 9378 # 4689 2
save_checkpoints_secs : 1800 # half hour save_summary_steps : 10 enable_mixed_precision: false loss_scale_factor: 8.0 clear_metrics_every_epoch: true }
eval_input_reader: { batch_size: 6 dataset: { dataset_class_name: "NuScenesDataset" kitti_info_path: "/media/ssd_2tb/dataset/nuscene/infos_val.pkl" kitti_root_path: "/media/ssd_2tb/dataset/nuscene" } preprocess: { max_number_of_voxels: 80000 shuffle_points: false num_workers: 3 anchor_area_threshold: -1 remove_environment: false } }
Traceback (most recent call last):
File "./pytorch/train.py", line 259, in train
ret_dict = net_parallel(example_torch)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call
result = self.forward(*input, kwargs)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 143, in forward
outputs = self.parallel_apply(replicas, inputs, kwargs)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 153, in parallel_apply
return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 83, in parallel_apply
raise output
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 59, in _worker
output = module(*input, *kwargs)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call
result = self.forward(input, kwargs)
File "/media/ssd_2tb/myGit/second.pytorch/second/pytorch/models/voxelnet.py", line 274, in forward
voxel_features, coors, batch_size_dev)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call
result = self.forward(*input, kwargs)
File "/media/ssd_2tb/myGit/second.pytorch/second/pytorch/models/middle.py", line 1021, in forward
ret = self.middle_conv(ret)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call
result = self.forward(*input, *kwargs)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/spconv/modules.py", line 123, in forward
input = module(input)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call
result = self.forward(input, kwargs)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/spconv/conv.py", line 155, in forward
self.stride, self.padding, self.dilation, self.output_padding, self.subm, self.transposed, grid=input.grid)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/spconv/ops.py", line 89, in get_indice_pairs
stride, padding, dilation, out_padding, int(subm), int(transpose))
RuntimeError: CUDA error: an illegal memory access was encountered (free_blocks at /opt/conda/conda-bld/pytorch_1550802451070/work/aten/src/THC/THCCachingAllocator.cpp:439)
frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits
run "test_conv.py", not reinstall spconv and rerun train... I want to know if this is a spconv problem.
(base) robin2002@robin2002-desktop:~/projects/spconv/test$ python test_conv.py
Traceback (most recent call last):
File "test_conv.py", line 615, in
you can try pointpillars with multi-gpu first, this problem requires c++ debugging to find out which function have problem, I don't have multi-gpu environment to do this.
If you are interest in debugging c++ code, you can add some print function to find out which getBlocks
function in CreateConvIndicePairFunctorP1
and CreateConvIndicePairFunctorP2
throw that error.
The bug is from spconv. It seems spconv cannot work in multi-gpu setting. https://github.com/traveller59/spconv/issues/35
(base) robin2002@robin2002-desktop:/media/ssd_2tb/myGit/second.pytorch/second$ CUDA_VISIBLE_DEVICES=0,1 python ./pytorch/train.py train --config_path=./configs/nuscenes/all.pp.config --model_dir=/media/ssd_2tb/myGit/second.pytorch/second/model/20190425_pp_nu --multi_gpu=Truenum parameters: 66 False _amp_stash MULTI-GPU: use 2 gpu feature_map_size [1, 248, 248] feature_map_size [1, 248, 248] model: { second: { voxel_generator { point_cloud_range : [-49.6, -49.6, -5, 49.6, 49.6, 3] voxel_size : [0.2, 0.2, 8] max_number_of_points_per_voxel : 40 } voxel_feature_extractor: { module_class_name: "PillarFeatureNet" num_filters: [64] with_distance: false num_input_features: 4 } middle_feature_extractor: { module_class_name: "PointPillarsScatter" downsample_factor: 1 num_input_features: 64 } rpn: { module_class_name: "RPNV2" layer_nums: [3, 5, 5] layer_strides: [2, 2, 2] num_filters: [64, 128, 256] upsample_strides: [1, 2, 4] num_upsample_filters: [128, 128, 128] use_groupnorm: false num_groups: 32 num_input_features: 64 } loss: { classification_loss: { weighted_sigmoid_focal: { alpha: 0.25 gamma: 2.0 anchorwise_output: true } } localization_loss: { weighted_smooth_l1: { sigma: 3.0 code_weight: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] } } classification_weight: 1.0 localization_weight: 2.0 } num_point_features: 4 # model's num point feature should be independent of dataset
use_sigmoid_score: true
encode_background_as_zeros: true
encode_rad_error_by_sin: true
use_direction_classifier: true
direction_loss_weight: 0.2
# Loss
pos_class_weight: 1.0
neg_class_weight: 1.0
loss_norm_type: NormByNumPositives
# Postprocess
post_center_limit_range: [-59.6, -59.6, -6, 59.6, 59.6, 4]
use_rotate_nms: false
use_multi_class_nms: false
nms_pre_max_size: 1000
nms_post_max_size: 300
nms_score_threshold: 0.05
nms_iou_threshold: 0.5
box_coder: {
ground_box3d_coder: {
linear_dim: false
encode_angle_vector: false
}
}
target_assigner: {
anchor_generators: {
anchor_generator_range: {
sizes: [1.95017717, 4.60718145, 1.72270761] # wlh
anchor_ranges: [-49.6, -49.6, -0.93897414, 49.6, 49.6, -0.93897414]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "car"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.60058911, 1.68452161, 1.27192197] # wlh
anchor_ranges: [-49.6, -49.6, -1.03743013, 49.6, 49.6, -1.03743013]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.4
unmatched_threshold : 0.2
class_name: "bicycle"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.94046906, 11.1885991, 3.47030982] # wlh
anchor_ranges: [-49.6, -49.6, -0.0715754, 49.6, 49.6, -0.0715754]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.7
unmatched_threshold : 0.4
class_name: "bus"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.73050468, 6.38352896, 3.13312415] # wlh
anchor_ranges: [-49.6, -49.6, -0.08168083, 49.6, 49.6, -0.08168083]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "construction_vehicle"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.76279481, 2.09973778, 1.44403034] # wlh
anchor_ranges: [-49.6, -49.6, -0.99194854, 49.6, 49.6, -0.99194854]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.4
unmatched_threshold : 0.2
class_name: "motorcycle"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.66344886, 0.7256437, 1.75748069] # wlh
anchor_ranges: [-49.6, -49.6, -0.73911038, 49.6, 49.6, -0.73911038]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.4
unmatched_threshold : 0.2
class_name: "pedestrian"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [0.39694519, 0.40359262, 1.06232151] # wlh
anchor_ranges: [-49.6, -49.6, -1.27868911, 49.6, 49.6, -1.27868911]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.3
unmatched_threshold : 0.15
class_name: "traffic_cone"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.87427237, 12.01320693, 3.81509561] # wlh
anchor_ranges: [-49.6, -49.6, 0.22228277, 49.6, 49.6, 0.22228277]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "trailer"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
anchor_generators: {
anchor_generator_range: {
sizes: [2.4560939, 6.73778078, 2.73004906] # wlh
anchor_ranges: [-49.6, -49.6, -0.37937912, 49.6, 49.6, -0.37937912]
rotations: [0, 1.57] # DON'T modify this unless you are very familiar with my code.
matched_threshold : 0.6
unmatched_threshold : 0.45
class_name: "truck"
}
region_similarity_calculator: {
nearest_iou_similarity: {
}
}
}
sample_positive_fraction : -1
sample_size : 512
}
} }
train_input_reader: { dataset: { dataset_class_name: "NuScenesDatasetD8" kitti_info_path: "/media/ssd_2tb/dataset/nuscene/infos_train.pkl" kitti_root_path: "/media/ssd_2tb/dataset/nuscene" }
batch_size: 3 preprocess: { max_number_of_voxels: 30000 shuffle_points: false num_workers: 4 groundtruth_localization_noise_std: [0, 0, 0] groundtruth_rotation_uniform_noise: [0, 0]
# groundtruth_rotation_uniform_noise: [-0.15707963267, 0.15707963267]
# global_rotation_uniform_noise: [-0.78539816, 0.78539816]
global_rotation_uniform_noise: [0, 0]
global_scaling_uniform_noise: [0.95, 1.05]
global_random_rotation_range_per_object: [0, 0]
global_translate_noise_std: [0.2, 0.2, 0.2]
anchor_area_threshold: -1
remove_points_after_sample: false
groundtruth_points_drop_percentage: 0.0
groundtruth_drop_max_keep_points: 15
remove_unknown_examples: false
remove_environment: false
database_sampler {
}
} }
train_config: { optimizer: { adam_optimizer: { learning_rate: { one_cycle: { lr_max: 3e-3 moms: [0.95, 0.85] div_factor: 10.0 pct_start: 0.4 } } weight_decay: 0.01 } fixed_weight_decay: true use_moving_average: false } steps: 58650 # 1173 50 (3517 // 3 + 1) steps_per_eval: 5865 # 1173 5 save_checkpoints_secs : 1800 # half hour save_summary_steps : 10 enable_mixed_precision: false loss_scale_factor: -1 clear_metrics_every_epoch: true }
eval_input_reader: { dataset: { dataset_class_name: "NuScenesDataset" kitti_info_path: "/media/ssd_2tb/dataset/nuscene/infos_val.pkl" kitti_root_path: "/media/ssd_2tb/dataset/nuscene" } batch_size: 1
preprocess: { max_number_of_voxels: 40000 shuffle_points: false num_workers: 3 anchor_area_threshold: -1 remove_environment: false } }
WORKER 0 seed: 1556235365
WORKER 1 seed: 1556235366
WORKER 2 seed: 1556235367
WORKER 3 seed: 1556235368
WORKER 4 seed: 1556235369
WORKER 5 seed: 1556235370
WORKER 6 seed: 1556235371
WORKER 7 seed: 1556235372
/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/_reduction.py:49: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
warnings.warn(warning.format(ret))
binary_op(): expected both inputs to be on same device, but input a is on cuda:0 and input b is on cuda:1
Traceback (most recent call last):
File "./pytorch/train.py", line 541, in
Hi @traveller59 ,
Thanks for this wonderful implementation. I was trying to train a model using two gpus. Bug It was not working. (Out of index error)
script: CUDA_VISIBLE_DEVICES=0,1 python ./pytorch/train.py train --config_path=./configs/car.fhd.config --model_dir=/data/second/model --multi_gpu=True
Error message: Traceback (most recent call last): Traceback (most recent call last): File "./pytorch/train.py", line 509, in
train(config_path=config_path, model_dir=model_dir, multi_gpu=multi_gpu)
File "./pytorch/train.py", line 369, in train
raise e
File "./pytorch/train.py", line 257, in train
ret_dict = net_parallel(example_torch)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call
result = self.forward(*input, kwargs)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 143, in forward
outputs = self.parallel_apply(replicas, inputs, kwargs)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 153, in parallel_apply
return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 83, in parallel_apply
raise output
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py", line 59, in _worker
output = module(*input, *kwargs)
File "/home/robin2002/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in call
result = self.forward(input, kwargs)
File "/media/ssd_2tb/myGit/second.pytorch/second/pytorch/models/voxelnet.py", line 252, in forward
voxel_list.append(voxels[i, :num_voxel])
IndexError: index 6 is out of bounds for dimension 0 with size 6