search

V2AI / Det3D

World's first general purpose 3D object detection codebse.
https://arxiv.org/abs/1908.09492
Apache License 2.0
1.48k stars 299 forks source link

multi-gpu training error #81

Closed 1453042287 closed 4 years ago

1453042287 commented 4 years ago

meet the following errors when using the train.sh to setup a multi-gpu trianing, with CUDA10, pytorch1.1,python3.6,cudnn7.5

Traceback (most recent call last): File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/errors.py", line 627, in new_error_context yield File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/lowering.py", line 258, in lower_block self.lower_inst(inst) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/lowering.py", line 301, in lower_inst val = self.lower_assign(ty, inst) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/lowering.py", line 454, in lower_assign return self.lower_expr(ty, value) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/lowering.py", line 1056, in lower_expr return self.context.build_list(self.builder, resty, castvals) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/targets/cpu.py", line 122, in build_list return listobj.build_list(self, builder, list_type, items) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/targets/listobj.py", line 450, in build_list inst = ListInstance.allocate(context, builder, list_type, nitems) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/targets/listobj.py", line 318, in allocate ok, self = cls.allocate_ex(context, builder, list_type, nitems) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/targets/listobj.py", line 267, in allocate_ex self.zfill(self.size.type(0), nitems) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/targets/listobj.py", line 221, in zfill cgutils.memset(builder, base, size, ir.IntType(8)(0)) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/cgutils.py", line 865, in memset builder.call(fn, [ptr, value, size, int32_t(0), bool_t(0)]) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/llvmlite-0.31.0-py3.6-linux-x86_64.egg/llvmlite/ir/builder.py", line 841, in call cconv=cconv, tail=tail, fastmath=fastmath) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/llvmlite-0.31.0-py3.6-linux-x86_64.egg/llvmlite/ir/instructions.py", line 84, in init raise TypeError(msg) TypeError: Type of #4 arg mismatch: i1 != i32

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "./tools/train.py", line 10, in from det3d.datasets import build_dataset File "/home/chi_zhang/yz/Det3D/det3d/datasets/init.py", line 4, in from .kitti import KittiDataset File "/home/chi_zhang/yz/Det3D/det3d/datasets/kitti/init.py", line 1, in from .kitti import KittiDataset File "/home/chi_zhang/yz/Det3D/det3d/datasets/kitti/kitti.py", line 7, in from det3d.core import box_np_ops File "/home/chi_zhang/yz/Det3D/det3d/core/init.py", line 4, in from .anchor import File "/home/chi_zhang/yz/Det3D/det3d/core/anchor/init.py", line 1, in from .anchor_generator import ( File "/home/chi_zhang/yz/Det3D/det3d/core/anchor/anchor_generator.py", line 2, in from det3d.core.bbox import box_np_ops File "/home/chi_zhang/yz/Det3D/det3d/core/bbox/init.py", line 42, in from . import box_coders, box_np_ops, box_torch_ops, geometry, region_similarity File "/home/chi_zhang/yz/Det3D/det3d/core/bbox/box_coders.py", line 5, in from . import box_np_ops, box_torch_ops File "/home/chi_zhang/yz/Det3D/det3d/core/bbox/box_torch_ops.py", line 6, in from det3d.ops.nms.nms_cpu import rotate_nms_cc File "/home/chi_zhang/yz/Det3D/det3d/ops/nms/init.py", line 1, in from det3d.ops.nms.nms_cpu import nms_jit, soft_nms_jit File "/home/chi_zhang/yz/Det3D/det3d/ops/nms/nms_cpu.py", line 81, in def soft_nms_jit(boxes, sigma=0.5, Nt=0.3, threshold=0.001, method=0): File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/decorators.py", line 183, in wrapper disp.compile(sig) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler_lock.py", line 32, in _acquire_compile_lock return func(args, *kwargs) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/dispatcher.py", line 658, in compile cres = self._compiler.compile(args, return_type) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/dispatcher.py", line 82, in compile pipeline_class=self.pipeline_class) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 941, in compile_extra return pipeline.compile_extra(func) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 372, in compile_extra return self._compile_bytecode() File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 872, in _compile_bytecode return self._compile_core() File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 859, in _compile_core res = pm.run(self.status) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler_lock.py", line 32, in _acquire_compile_lock return func(args, **kwargs) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 253, in run raise patched_exception File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 244, in run stage() File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 731, in stage_nopython_backend self._backend(lowerfn, objectmode=False) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 681, in _backend lowered = lowerfn() File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 668, in backend_nopython_mode self.metadata) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/compiler.py", line 1063, in native_lowering_stage lower.lower() File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/lowering.py", line 177, in lower self.lower_normal_function(self.fndesc) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/lowering.py", line 218, in lower_normal_function entry_block_tail = self.lower_function_body() File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/lowering.py", line 243, in lower_function_body self.lower_block(block) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/lowering.py", line 258, in lower_block self.lower_inst(inst) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/contextlib.py", line 99, in exit self.gen.throw(type, value, traceback) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/errors.py", line 635, in new_error_context six.reraise(type(newerr), newerr, tb) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/numba/six.py", line 659, in reraise raise value numba.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend) Type of #4 arg mismatch: i1 != i32

File "det3d/ops/nms/nms_cpu.py", line 168: def soft_nms_jit(boxes, sigma=0.5, Nt=0.3, threshold=0.001, method=0):

keep = [i for i in range(N)]
^

[1] During: lowering "$0.1.1 = build_list(items=[])" at /home/chi_zhang/yz/Det3D/det3d/ops/nms/nms_cpu.py (168) Traceback (most recent call last): File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/runpy.py", line 193, in _run_module_as_main "main", mod_spec) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/runpy.py", line 85, in _run_code exec(code, run_globals) File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/torch/distributed/launch.py", line 235, in main() File "/home/chi_zhang/anaconda3/envs/det3d/lib/python3.6/site-packages/torch/distributed/launch.py", line 231, in main cmd=process.args) subprocess.CalledProcessError: Command '['/home/chi_zhang/anaconda3/envs/det3d/bin/python', '-u', './tools/train.py', '--local_rank=0', 'examples/second/configs/car.py', '--work_dir=/home/chi_zhang/yz/det3d-outputs/SECOND_test_20200227-014111']' returned non-zero exit status 1.

poodarchu commented 4 years ago

numba.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend), try compile and run again.