Closed KEVIN666666666 closed 3 years ago
When I am doing test and inference, there are always Segmentation fault come out and interrupt the process. I can't figure out where is wrong with the memory allocation, please have a look with the Backtrace.
Segmentation fault
(vedadet) [root@localhost vedadet]# CUDA_VISIBLE_DEVICES="1,2" python tools/infer.py configs/infer/tinaface/tinaface.py t1.jpg /data/anaconda3/envs/vedadet/lib/python3.8/site-packages/torch/nn/functional.py:3060: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details. warnings.warn("Default upsampling behavior when mode={} is changed " *** Error in `python': free(): invalid pointer: 0x000055b33065d600 *** ======= Backtrace: ========= /lib64/libc.so.6(+0x81299)[0x7f8f43d7f299] /usr/local/cuda-11.0/lib64/libcublasLt.so.11(free_gemm_select+0x4d)[0x7f8f03b5ea3d] /data/anaconda3/envs/vedadet/lib/python3.8/site-packages/torch/lib/../../../../libcublas.so.11(cublasDestroy_v2+0x165)[0x7f8f0e991af5] /data/anaconda3/envs/vedadet/lib/python3.8/site-packages/torch/lib/libtorch_cuda.so(+0x25a4345d)[0x7f8ea992f45d] /data/anaconda3/envs/vedadet/lib/python3.8/site-packages/torch/lib/libtorch_cuda.so(+0x25a43561)[0x7f8ea992f561] /lib64/libc.so.6(+0x39ce9)[0x7f8f43d37ce9] /lib64/libc.so.6(+0x39d37)[0x7f8f43d37d37] /lib64/libc.so.6(__libc_start_main+0xfc)[0x7f8f43d2055c] python(+0x1dee93)[0x55b2d9117e93] ======= Memory map: ======== 200000000-200200000 ---p 00000000 00:00 0 200200000-200400000 rw-s 00000000 00:05 19850 /dev/nvidiactl 200400000-202400000 rw-s 00000000 00:05 19850 /dev/nvidiactl 202400000-205400000 rw-s 00000000 00:05 19850 /dev/nvidiactl 205400000-206000000 ---p 00000000 00:00 0 206000000-206200000 rw-s 00000000 00:05 19850 /dev/nvidiactl 206200000-206400000 rw-s 00000000 00:05 19850 /dev/nvidiactl 206400000-206600000 rw-s 206400000 00:05 63275 /dev/nvidia-uvm 206600000-206800000 rw-s 00000000 00:05 19850 /dev/nvidiactl 206800000-206a00000 ---p 00000000 00:00 0 206a00000-206c00000 rw-s 00000000 00:05 19850 /dev/nvidiactl 206c00000-400200000 ---p 00000000 00:00 0 10000000000-10104000000 ---p 00000000 00:00 0 55b2d8f39000-55b2d8f98000 r--p 00000000 fd:04 4026537124 /data/anaconda3/envs/vedadet/bin/python3.8 55b2d8f98000-55b2d9189000 r-xp 0005f000 fd:04 4026537124 /data/anaconda3/envs/vedadet/bin/python3.8 55b2d9189000-55b2d926f000 r--p 00250000 fd:04 4026537124 /data/anaconda3/envs/vedadet/bin/python3.8 55b2d9270000-55b2d9275000 r--p 00336000 fd:04 4026537124 /data/anaconda3/envs/vedadet/bin/python3.8 55b2d9275000-55b2d92ad000 rw-p 0033b000 fd:04 4026537124 /data/anaconda3/envs/vedadet/bin/python3.8 55b2d92ad000-55b2d92cd000 rw-p 00000000 00:00 0 55b2d9f58000-55b331409000 rw-p 00000000 00:00 0 [heap] 7f8c54000000-7f8c54021000 rw-p 00000000 00:00 0 7f8c54021000-7f8c58000000 ---p 00000000 00:00 0 7f8c5c000000-7f8c9b200000 ---p 00000000 00:00 0 7f8c9b200000-7f8c9b400000 rw-s 00000000 00:04 44358018 /dev/zero (deleted) 7f8c9b400000-7f8c9c000000 ---p 00000000 00:00 0 7f8c9c000000-7f8c9c021000 rw-p 00000000 00:00 0 7f8c9c021000-7f8ca0000000 ---p 00000000 00:00 0 7f8ca0000000-7f8ca0021000 rw-p 00000000 00:00 0 7f8ca0021000-7f8ca4000000 ---p 00000000 00:00 0 7f8ca4000000-7f8ca4021000 rw-p 00000000 00:00 0 7f8ca4021000-7f8ca8000000 ---p 00000000 00:00 0 7f8caaffe000-7f8caafff000 ---p 00000000 00:00 0 7f8caafff000-7f8cab7ff000 rw-p 00000000 00:00 0 7f8cab7ff000-7f8cab800000 ---p 00000000 00:00 0 7f8cab800000-7f8cac000000 rw-p 00000000 00:00 0 7f8cac000000-7f8cac021000 rw-p 00000000 00:00 0 7f8cac021000-7f8cb0000000 ---p 00000000 00:00 0 7f8cb0000000-7f8cb0021000 rw-p 00000000 00:00 0 7f8cb0021000-7f8cb4000000 ---p 00000000 00:00 0 7f8cb4000000-7f8cb4021000 rw-p 00000000 00:00 0 7f8cb4021000-7f8cb8000000 ---p 00000000 00:00 0 7f8cb8000000-7f8cb8021000 rw-p 00000000 00:00 0 7f8cb8021000-7f8cbc000000 ---p 00000000 00:00 0 7f8cbc000000-7f8cbc021000 rw-p 00000000 00:00 0 7f8cbc021000-7f8cc0000000 ---p 00000000 00:00 0 7f8cc0000000-7f8cc0021000 rw-p 00000000 00:00 0 7f8cc0021000-7f8cc4000000 ---p 00000000 00:00 0 7f8cc4000000-7f8cc4021000 rw-p 00000000 00:00 0 7f8cc4021000-7f8cc8000000 ---p 00000000 00:00 0 7f8cc87f9000-7f8cc87fa000 ---p 00000000 00:00 0 7f8cc87fa000-7f8cc8ffa000 rw-p 00000000 00:00 0 7f8cc8ffa000-7f8cc8ffb000 ---p 00000000 00:00 0 7f8cc8ffb000-7f8cc97fb000 rw-p 00000000 00:00 0 7f8cc97fb000-7f8cc97fc000 ---p 00000000 00:00 0 7f8cc97fc000-7f8cc9ffc000 rw-p 00000000 00:00 0 7f8cc9ffc000-7f8cc9ffd000 ---p 00000000 00:00 0 7f8cc9ffd000-7f8cca7fd000 rw-p 00000000 00:00 0 7f8cca7fd000-7f8cca7fe000 ---p 00000000 00:00 0 7f8cca7fe000-7f8ccaffe000 rw-p 00000000 00:00 0 7f8ccaffe000-7f8ccafff000 ---p 00000000 00:00 0 7f8ccafff000-7f8ccb7ff000 rw-p 00000000 00:00 0 7f8ccb7ff000-7f8ccb800000 ---p 00000000 00:00 0 7f8ccb800000-7f8ccc000000 rw-p 00000000 00:00 0 7f8ccc000000-7f8ccc021000 rw-p 00000000 00:00 0 7f8ccc021000-7f8cd0000000 ---p 00000000 00:00 0 7f8cd0000000-7f8cd0021000 rw-p 00000000 00:00 0 7f8cd0021000-7f8cd4000000 ---p 00000000 00:00 0 7f8cd4000000-7f8cd4021000 rw-p 00000000 00:00 0 7f8cd4021000-7f8cd8000000 ---p 00000000 00:00 0 7f8cd8000000-7f8cd8021000 rw-p 00000000 00:00 0 7f8cd8021000-7f8cdc000000 ---p 00000000 00:00 0 7f8cdc000000-7f8cdc021000 rw-p 00000000 00:00 0 7f8cdc021000-7f8ce0000000 ---p 00000000 00:00 0 7f8ce0000000-7f8ce0021000 rw-p 00000000 00:00 0 7f8ce0021000-7f8ce4000000 ---p 00000000 00:00 0 7f8ce4000000-7f8ce4021000 rw-p 00000000 00:00 0 7f8ce4021000-7f8ce8000000 ---p 00000000 00:00 0 7f8ce8000000-7f8ce8021000 rw-p 00000000 00:00 0 7f8ce8021000-7f8cec000000 ---p 00000000 00:00 0 7f8cec7f9000-7f8cec7fa000 ---p 00000000 00:00 0 7f8cec7fa000-7f8cecffa000 rw-p 00000000 00:00 0 7f8cecffa000-7f8cecffb000 ---p 00000000 00:00 0 7f8cecffb000-7f8ced7fb000 rw-p 00000000 00:00 0 7f8ced7fb000-7f8ced7fc000 ---p 00000000 00:00 0 7f8ced7fc000-7f8cedffc000 rw-p 00000000 00:00 0 7f8cedffc000-7f8cedffd000 ---p 00000000 00:00 0 7f8cedffd000-7f8cee7fd000 rw-p 00000000 00:00 0 7f8cee7fd000-7f8cee7fe000 ---p 00000000 00:00 0 7f8cee7fe000-7f8ceeffe000 rw-p 00000000 00:00 0 7f8ceeffe000-7f8ceefff000 ---p 00000000 00:00 0 7f8ceefff000-7f8cef7ff000 rw-p 00000000 00:00 0 7f8cef7ff000-7f8cef800000 ---p 00000000 00:00 0 7f8cef800000-7f8cf0000000 rw-p 00000000 00:00 0 7f8cf0000000-7f8cf0021000 rw-p 00000000 00:00 0 7f8cf0021000-7f8cf4000000 ---p 00000000 00:00 0 7f8cf4000000-7f8cf4021000 rw-p 00000000 00:00 0 7f8cf4021000-7f8cf8000000 ---p 00000000 00:00 0 7f8cf8000000-7f8cf8021000 rw-p 00000000 00:00 0 7f8cf8021000-7f8cfc000000 ---p 00000000 00:00 0 7f8cfc000000-7f8cfc021000 rw-p 00000000 00:00 0 7f8cfc021000-7f8d00000000 ---p 00000000 00:00 0 7f8d00000000-7f8d00021000 rw-p 00000000 00:00 0 7f8d00021000-7f8d04000000 ---p 00000000 00:00 0 7f8d04000000-7f8d04021000 rw-p 00000000 00:00 0 7f8d04021000-7f8d08000000 ---p 00000000 00:00 0 7f8d08000000-7f8d08021000 rw-p 00000000 00:00 0 7f8d08021000-7f8d0c000000 ---p 00000000 00:00 0 7f8d0c000000-7f8d0c021000 rw-p 00000000 00:00 0 7f8d0c021000-7f8d10000000 ---p 00000000 00:00 0 7f8d10000000-7f8d10021000 rw-p 00000000 00:00 0 7f8d10021000-7f8d14000000 ---p 00000000 00:00 0 7f8d14000000-7f8d14021000 rw-p 00000000 00:00 0 7f8d14021000-7f8d18000000 ---p 00000000 00:00 0 7f8d18000000-7f8d18021000 rw-p 00000000 00:00 0 7f8d18021000-7f8d1c000000 ---p 00000000 00:00 0 7f8d1c7f9000-7f8d1c7fa000 ---p 00000000 00:00 0 7f8d1c7fa000-7f8d1cffa000 rw-p 00000000 00:00 0 7f8d1cffa000-7f8d1cffb000 ---p 00000000 00:00 0 7f8d1cffb000-7f8d1d7fb000 rw-p 00000000 00:00 0 7f8d1d7fb000-7f8d1d7fc000 ---p 00000000 00:00 0 7f8d1d7fc000-7f8d1dffc000 rw-p 00000000 00:00 0 7f8d1dffc000-7f8d1dffd000 ---p 00000000 00:00 0 7f8d1dffd000-7f8d1e7fd000 rw-p 00000000 00:00 0 7f8d1e7fd000-7f8d1e7fe000 ---p 00000000 00:00 0 7f8d1e7fe000-7f8d1effe000 rw-p 00000000 00:00 0 7f8d1effe000-7f8d1efff000 ---p 00000000 00:00 0 7f8d1efff000-7f8d1f7ff000 rw-p 00000000 00:00 0 7f8d1f7ff000-7f8d1f800000 ---p 00000000 00:00 0 7f8d1f800000-7f8d20000000 rw-p 00000000 00:00 0 7f8d20000000-7f8d20021000 rw-p 00000000 00:00 0 7f8d20021000-7f8d24000000 ---p 00000000 00:00 0 7f8d24000000-7f8d24021000 rw-p 00000000 00:00 0 7f8d24021000-7f8d28000000 ---p 00000000 00:00 0 7f8d28000000-7f8d28021000 rw-p 00000000 00:00 0 7f8d28021000-7f8d2c000000 ---p 00000000 00:00 0 7f8d2c000000-7f8d2c021000 rw-p 00000000 00:00 0 7f8d2c021000-7f8d30000000 ---p 00000000 00:00 0 7f8d30000000-7f8d30021000 rw-p 00000000 00:00 0 7f8d30021000-7f8d34000000 ---p 00000000 00:00 0 7f8d34000000-7f8d34021000 rw-p 00000000 00:00 0 7f8d34021000-7f8d38000000 ---p 00000000 00:00 0 7f8d38000000-7f8d38021000 rw-p 00000000 00:00 0 7f8d38021000-7f8d3c000000 ---p 00000000 00:00 0 7f8d3c000000-7f8d3c021000 rw-p 00000000 00:00 0 7f8d3c021000-7f8d40000000 ---p 00000000 00:00 0 7f8d40000000-7f8d40021000 rw-p 00000000 00:00 0 7f8d40021000-7f8d44000000 ---p 00000000 00:00 0 7f8d44000000-7f8d44021000 rw-p 00000000 00:00 0 7f8d44021000-7f8d48000000 ---p 00000000 00:00 0 7f8d48000000-7f8d57000000 ---p 00000000 00:00 0 7f8d57000000-7f8d57200000 rw-s 00000000 00:04 44358017 /dev/zero (deleted) 7f8d57200000-7f8d62400000 ---p 00000000 00:00 0 7f8d62400000-7f8d62600000 rw-s 00000000 00:04 44358012 /dev/zero (deleted) 7f8d62600000-7f8d62800000 rw-s 00000000 00:05 19850 /dev/nvidiactl 7f8d62800000-7f8d62a00000 rw-s 00000000 00:04 44358013 /dev/zero (deleted) 7f8d62a00000-7f8d62e00000 ---p 00000000 00:00 0 7f8d62e00000-7f8d63000000 rw-s 00000000 00:05 19850 /dev/nvidiactl 7f8d63000000-7f8d63200000 rw-s 00000000 00:04 44358015 /dev/zero (deleted) 7f8d63200000-7f8d634d6000 rw-s 00000000 00:05 19850 /dev/nvidiactl 7f8d634d6000-7f8d64000000 ---p 00000000 00:00 0 7f8d64000000-7f8d64021000 rw-p 00000000 00:00 0 7f8d64021000-7f8d68000000 ---p 00000000 00:00 0 7f8d686a7000-7f8d686a8000 ---p 00000000 00:00 0 7f8d686a8000-7f8d68ea8000 rw-p 00000000 00:00 0 7f8d68ea8000-7f8d68ea9000 ---p 00000000 00:00 0 7f8d68ea9000-7f8d696a9000 rw-p 00000000 00:00 0 7f8d696a9000-7f8d696aa000 ---p 00000000 00:00 0 7f8d696aa000-7f8d69eaa000 rw-p 00000000 00:00 0 7f8d69eaa000-7f8d6a0aa000 rw-s 00000000 00:04 44358014 /dev/zero (deleted) 7f8d6a0aa000-7f8d6a0ab000 ---p 00000000 00:00 0 7f8d6a0ab000-7f8d6a8ab000 rw-p 00000000 00:00 0 7f8d6a8ab000-7f8d6a8ac000 ---p 00000000 00:00 0 7f8d6a8ac000-7f8d6b0ac000 rw-p 00000000 00:00 0 7f8d6b7f7000-7f8d6b7f8000 ---p 00000000 00:00 0 7f8d6b7f8000-7f8d6bff8000 rw-p 00000000 00:00 0 7f8d6bff8000-7f8d6bff9000 ---p 00000000 00:00 0 7f8d6bff9000-7f8d6c7f9000 rw-p 00000000 00:00 0 7f8d6c7f9000-7f8d6c7fa000 ---p 00000000 00:00 0 7f8d6c7fa000-7f8d6cffa000 rw-p 00000000 00:00 0 7f8d6cffa000-7f8d6cffb000 ---p 00000000 00:00 0 7f8d6cffb000-7f8d6d7fb000 rw-p 00000000 00:00 0 7f8d6d7fb000-7f8d6d7fc000 ---p 00000000 00:00 0 7f8d6d7fc000-7f8d6dffc000 rw-p 00000000 00:00 0 7f8d6dffc000-7f8d6dffd000 ---p 00000000 00:00 0 7f8d6dffd000-7f8d6e7fd000 rw-p 00000000 00:00 0 7f8d6e7fd000-7f8d6e7fe000 ---p 00000000 00:00 0 7f8d6e7fe000-7f8d6effe000 rw-p 00000000 00:00 0 7f8d6effe000-7f8d6efff000 ---p 00000000 00:00 0 7f8d6efff000-7f8d6f7ff000 rw-p 00000000 00:00 0 7f8d6f7ff000-7f8d6f800000 ---p 00000000 00:00 0 7f8d6f800000-7f8d70000000 rw-p 00000000 00:00 0 7f8d70000000-7f8d70021000 rw-p 00000000 00:00 0 7f8d70021000-7f8d74000000 ---p 00000000 00:00 0 7f8d74000000-7f8d74021000 rw-p 00000000 00:00 0 7f8d74021000-7f8d78000000 ---p 00000000 00:00 0 7f8d78000000-7f8d78021000 rw-p 00000000 00:00 0 7f8d78021000-7f8d7c000000 ---p 00000000 00:00 0 7f8d7c000000-7f8d7c021000 rw-p 00000000 00:00 0 7f8d7c021000-7f8d80000000 ---p 00000000 00:00 0 7f8d80000000-7f8d80021000 rw-p 00000000 00:00 0 7f8d80021000-7f8d84000000 ---p 00000000 00:00 0 7f8d84000000-7f8d84021000 rw-p 00000000 00:00 0 7f8d84021000-7f8d88000000 ---p 00000000 00:00 0 7f8d88000000-7f8d88021000 rw-p 00000000 00:00 0 7f8d88021000-7f8d8c000000 ---p 00000000 00:00 0 7f8d8c000000-7f8d8c021000 rw-p 00000000 00:00 0 7f8d8c021000-7f8d90000000 ---p 00000000 00:00 0 7f8d90000000-7f8d90021000 rw-p 00000000 00:00 0 7f8d90021000-7f8d94000000 ---p 00000000 00:00 0 7f8d947f6000-7f8d947f7000 ---p 00000000 00:00 0 7f8d947f7000-7f8d94ff7000 rw-p 00000000 00:00 0 7f8d94ff7000-7f8d94ff8000 ---p 00000000 00:00 0 7f8d94ff8000-7f8d957f8000 rw-p 00000000 00:00 0 7f8d957f8000-7f8d957f9000 ---p 00000000 00:00 0 7f8d957f9000-7f8d95ff9000 rw-p 00000000 00:00 0 7f8d95ff9000-7f8d95ffa000 ---p 00000000 00:00 0 7f8d95ffa000-7f8d967fa000 rw-p 00000000 00:00 0 7f8d967fa000-7f8d967fb000 ---p 00000000 00:00 0 7f8d967fb000-7f8d96ffc000 rw-p 00000000 00:00 0 7f8d96ffc000-7f8d96ffd000 ---p 00000000 00:00 0 7f8d96ffd000-7f8d977fe000 rw-p 00000000 00:00 0 7f8d977fe000-7f8d977ff000 ---p 00000000 00:00 0 7f8d977ff000-7f8d98000000 rw-p 00000000 00:00 0 7f8d98000000-7f8d98021000 rw-p 00000000 00:00 0 7f8d98021000-7f8d9c000000 ---p 00000000 00:00 0 7f8d9c000000-7f8d9c021000 rw-p 00000000 00:00 0 7f8d9c021000-7f8da0000000 ---p 00000000 00:00 0 7f8da0000000-7f8da0021000 rw-p 00000000 00:00 0 7f8da0021000-7f8da4000000 ---p 00000000 00:00 0 7f8da4000000-7f8da4021000 rw-p 00000000 00:00 0 7f8da4021000-7f8da8000000 ---p 00000000 00:00 0 7f8da8000000-7f8da8021000 rw-p 00000000 00:00 0 7f8da8021000-7f8dac000000 ---p 00000000 00:00 0 7f8dac000000-7f8dac021000 rw-p 00000000 00:00 0 7f8dac021000-7f8db0000000 ---p 00000000 00:00 0 7f8db0000000-7f8db0021000 rw-p 00000000 00:00 0 7f8db0021000-7f8db4000000 ---p 00000000 00:00 0 7f8db4000000-7f8db4021000 rw-p 00000000 00:00 0 7f8db4021000-7f8db8000000 ---p 00000000 00:00 0 7f8db8000000-7f8db8021000 rw-p 00000000 00:00 0 7f8db8021000-7f8dbc000000 ---p 00000000 00:00 0 7f8dbc000000-7f8dbc021000 rw-p 00000000 00:00 0 7f8dbc021000-7f8dc0000000 ---p 00000000 00:00 0 7f8dc0000000-7f8dc0001000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0001000-7f8dc0002000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0002000-7f8dc0003000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0003000-7f8dc0004000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0004000-7f8dc0005000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0005000-7f8dc0006000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0006000-7f8dc0007000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0007000-7f8dc0008000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0008000-7f8dc0009000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0009000-7f8dc000a000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc000a000-7f8dc000b000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc000b000-7f8dc000c000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc000c000-7f8dc000d000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc000d000-7f8dc000e000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc000e000-7f8dc000f000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc000f000-7f8dc0010000 rw-s 00000000 00:05 14910 /dev/nvidia1 7f8dc0010000-7f8dd0000000 ---p 00000000 00:00 0 7f8dd01be000-7f8dd023e000 rw-p 00000000 00:00 0 7f8dd023e000-7f8dd023f000 ---p 00000000 00:00 0 7f8dd023f000-7f8dd0a40000 rw-p 00000000 00:00 0 7f8dd0a40000-7f8dd0a41000 ---p 00000000 00:00 0 7f8dd0a41000-7f8dd1242000 rw-p 00000000 00:00 0 7f8dd1242000-7f8dd1243000 ---p 00000000 00:00 0 7f8dd1243000-7f8dd1a44000 rw-p 00000000 00:00 0 7f8dd1a44000-7f8dd1a45000 ---p 00000000 00:00 0 7f8dd1a45000-7f8dd2246000 rw-p 00000000 00:00 0 7f8dd2246000-7f8dd2247000 ---p 00000000 00:00 0 7f8dd2247000-7f8dd2a48000 rw-p 00000000 00:00 0 7f8dd2a48000-7f8dd2a49000 ---p 00000000 00:00 0 7f8dd2a49000-7f8dd324a000 rw-p 00000000 00:00 0 Aborted
@KEVIN666666666 please check the torch and cuda version.
When I am doing test and inference, there are always
Segmentation fault
come out and interrupt the process. I can't figure out where is wrong with the memory allocation, please have a look with the Backtrace.