How to solve nninja: build stopped: subcommand failed.\n'?

_12_epoch484.yamlts/cityscapes/seg_hrnet_w48_train_512x1024_sgd_lr1e-2_wd5e-4bs Traceback (most recent call last): File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 949, in _build_extension_module check=True) File "/opt/conda/lib/python3.6/subprocess.py", line 438, in run output=stdout, stderr=stderr) subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "tools/train.py", line 27, in import models File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/init.py", line 11, in import models.seg_hrnet File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/seg_hrnet.py", line 22, in from .sync_bn.inplace_abn.bn import InPlaceABNSync File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/init.py", line 1, in from .inplace_abn import bn File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/init.py", line 1, in from .bn import ABN, InPlaceABN, InPlaceABNSync File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/bn.py", line 14, in from functions import File "/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/functions.py", line 16, in extra_cuda_cflags=["--expt-extended-lambda"]) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 644, in load is_python_module) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 813, in _jit_compile with_cuda=with_cuda) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 866, in _write_ninja_file_and_build _build_extension_module(name, build_directory, verbose) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 962, in _build_extension_module raise RuntimeError(message) RuntimeError: Error building extension 'inplace_abn': b'[1/4] /usr/local/cuda/bin/nvcc -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -DCUDA_NO_HALF_OPERATORS -DCUDA_NO_HALF_CONVERSIONS -DCUDA_NO_HALF2_OPERATORS__ --compiler-options \'-fPIC\' --expt-extended-lambda -std=c++11 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cuda.cu -o inplace_abn_cuda.cuda.o\nFAILED: inplace_abn_cuda.cuda.o \n/usr/local/cuda/bin/nvcc -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -DCUDA_NO_HALF_OPERATORS -DCUDA_NO_HALF_CONVERSIONS -DCUDA_NO_HALF2_OPERATORS --compiler-options \'-fPIC\' --expt-extended-lambda -std=c++11 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cuda.cu -o inplace_abn_cuda.cuda.o\nnvcc fatal : Could not open output file \'/local_scratch/pbs.492439.pbs02/tmpxft_002ac4fd_00000000\'\n[2/4] c++ -MMD -MF inplace_abn_cpu.o.d -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++11 -O3 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp -o inplace_abn_cpu.o\nFAILED: inplace_abn_cpu.o \nc++ -MMD -MF inplace_abn_cpu.o.d -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++11 -O3 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp -o inplace_abn_cpu.o\n/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp: In function \'std::vector backward_cpu(at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, bool, float)\':\n/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp:82:34: error: could not convert \'z.at::Tensor::type()\' from \'at::DeprecatedTypeProperties\' to \'c10::IntArrayRef {aka c10::ArrayRef}\'\n auto dweight = at::empty(z.type(), {0});\n ^\n/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp:83:32: error: could not convert \'z.at::Tensor::type()\' from \'at::DeprecatedTypeProperties\' to \'c10::IntArrayRef {aka c10::ArrayRef}\'\n auto dbias = at::empty(z.type(), {0});\n ^\n/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn_cpu.cpp:89:29: error: could not convert \'{dx, dweight, dbias}\' from \'\' to \'std::vector\'\n return {dx, dweight, dbias};\n ^\n[3/4] c++ -MMD -MF inplace_abn.o.d -DTORCH_EXTENSION_NAME=inplace_abn -DTORCH_API_INCLUDE_EXTENSION_H -isystem /opt/conda/lib/python3.6/site-packages/torch/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -isystem /opt/conda/lib/python3.6/site-packages/torch/include/TH -isystem /opt/conda/lib/python3.6/site-packages/torch/include/THC -isystem /usr/local/cuda/include -isystem /opt/conda/include/python3.6m -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++11 -O3 -c /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn.cpp -o inplace_abn.o\nIn file included from /home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/src/inplace_abn.cpp:1:0:\n/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include/torch/torch.h:7:2: warning: #warning "Including torch/torch.h for C++ extensions is deprecated. Please include torch/extension.h" [-Wcpp]\n #warning \\n ^\nninja: build stopped: subcommand failed.\n' Traceback (most recent call last): File "tools/train.py", line 27, in Traceback (most recent call last): File "tools/train.py", line 27, in import models import models File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/init.py", line 11, in File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/init.py", line 11, in import models.seg_hrnet File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/seg_hrnet.py", line 22, in import models.seg_hrnet File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/seg_hrnet.py", line 22, in from .sync_bn.inplace_abn.bn import InPlaceABNSync File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/init.py", line 1, in from .sync_bn.inplace_abn.bn import InPlaceABNSync File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/init.py", line 1, in from .inplace_abn import bn File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/init__.py", line 1, in from .inplace_abn import bn File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/init.py", line 1, in from .bn import ABN, InPlaceABN, InPlaceABNSync File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/bn.py", line 14, in from .bn import ABN, InPlaceABN, InPlaceABNSync File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/bn.py", line 14, in from functions import File "/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/functions.py", line 16, in from functions import File "/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/functions.py", line 16, in extra_cuda_cflags=["--expt-extended-lambda"]) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 644, in load extra_cuda_cflags=["--expt-extended-lambda"]) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 644, in load is_python_module) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 824, in _jit_compile is_python_module) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 824, in _jit_compile return _import_module_from_library(name, build_directory, is_python_module) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 967, in _import_module_from_library return _import_module_from_library(name, build_directory, is_python_module) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 967, in _import_module_from_library file, path, description = imp.find_module(module_name, [path]) File "/opt/conda/lib/python3.6/imp.py", line 297, in find_module file, path, description = imp.find_module(module_name, [path]) File "/opt/conda/lib/python3.6/imp.py", line 297, in find_module Traceback (most recent call last): File "tools/train.py", line 27, in import models File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/init.py", line 11, in import models.seg_hrnet File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/seg_hrnet.py", line 22, in from .sync_bn.inplace_abn.bn import InPlaceABNSync File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/init.py", line 1, in raise ImportError(_ERR_MSG.format(name), name=name) ImportError: No module named 'inplace_abn' raise ImportError(_ERR_MSG.format(name), name=name) ImportError: No module named 'inplace_abn' from .inplace_abn import bn File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/init.py", line 1, in from .bn import ABN, InPlaceABN, InPlaceABNSync File "/home/zhiminc/HRNet-Semantic-Segmentation/tools/../lib/models/sync_bn/inplace_abn/bn.py", line 14, in from functions import File "/home/zhiminc/HRNet-Semantic-Segmentation/lib/models/sync_bn/inplace_abn/functions.py", line 16, in extra_cuda_cflags=["--expt-extended-lambda"]) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 644, in load is_python_module) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 824, in _jit_compile return _import_module_from_library(name, build_directory, is_python_module) File "/opt/conda/lib/python3.6/site-packages/torch/utils/cpp_extension.py", line 967, in _import_module_from_library file, path, description = imp.find_module(module_name, [path]) File "/opt/conda/lib/python3.6/imp.py", line 297, in find_module raise ImportError(_ERR_MSG.format(name), name=name) ImportError: No module named 'inplace_abn' Traceback (most recent call last): File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main "main", mod_spec) File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code exec(code, run_globals) File "/opt/conda/lib/python3.6/site-packages/torch/distributed/launch.py", line 235, in main() File "/opt/conda/lib/python3.6/site-packages/torch/distributed/launch.py", line 231, in main cmd=process.args) subprocess.CalledProcessError: Command '['/opt/conda/bin/python', '-u', 'tools/train.py', '--local_rank=0', '--cfg', 'experiments/cityscapes/seg_hrnet_w48_train_512x1024_sgd_lr1e-2_wd5e-4_bs_12_epoch484.yaml']' returned non-zero exit status 1. Singularity> ninja version

HRNet / HRNet-Semantic-Segmentation

How to solve nninja: build stopped: subcommand failed.\n'? #187