Open jorgtied opened 3 years ago
Training breaks with
[2021-05-12 10:19:19] [training] skipping 250846-th update due to loss being nan [2021-05-12 10:19:19] Error: CUDA error 700 'an illegal memory access was encountered' - /users/tiedeman/projappl/install/marian/src/tensors/gpu/cuda_helpers.h:67: cudaMemcpy(dest, start, (end - start) * sizeof(T), cudaMemcpyDefault) [2021-05-12 10:19:19] Error: Aborted from void CudaCopy(const T*, const T*, T*) [with T = unsigned int] in /users/tiedeman/projappl/install/marian/src/tensors/gpu/cuda_helpers.h:67
when training big transformer models on NVIDIA v100.
my command line:
marian --guided-alignment /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/train/opus.spm32k-spm32k.src-trg.alg.gz --early-stopping 10 --valid-freq 10000 --valid-sets /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/val/Tatoeba-dev.src.spm32k /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/val/Tatoeba-dev.trg.spm32k --valid-metrics perplexity --valid-mini-batch 16 --valid-log /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/opus.spm32k-spm32k.transformer-big-align.valid1.log --beam-size 12 --normalize 1 --allow-unk --overwrite --keep-best --model /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/opus.spm32k-spm32k.transformer-big-align.model1.npz --train-sets /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/train/opus.src.clean.spm32k.gz /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/train/opus.trg.clean.spm32k.gz --max-length 500 --vocabs /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/opus.spm32k-spm32k.vocab.yml /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/opus.spm32k-spm32k.vocab.yml --mini-batch-fit -w 24000 --maxi-batch 500 --save-freq 10000 --disp-freq 10000 --log /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/opus.spm32k-spm32k.transformer-big-align.train1.log --type transformer --enc-depth 12 --dec-depth 6 --dim-emb 1024 --transformer-heads 16 --transformer-postprocess-emb d --transformer-postprocess dan --transformer-dropout 0.1 --label-smoothing 0.1 --learn-rate 0.0003 --lr-warmup 16000 --lr-decay-inv-sqrt 16000 --lr-report --optimizer-params 0.9 0.98 1e-09 --clip-norm 5 --fp16 --tied-embeddings-all --devices 0 1 2 3 --sync-sgd --seed 1111 --sqlite --tempdir /run/nvme/job_5803302/data --exponential-smoothing
Size of training data: ca 45 million sentence pairs. Training works fine with smaller transformer models on the same data set.
Marian version: Paste the output of --version here v1.10.0 6f6d484 2021-02-06 15:35:16 -0800
--version
CMake command: Type the cmake command you used and attach the output of --build-info all
--build-info all
AVX2_FOUND=true AVX512_FOUND=true AVX_FOUND=true BUILD_ARCH=native CMAKE_ADDR2LINE=/usr/bin/addr2line CMAKE_AR=/usr/bin/ar CMAKE_BUILD_TYPE=Release CMAKE_COLOR_MAKEFILE=ON CMAKE_CXX_COMPILER=/appl/spack/install-tree/gcc-4.8.5/gcc-9.1.0-vpjht2/bin/g++ CMAKE_CXX_COMPILER_AR=/appl/spack/install-tree/gcc-4.8.5/gcc-9.1.0-vpjht2/bin/gcc-ar CMAKE_CXX_COMPILER_RANLIB=/appl/spack/install-tree/gcc-4.8.5/gcc-9.1.0-vpjht2/bin/gcc-ranlib CMAKE_CXX_FLAGS=-std=c++11 -pthread -Wl,--no-as-needed -fPIC -Wno-unused-result -march=native -msse2 -msse3 -msse4.1 -msse4.2 -mavx -mavx2 -mavx512f -DUSE_SENTENCEPIECE -DCUDA_FOUND -DUSE_NCCL -DMKL_ILP64 -m64 CMAKE_CXX_FLAGS_DEBUG=-O0 -g -rdynamic CMAKE_CXX_FLAGS_MINSIZEREL=-Os -DNDEBUG CMAKE_CXX_FLAGS_RELEASE=-O3 -m64 -funroll-loops -g -rdynamic CMAKE_CXX_FLAGS_RELWITHDEBINFO=-O3 -m64 -funroll-loops -g -rdynamic CMAKE_C_COMPILER=/appl/spack/install-tree/gcc-4.8.5/gcc-9.1.0-vpjht2/bin/gcc CMAKE_C_COMPILER_AR=/appl/spack/install-tree/gcc-4.8.5/gcc-9.1.0-vpjht2/bin/gcc-ar CMAKE_C_COMPILER_RANLIB=/appl/spack/install-tree/gcc-4.8.5/gcc-9.1.0-vpjht2/bin/gcc-ranlib CMAKE_C_FLAGS=-pthread -Wl,--no-as-needed -fPIC -Wno-unused-result -march=native -msse2 -msse3 -msse4.1 -msse4.2 -mavx -mavx2 -mavx512f -DMKL_ILP64 -m64 CMAKE_C_FLAGS_DEBUG=-O0 -g -rdynamic CMAKE_C_FLAGS_MINSIZEREL=-Os -DNDEBUG CMAKE_C_FLAGS_RELEASE=-O3 -m64 -funroll-loops -g -rdynamic CMAKE_C_FLAGS_RELWITHDEBINFO=-O3 -m64 -funroll-loops -g -rdynamic CMAKE_DLLTOOL=CMAKE_DLLTOOL-NOTFOUND CMAKE_EXE_LINKER_FLAGS=-static-libgcc -static-libstdc++ CMAKE_INSTALL_BINDIR=bin CMAKE_INSTALL_DATAROOTDIR=share CMAKE_INSTALL_INCLUDEDIR=include CMAKE_INSTALL_LIBDIR=lib64 CMAKE_INSTALL_LIBEXECDIR=libexec CMAKE_INSTALL_LOCALSTATEDIR=var CMAKE_INSTALL_OLDINCLUDEDIR=/usr/include CMAKE_INSTALL_PREFIX=/users/tiedeman/projappl CMAKE_INSTALL_SBINDIR=sbin CMAKE_INSTALL_SHAREDSTATEDIR=com CMAKE_INSTALL_SYSCONFDIR=etc CMAKE_LINKER=/usr/bin/ld CMAKE_MAKE_PROGRAM=/usr/bin/gmake CMAKE_NM=/usr/bin/nm CMAKE_OBJCOPY=/usr/bin/objcopy CMAKE_OBJDUMP=/usr/bin/objdump CMAKE_RANLIB=/usr/bin/ranlib CMAKE_READELF=/usr/bin/readelf CMAKE_SKIP_INSTALL_RPATH=NO CMAKE_SKIP_RPATH=NO CMAKE_STRIP=/usr/bin/strip CMAKE_VERBOSE_MAKEFILE=FALSE COMPILE_CPU=on COMPILE_CUDA=ON COMPILE_CUDA_SM35=ON COMPILE_CUDA_SM50=ON COMPILE_CUDA_SM60=ON COMPILE_CUDA_SM70=ON COMPILE_CUDA_SM75=ON COMPILE_CUDA_SM80=ON COMPILE_EXAMPLES=OFF COMPILE_LIBRARY_ONLY=OFF COMPILE_SERVER=OFF COMPILE_TESTS=OFF CUDA_64_BIT_DEVICE_CODE=ON CUDA_ATTACH_VS_BUILD_RULE_TO_CUDA_FILE=ON CUDA_BUILD_CUBIN=OFF CUDA_BUILD_EMULATION=OFF CUDA_CUDART_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcudart_static.a CUDA_CUDA_LIBRARY=CUDA_CUDA_LIBRARY-NOTFOUND CUDA_HOST_COMPILATION_CPP=ON CUDA_HOST_COMPILER=/appl/spack/install-tree/gcc-4.8.5/gcc-9.1.0-vpjht2/bin/gcc CUDA_NVCC_EXECUTABLE=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/bin/nvcc CUDA_NVCC_FLAGS=-DUSE_SENTENCEPIECE-DCUDA_FOUND-DUSE_NCCL--default-streamper-thread-O3-g--use_fast_math-arch=sm_35-gencode=arch=compute_35,code=sm_35-gencode=arch=compute_50,code=sm_50-gencode=arch=compute_52,code=sm_52-gencode=arch=compute_60,code=sm_60-gencode=arch=compute_61,code=sm_61-gencode=arch=compute_70,code=sm_70-gencode=arch=compute_70,code=compute_70-gencode=arch=compute_75,code=sm_75-gencode=arch=compute_75,code=compute_75-gencode=arch=compute_80,code=sm_80-gencode=arch=compute_80,code=compute_80-ccbin/appl/spack/install-tree/gcc-4.8.5/gcc-9.1.0-vpjht2/bin/gcc-std=c++11-Xcompiler -fPIC-Xcompiler -Wno-unused-result-Xcompiler -Wno-deprecated-Xcompiler -Wno-pragmas-Xcompiler -Wno-unused-value-Xcompiler -Werror-Xcompiler -msse2-Xcompiler -msse3-Xcompiler -msse4.1-Xcompiler -msse4.2-Xcompiler -mavx-Xcompiler -mavx2-Xcompiler -mavx512f CUDA_OpenCL_LIBRARY=CUDA_OpenCL_LIBRARY-NOTFOUND CUDA_PROPAGATE_HOST_FLAGS=OFF CUDA_SDK_ROOT_DIR=CUDA_SDK_ROOT_DIR-NOTFOUND CUDA_SEPARABLE_COMPILATION=OFF CUDA_TOOLKIT_INCLUDE=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/include CUDA_TOOLKIT_ROOT_DIR=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2 CUDA_USE_STATIC_CUDA_RUNTIME=ON CUDA_VERBOSE_BUILD=OFF CUDA_VERSION=11.1 CUDA_cublasLt_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcublasLt_static.a CUDA_cublas_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcublas_static.a CUDA_cudadevrt_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcudadevrt.a CUDA_cudart_static_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcudart_static.a CUDA_cufft_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcufft_static.a CUDA_culibos_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libculibos.a CUDA_cupti_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/extras/CUPTI/lib64/libcupti_static.a CUDA_curand_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcurand_static.a CUDA_cusolver_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcusolver_static.a CUDA_cusparse_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libcusparse_static.a CUDA_nppc_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppc_static.a CUDA_nppial_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppial_static.a CUDA_nppicc_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppicc_static.a CUDA_nppidei_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppidei_static.a CUDA_nppif_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppif_static.a CUDA_nppig_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppig_static.a CUDA_nppim_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppim_static.a CUDA_nppist_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppist_static.a CUDA_nppisu_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppisu_static.a CUDA_nppitc_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnppitc_static.a CUDA_npps_LIBRARY=/appl/spack/install-tree/gcc-9.1.0/cuda-11.1.0-vvfuk2/lib64/libnpps_static.a CUDA_nvToolsExt_LIBRARY=CUDA_nvToolsExt_LIBRARY-NOTFOUND CUDA_rt_LIBRARY=/usr/lib64/librt.a DOXYGEN_DOT_EXECUTABLE=DOXYGEN_DOT_EXECUTABLE-NOTFOUND DOXYGEN_EXECUTABLE=/usr/bin/doxygen GENERATE_MARIAN_INSTALL_TARGETS=OFF GIT_EXECUTABLE=/usr/bin/git INTEL_ROOT=/opt/intel INTGEMM_DONT_BUILD_TESTS=ON MKL_CORE_LIBRARY=/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64/libmkl_core.a MKL_INCLUDE_DIR=/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl/include MKL_INCLUDE_DIRS=/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl/include MKL_INTERFACE_LIBRARY=/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64/libmkl_intel_ilp64.a MKL_LIBRARIES=-Wl,--start-group/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64/libmkl_intel_ilp64.a/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64/libmkl_sequential.a/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64/libmkl_core.a-Wl,--end-group MKL_ROOT=/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl MKL_SEQUENTIAL_LAYER_LIBRARY=/appl/opt/cluster_studio_xe2019/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64/libmkl_sequential.a MPIEXEC_EXECUTABLE=/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/bin/mpiexec MPIEXEC_MAX_NUMPROCS=40 MPIEXEC_NUMPROC_FLAG=-n MPI_CXX_COMPILER=/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/bin/mpicxx MPI_CXX_COMPILER_INCLUDE_DIRS=/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include/openmpi/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include/openmpi/opal/mca/hwloc/hwloc201/hwloc/include/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include/openmpi/opal/mca/event/libevent2022/libevent/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include/openmpi/opal/mca/event/libevent2022/libevent/include/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include MPI_CXX_COMPILE_OPTIONS=-pthread MPI_CXX_HEADER_DIR=/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include MPI_CXX_LIB_NAMES=mpi MPI_CXX_LINK_FLAGS=-Wl,-rpath -Wl,/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/lib -L/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/lib -pthread MPI_CXX_SKIP_MPICXX=FALSE MPI_C_COMPILER=/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/bin/mpicc MPI_C_COMPILER_INCLUDE_DIRS=/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include/openmpi/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include/openmpi/opal/mca/hwloc/hwloc201/hwloc/include/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include/openmpi/opal/mca/event/libevent2022/libevent/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include/openmpi/opal/mca/event/libevent2022/libevent/include/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include MPI_C_COMPILE_OPTIONS=-pthread MPI_C_HEADER_DIR=/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/include MPI_C_LIB_NAMES=mpi MPI_C_LINK_FLAGS=-Wl,-rpath -Wl,/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/lib -L/appl/spack/install-tree/gcc-9.1.0/hpcx-mpi-2.4.0-dnpuei/lib -pthread MPI_mpi_LIBRARY=/lib64/libexempi.so Protobuf_INCLUDE_DIR=/projappl/project_2001194/usr/include Protobuf_LIBRARY=/projappl/project_2001194/usr/lib/libprotobuf.so Protobuf_LITE_LIBRARY_DEBUG=Protobuf_LITE_LIBRARY_DEBUG-NOTFOUND Protobuf_LITE_LIBRARY_RELEASE=/projappl/project_2001194/usr/lib/libprotobuf-lite.so Protobuf_PROTOC_EXECUTABLE=/projappl/project_2001194/usr/bin/protoc Protobuf_PROTOC_LIBRARY_DEBUG=Protobuf_PROTOC_LIBRARY_DEBUG-NOTFOUND Protobuf_PROTOC_LIBRARY_RELEASE=/projappl/project_2001194/usr/lib/libprotobuf.so SPM_BUILD_TEST=OFF SPM_COVERAGE=OFF SPM_ENABLE_NFKC_COMPILE=OFF SPM_ENABLE_SHARED=OFF SPM_ENABLE_TCMALLOC=ON SPM_ENABLE_TENSORFLOW_SHARED=OFF SPM_NO_THREADLOCAL=OFF SPM_TCMALLOC_STATIC=on SPM_USE_BUILTIN_PROTOBUF=off SQLITE_ENABLE_ASSERT_HANDLER=OFF SQLITE_ENABLE_COLUMN_METADATA=ON SQLITE_USE_LEGACY_STRUCT=OFF SSE2_FOUND=true SSE3_FOUND=true SSE4_1_FOUND=true SSE4_2_FOUND=true SSSE3_FOUND=true TCMALLOC_LIB=/projappl/project_2001194/lib Tcmalloc_INCLUDE_DIR=/users/tiedeman/projappl/usr/include Tcmalloc_LIBRARY=/users/tiedeman/projappl/usr/lib/libtcmalloc_minimal.a Tcmalloc_ROOT=/projappl/project_2001194/usr USE_APPLE_ACCELERATE=OFF USE_CCACHE=OFF USE_CUDNN=OFF USE_DOXYGEN=ON USE_FBGEMM=OFF USE_MKL=ON USE_MPI=on USE_NCCL=ON USE_OPENMP=OFF USE_SENTENCEPIECE=on USE_STATIC_LIBS=on
Log file: Attach your training/decoding logs
... [2021-05-12 10:08:32] Loading model from /users/tiedeman/research/Opus-MT-train/work-tatoeba/fin-eng/opus.spm32k-spm32k.transformer-big-align.model1.npz [2021-05-12 10:08:35] [memory] Reserving 906 MB, device cpu0 [2021-05-12 10:08:35] [memory] Reserving 226 MB, device gpu0 [2021-05-12 10:08:35] [memory] Reserving 226 MB, device gpu1 [2021-05-12 10:08:35] [memory] Reserving 226 MB, device gpu2 [2021-05-12 10:08:35] [memory] Reserving 226 MB, device gpu3 [2021-05-12 10:19:17] [training] skipping 250843-th update due to loss being nan [2021-05-12 10:19:18] [training] skipping 250844-th update due to loss being nan [2021-05-12 10:19:18] [training] skipping 250845-th update due to loss being nan [2021-05-12 10:19:19] [training] skipping 250846-th update due to loss being nan [2021-05-12 10:19:19] Error: CUDA error 700 'an illegal memory access was encountered' - /users/tiedeman/projappl/install/marian/src/tensors/gpu/cuda_helpers.h:67: cudaMemcpy(dest, start, (end - start) * sizeof(T), cudaMemcpyDefault) [2021-05-12 10:19:19] Error: Aborted from void CudaCopy(const T*, const T*, T*) [with T = unsigned int] in /users/tiedeman/projappl/install/marian/src/tensors/gpu/cuda_helpers.h:67 CUDA error 700 'an illegal memory access was encountered' - /users/tiedeman/projappl/install/marian/src/tensors/gpu/algorithm.cu:54: cudaStreamSynchronize(0) Aborted from void marian::gpu::fill(marian::Ptr<marian::Backend>, T*, T*, T) [with T = float; marian::Ptr<marian::Backend> = std::shared_ptr<marian::Backend>] in /users/tiedeman/projappl/install/marian/src/tensors/gpu/algorithm.cu:54 CUDA error 700 'an illegal memory access was encountered' - /users/tiedeman/projappl/install/marian/src/tensors/gpu/algorithm.cu:54: cudaStreamSynchronize(0) Aborted from void marian::gpu::fill(marian::Ptr<marian::Backend>, T*, T*, T) [with T = float; marian::Ptr<marian::Backend> = std::shared_ptr<marian::Backend>] in /users/tiedeman/projappl/install/marian/src/tensors/gpu/algorithm.cu:54 CUDA error 700 'an illegal memory access was encountered' - /users/tiedeman/projappl/install/marian/src/tensors/gpu/algorithm.cu:54: cudaStreamSynchronize(0) Aborted from void marian::gpu::fill(marian::Ptr<marian::Backend>, T*, T*, T) [with T = float; marian::Ptr<marian::Backend> = std::shared_ptr<marian::Backend>] in /users/tiedeman/projappl/install/marian/src/tensors/gpu/algorithm.cu:54
[CALL STACK] [0x1b0e567] void marian::gpu:: fill (std::shared_ptr, float, float, float) + 0x627 [0x1389e3d] void marian::TensorBase:: set (float) + 0x35d [0x157a7ca] [0x157e463] marian::inits::LambdaInit:: apply (IntrusivePtr) + 0x33 [0x157443f] marian::ConstantNode:: init () + 0x3f [0x1565b2d] marian::ExpressionGraph:: forward (std::cxx11::list<IntrusivePtr<marian::Chainable<IntrusivePtr>>,std::allocator<IntrusivePtr<marian::Chainable<IntrusivePtr>>>>&, bool) + 0x5d [0x15672f5] marian::ExpressionGraph:: forwardNext () + 0x2c5 [0x1734548] [0x17cb2b4] marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1}:: operator() () const + 0x54 [0x17cbdb0] std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base,std::future_base::_Result_base::_Deleter> (),std::future_base::_Task_setter<std::unique_ptr<std::future_base::_Result,std::future_base::_Result_base::_Deleter>,std::future_base::_Task_state<marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1},std::allocator,void ()>::_M_run()::{lambda()#1},void>>:: _M_invoke (std::_Any_data const&) + 0x20 [0x12e032b] std::future_base::_State_baseV2:: _M_do_set (std::function<std::unique_ptr<std::__future_base::_Result_base,std::future_base::_Result_base::_Deleter> ()>, bool) + 0x1b [0x7f694073d20b] + 0x620b [0x17c0e38] std::_Function_handler<void (),marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#3}>:: _M_invoke (std::_Any_data const&) + 0x108 [0x12e1d27] std::thread::_State_impl<std::thread::_Invoker<std::tuple<marian::ThreadPool::reserve(unsigned long)::{lambda()#1}>>>:: _M_run () + 0x157 [0x504ab20] [0x7f694073eea5] + 0x7ea5 [0x7f69401658cd] clone + 0x6d
[CALL STACK] [0x1afe617] void CudaCopy (unsigned int const, unsigned int const, unsigned int) + 0x3f7 [0x1aff02e] void marian::gpu:: copy (std::shared_ptr, unsigned int const, unsigned int const, unsigned int) + 0x45e [0x1587e63] void marian::TensorBase:: set (unsigned int const, unsigned int const) + 0x6f3 [0x15880f0] std::_Function_handler<void (IntrusivePtr),marian::inits::fromVector(std::vector<unsigned int,std::allocator> const&)::{lambda(IntrusivePtr)#1}>:: _M_invoke (std::_Any_data const&, IntrusivePtr&&) + 0x20 [0x1581137] marian::inits::LambdaInitConvert:: apply (IntrusivePtr) + 0x67 [0x157443f] marian::ConstantNode:: init () + 0x3f [0x1565b2d] marian::ExpressionGraph:: forward (std::cxx11::list<IntrusivePtr<marian::Chainable<IntrusivePtr>>,std::allocator<IntrusivePtr<marian::Chainable<IntrusivePtr>>>>&, bool) + 0x5d [0x15672f5] marian::ExpressionGraph:: forwardNext () + 0x2c5 [0x1734548] [0x17cb2b4] marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1}:: operator() () const + 0x54 [0x17cbdb0] std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base,std::future_base::_Result_base::_Deleter> (),std::future_base::_Task_setter<std::unique_ptr<std::future_base::_Result,std::future_base::_Result_base::_Deleter>,std::future_base::_Task_state<marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1},std::allocator,void ()>::_M_run()::{lambda()#1},void>>:: _M_invoke (std::_Any_data const&) + 0x20 [0x12e032b] std::future_base::_State_baseV2:: _M_do_set (std::function<std::unique_ptr<std::__future_base::_Result_base,std::future_base::_Result_base::_Deleter> ()>, bool) + 0x1b [0x7f694073d20b] + 0x620b [0x17c0e38] std::_Function_handler<void (),marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#3}>:: _M_invoke (std::_Any_data const&) + 0x108 [0x12e1d27] std::thread::_State_impl<std::thread::_Invoker<std::tuple<marian::ThreadPool::reserve(unsigned long)::{lambda()#1}>>>:: _M_run () + 0x157 [0x504ab20] [0x7f694073eea5] + 0x7ea5 [0x7f69401658cd] clone + 0x6d
Bug description
Training breaks with
when training big transformer models on NVIDIA v100.
How to reproduce
my command line:
Size of training data: ca 45 million sentence pairs. Training works fine with smaller transformer models on the same data set.
Context
Marian version: Paste the output of
--version
here v1.10.0 6f6d484 2021-02-06 15:35:16 -0800CMake command: Type the cmake command you used and attach the output of
--build-info all
Log file: Attach your training/decoding logs
[CALL STACK] [0x1b0e567] void marian::gpu:: fill(std::shared_ptr, float, float, float) + 0x627
[0x1389e3d] void marian::TensorBase:: set (float) + 0x35d
[0x157a7ca]) + 0x33
[0x157443f] marian::ConstantNode:: init () + 0x3f
[0x1565b2d] marian::ExpressionGraph:: forward (std::cxx11::list<IntrusivePtr<marian::Chainable<IntrusivePtr>>,std::allocator<IntrusivePtr<marian::Chainable<IntrusivePtr>>>>&, bool) + 0x5d
[0x15672f5] marian::ExpressionGraph:: forwardNext () + 0x2c5
[0x1734548] future_base::_Result_base::_Deleter> (),std::future_base::_Task_setter<std::unique_ptr<std::future_base::_Result,std::future_base::_Result_base::_Deleter>,std::future_base::_Task_state<marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1},std::allocator,void ()>::_M_run()::{lambda()#1},void>>:: _M_invoke (std::_Any_data const&) + 0x20
[0x12e032b] std::future_base::_State_baseV2:: _M_do_set (std::function<std::unique_ptr<std::__future_base::_Result_base,std::future_base::_Result_base::_Deleter> ()>, bool) + 0x1b
[0x7f694073d20b] + 0x620b
[0x17c0e38] std::_Function_handler<void (),marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#3}>:: _M_invoke (std::_Any_data const&) + 0x108
[0x12e1d27] std::thread::_State_impl<std::thread::_Invoker<std::tuple<marian::ThreadPool::reserve(unsigned long)::{lambda()#1}>>>:: _M_run () + 0x157
[0x504ab20]
[0x157e463] marian::inits::LambdaInit:: apply (IntrusivePtr
[0x17cb2b4] marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1}:: operator() () const + 0x54 [0x17cbdb0] std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base,std::
[0x7f694073eea5] + 0x7ea5 [0x7f69401658cd] clone + 0x6d
[CALL STACK] [0x1b0e567] void marian::gpu:: fill(std::shared_ptr, float, float, float) + 0x627
[0x1389e3d] void marian::TensorBase:: set (float) + 0x35d
[0x157a7ca]) + 0x33
[0x157443f] marian::ConstantNode:: init () + 0x3f
[0x1565b2d] marian::ExpressionGraph:: forward (std::cxx11::list<IntrusivePtr<marian::Chainable<IntrusivePtr>>,std::allocator<IntrusivePtr<marian::Chainable<IntrusivePtr>>>>&, bool) + 0x5d
[0x15672f5] marian::ExpressionGraph:: forwardNext () + 0x2c5
[0x1734548] future_base::_Result_base::_Deleter> (),std::future_base::_Task_setter<std::unique_ptr<std::future_base::_Result,std::future_base::_Result_base::_Deleter>,std::future_base::_Task_state<marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1},std::allocator,void ()>::_M_run()::{lambda()#1},void>>:: _M_invoke (std::_Any_data const&) + 0x20
[0x12e032b] std::future_base::_State_baseV2:: _M_do_set (std::function<std::unique_ptr<std::__future_base::_Result_base,std::future_base::_Result_base::_Deleter> ()>, bool) + 0x1b
[0x7f694073d20b] + 0x620b
[0x17c0e38] std::_Function_handler<void (),marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#3}>:: _M_invoke (std::_Any_data const&) + 0x108
[0x12e1d27] std::thread::_State_impl<std::thread::_Invoker<std::tuple<marian::ThreadPool::reserve(unsigned long)::{lambda()#1}>>>:: _M_run () + 0x157
[0x504ab20]
[0x157e463] marian::inits::LambdaInit:: apply (IntrusivePtr
[0x17cb2b4] marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1}:: operator() () const + 0x54 [0x17cbdb0] std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base,std::
[0x7f694073eea5] + 0x7ea5 [0x7f69401658cd] clone + 0x6d
[CALL STACK] [0x1b0e567] void marian::gpu:: fill(std::shared_ptr, float, float, float) + 0x627
[0x1389e3d] void marian::TensorBase:: set (float) + 0x35d
[0x157a7ca]) + 0x33
[0x157443f] marian::ConstantNode:: init () + 0x3f
[0x1565b2d] marian::ExpressionGraph:: forward (std::cxx11::list<IntrusivePtr<marian::Chainable<IntrusivePtr>>,std::allocator<IntrusivePtr<marian::Chainable<IntrusivePtr>>>>&, bool) + 0x5d
[0x15672f5] marian::ExpressionGraph:: forwardNext () + 0x2c5
[0x1734548] future_base::_Result_base::_Deleter> (),std::future_base::_Task_setter<std::unique_ptr<std::future_base::_Result,std::future_base::_Result_base::_Deleter>,std::future_base::_Task_state<marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1},std::allocator,void ()>::_M_run()::{lambda()#1},void>>:: _M_invoke (std::_Any_data const&) + 0x20
[0x12e032b] std::future_base::_State_baseV2:: _M_do_set (std::function<std::unique_ptr<std::__future_base::_Result_base,std::future_base::_Result_base::_Deleter> ()>, bool) + 0x1b
[0x7f694073d20b] + 0x620b
[0x17c0e38] std::_Function_handler<void (),marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#3}>:: _M_invoke (std::_Any_data const&) + 0x108
[0x12e1d27] std::thread::_State_impl<std::thread::_Invoker<std::tuple<marian::ThreadPool::reserve(unsigned long)::{lambda()#1}>>>:: _M_run () + 0x157
[0x504ab20]
[0x157e463] marian::inits::LambdaInit:: apply (IntrusivePtr
[0x17cb2b4] marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1}:: operator() () const + 0x54 [0x17cbdb0] std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base,std::
[0x7f694073eea5] + 0x7ea5 [0x7f69401658cd] clone + 0x6d
[CALL STACK] [0x1afe617] void CudaCopy(unsigned int const, unsigned int const, unsigned int) + 0x3f7
[0x1aff02e] void marian::gpu:: copy (std::shared_ptr, unsigned int const , unsigned int const, unsigned int) + 0x45e
[0x1587e63] void marian::TensorBase:: set (unsigned int const, unsigned int const) + 0x6f3
[0x15880f0] std::_Function_handler<void (IntrusivePtr),marian::inits::fromVector(std::vector<unsigned int,std::allocator> const&)::{lambda(IntrusivePtr)#1}>:: _M_invoke (std::_Any_data const&, IntrusivePtr&&) + 0x20
[0x1581137] marian::inits::LambdaInitConvert:: apply (IntrusivePtr) + 0x67
[0x157443f] marian::ConstantNode:: init () + 0x3f
[0x1565b2d] marian::ExpressionGraph:: forward (std::cxx11::list<IntrusivePtr<marian::Chainable<IntrusivePtr>>,std::allocator<IntrusivePtr<marian::Chainable<IntrusivePtr>>>>&, bool) + 0x5d
[0x15672f5] marian::ExpressionGraph:: forwardNext () + 0x2c5
[0x1734548] future_base::_Result_base::_Deleter> (),std::future_base::_Task_setter<std::unique_ptr<std::future_base::_Result,std::future_base::_Result_base::_Deleter>,std::future_base::_Task_state<marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1},std::allocator,void ()>::_M_run()::{lambda()#1},void>>:: _M_invoke (std::_Any_data const&) + 0x20
[0x12e032b] std::future_base::_State_baseV2:: _M_do_set (std::function<std::unique_ptr<std::__future_base::_Result_base,std::future_base::_Result_base::_Deleter> ()>, bool) + 0x1b
[0x7f694073d20b] + 0x620b
[0x17c0e38] std::_Function_handler<void (),marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#3}>:: _M_invoke (std::_Any_data const&) + 0x108
[0x12e1d27] std::thread::_State_impl<std::thread::_Invoker<std::tuple<marian::ThreadPool::reserve(unsigned long)::{lambda()#1}>>>:: _M_run () + 0x157
[0x504ab20]
[0x17cb2b4] marian::ThreadPool::enqueue<std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&>(std::function<void (unsigned long,unsigned long,unsigned long)> const&,unsigned long&,unsigned long&,unsigned long&)::{lambda()#1}:: operator() () const + 0x54 [0x17cbdb0] std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base,std::
[0x7f694073eea5] + 0x7ea5 [0x7f69401658cd] clone + 0x6d