search

STEllAR-GROUP / hpx

The C++ Standard Library for Parallelism and Concurrency
https://hpx.stellar-group.org
Boost Software License 1.0
2.52k stars 429 forks source link

Performance drop on Piz Daint #5767

Closed diehlpk closed 2 years ago

diehlpk commented 2 years ago

Expected Behavior

Same version as executed on September 21 should result in similar timings as in February 22.

Actual Behavior

HPX is four times slower

Steps to Reproduce the Problem

... Please be as specific as possible while describing how to reproduce your problem.

  1. Compile same versions and run the same example

Specifications

... Please describe your environment

diehlpk commented 2 years ago

printenv

ASSEMBLER_AARCH64=/opt/cray/pe/cce/11.0.0/binutils/cross/x86_64-aarch64/aarch64-linux-gnu/bin/as
PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_skylake=8.2
XALT_DIR=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10
LS_COLORS=no=00:fi=00:di=01;34:ln=00;36:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=41;33;01:ex=00;32:*.cmd=00;32:*.exe=01;32:*.com=01;32:*.bat=01;32:*.btm=01;32:*.dll=01;32:*.tar=00;31:*.tbz=00;31:*.tgz=00;31:*.rpm=00;31:*.deb=00;31:*.arj=00;31:*.taz=00;31:*.lzh=00;31:*.lzma=00;31:*.zip=00;31:*.zoo=00;31:*.z=00;31:*.Z=00;31:*.gz=00;31:*.bz2=00;31:*.tb2=00;31:*.tz2=00;31:*.tbz2=00;31:*.xz=00;31:*.avi=01;35:*.bmp=01;35:*.dl=01;35:*.fli=01;35:*.gif=01;35:*.gl=01;35:*.jpg=01;35:*.jpeg=01;35:*.mkv=01;35:*.mng=01;35:*.mov=01;35:*.mp4=01;35:*.mpg=01;35:*.pcx=01;35:*.pbm=01;35:*.pgm=01;35:*.png=01;35:*.ppm=01;35:*.svg=01;35:*.tga=01;35:*.tif=01;35:*.webm=01;35:*.webp=01;35:*.wmv=01;35:*.xbm=01;35:*.xcf=01;35:*.xpm=01;35:*.aiff=00;32:*.ape=00;32:*.au=00;32:*.flac=00;32:*.m4a=00;32:*.mid=00;32:*.mp3=00;32:*.mpc=00;32:*.ogg=00;32:*.voc=00;32:*.wav=00;32:*.wma=00;32:*.wv=00;32:
LD_LIBRARY_PATH=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/lib64:/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/lib:/opt/cray/pe/papi/6.0.0.4/lib64:/opt/cray/job/2.2.4-7.0.2.1_2.86__g36b56f4.ari/lib64:/scratch/snx3000/pdiehl/OctoTigerBuildChain/build/boost/lib/
PE_ATP_PKGCONFIG_VARIABLES=ATP_CFLAGS_@prgenv@_@language@
PE_LIBSCI_DEFAULT_VOLATILE_PRGENV=CRAYCLANG GNU INTEL
CRAYPE_LINK_TYPE=dynamic
PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_sandybridge=8.2
PE_TPSL_64_DEFAULT_GENCOMPS_CRAYCLANG_x86_64=90
HOSTTYPE=x86_64
ALPS_LLI_STATUS_OFFSET=1
APEX_TASK_SCATTERPLOT=0
CSCS_CUSTOM_ENV=true
XTPE_NETWORK_TARGET=aries
ATP_IGNORE_SIGTERM=1
PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_skylake=8.2
CRAY_UDREG_POST_LINK_OPTS=-L/opt/cray/udreg/2.3.2-7.0.2.1_2.52__g8175d3d.ari/lib64
PE_TPSL_DEFAULT_GENCOMPS_CRAYCLANG_haswell=90
SLURM_NODEID=0
PE_FFTW_DEFAULT_TARGET_ivybridge=ivybridge
PE_FFTW_DEFAULT_TARGET_share=share
SLURM_TASK_PID=26065
PE_TRILINOS_DEFAULT_GENCOMPS_CRAYCLANG_x86_64=90
PRGENVMODULES=PrgEnv-cray:PrgEnv-gnu:PrgEnv-intel:PrgEnv-pgi
PKG_CONFIG_PATH_DEFAULT=/opt/cray/pe/papi/6.0.0.4/lib64/pkgconfig
PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_sandybridge=19.0
PE_LIBSCI_OMP_REQUIRES=
PE_TRILINOS_DEFAULT_GENCOMPS_GNU_x86_64=82
SSH_CONNECTION=148.187.1.7 53552 148.187.26.65 22
PE_MPICH_NV_LIBS_nvidia35=-lcudart
LESSCLOSE=lessclose.sh %s %s
CRAY_LD_LIBRARY_PATH=/usr/local/cuda-11.0/lib64:/opt/cray/pe/mpt/7.7.16/gni/mpich-cray/9.0/lib:/opt/cray/pe/mpt/7.7.16/gni/mpich-crayclang/9.0/lib:/opt/cray/pe/perftools/20.10.0/lib64:/opt/cray/rca/2.2.20-7.0.2.1_2.93__g8e3fb5b.ari/lib64:/opt/cray/alps/6.6.59-7.0.2.1_3.85__g872a8d62.ari/lib64:/opt/cray/xpmem/2.2.20-7.0.2.1_2.72__g87eb960.ari/lib64:/opt/cray/dmapp/7.1.1-7.0.2.1_2.98__g38cf134.ari/lib64:/opt/cray/pe/pmi/5.0.17/lib64:/opt/cray/ugni/6.0.14.0-7.0.2.1_3.77__ge78e5b0.ari/lib64:/opt/cray/udreg/2.3.2-7.0.2.1_2.52__g8175d3d.ari/lib64:/opt/cray/pe/libsci/20.09.1/CRAYCLANG/9.0/x86_64/lib:/opt/cray/pe/cce/11.0.0/cce-clang/x86_64/lib:/opt/cray/pe/cce/11.0.0/cce/x86_64/lib
PE_SMA_DEFAULT_DIR_PGI_DEFAULT64=64
PE_MPICH_DEFAULT_DIR_CRAY_DEFAULT64=64
PE_PAPI_DEFAULT_ACCEL_FAMILY_LIBS_nvidia=,-lcupti,-lcudart,-lcuda
PE_LIBSCI_ACC_DEFAULT_PKGCONFIG_VARIABLES=PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_@accelerator@
SLURM_PRIO_PROCESS=0
XKEYSYMDB=/usr/X11R6/lib/X11/XKeysymDB
PE_TRILINOS_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/trilinos/12.18.1.1/@PRGENV@/@PE_TRILINOS_DEFAULT_GENCOMPS@/@PE_TRILINOS_DEFAULT_TARGET@/lib/pkgconfig
PE_ENV=CRAY
LINKER_AARCH64=/opt/cray/pe/cce/11.0.0/binutils/cross/x86_64-aarch64/aarch64-linux-gnu/bin/ld
PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_x86_skylake=190
CRAY_MPICH2_DIR=/opt/cray/pe/mpt/7.7.16/gni/mpich-crayclang/9.0
PE_LIBSCI_DEFAULT_GENCOMPS_GNU_x86_64=81
PE_PETSC_DEFAULT_GENCOMPS_GNU_x86_skylake=82
PE_PETSC_DEFAULT_GENCOMPILERS_CRAYCLANG_haswell=9.0
CRAYPAT_LD_LIBRARY_PATH=/opt/cray/pe/gcc-libs:/opt/cray/gcc-libs:/opt/cray/pe/perftools/20.10.0/lib64
SLURM_LOG_ACTIONS=yes
APPS_CSCS=/apps/cscs/daint
PE_PETSC_DEFAULT_VOLATILE_PRGENV=CRAYCLANG CRAYCLANG64 GNU GNU64 INTEL INTEL64
FTN_X86_64=/opt/cray/pe/cce/11.0.0/cce/x86_64
PE_PRODUCT_LIST=CRAYPE_HASWELL:CRAY_RCA:CRAY_ALPS:DVS:CRAY_XPMEM:CRAY_DMAPP:CRAY_PMI:CRAY_UGNI:CRAY_UDREG:CRAY_LIBSCI:CRAYPE:CRAY:PERFTOOLS:CRAYPAT
CSCS_PELOCAL_PRGENV=true
CRAYPAT_ROOT=/opt/cray/pe/perftools/20.10.0
CRAY_CRAYPE_PREFIX=/opt/cray/pe/craype/2.7.3
PE_TPSL_64_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI
CRAY_DMAPP_INCLUDE_OPTS=-I/opt/cray/dmapp/7.1.1-7.0.2.1_2.98__g38cf134.ari/include -I/opt/cray/gni-headers/5.0.12.0-7.0.2.1_2.31__g3b1768f.ari/include
PE_TPSL_DEFAULT_GENCOMPILERS_GNU_x86_64=8.2
PE_LIBSCI_MODULE_NAME=cray-libsci/20.09.1
LANG=en_US.utf8
PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_skylake=82
TZ=Europe/Zurich
PE_MPICH_DEFAULT_DIR_CRAYCLANG_DEFAULT64=64
CRAY_BINUTILS_ROOT_X86_64=/opt/cray/pe/cce/11.0.0/binutils/x86_64/x86_64-pc-linux-gnu/../
SLURM_SUBMIT_DIR=/scratch/snx3000/pdiehl/runs
PE_TPSL_DEFAULT_GENCOMPS_CRAYCLANG_x86_skylake=90
PE_MPICH_CXX_PKGCONFIG_LIBS=mpichcxx
PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_x86_skylake=19.0
PE_TPSL_64_DEFAULT_GENCOMPS_GNU_haswell=82
WINDOWMANAGER=xterm
PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_skylake=9.0
LESS=-M -I -R
PE_LIBSCI_DEFAULT_GENCOMPS_CRAYCLANG_x86_64=90
PE_PETSC_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/petsc/3.13.3.0/complex/@PRGENV@/@PE_PETSC_DEFAULT_GENCOMPS@/@PE_PETSC_DEFAULT_TARGET@/lib/pkgconfig
ATP_INSTALL_DIR=/opt/cray/pe/atp/3.8.1
JAVA_ROOT=/usr/lib64/jvm/java
XALT_BINARYDATA_SIZE=5000
HOSTNAME=nid02126
PE_TPSL_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_skylake=9.0
SLURM_CSCS=yes
PE_TPSL_DEFAULT_GENCOMPILERS_CRAYCLANG_sandybridge=9.0
OLDPWD=/scratch/snx3000/pdiehl
PE_TPSL_DEFAULT_GENCOMPS_GNU_haswell=82
APPS=/apps/daint
CSHEDIT=emacs
PE_TPSL_DEFAULT_GENCOMPILERS_GNU_haswell=8.2
SLURM_CPUS_PER_TASK=12
CRAY_RCA_INCLUDE_OPTS=-I/opt/cray/rca/2.2.20-7.0.2.1_2.93__g8e3fb5b.ari/include -I/opt/cray/krca/2.2.7-7.0.2.1_2.81__ge897ee1.ari/include -I/opt/cray-hss-devel/9.0.0/include
ENVIRONMENT=BATCH
PE_TPSL_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/tpsl/20.03.2/@PRGENV@/@PE_TPSL_DEFAULT_GENCOMPS@/@PE_TPSL_DEFAULT_TARGET@/lib/pkgconfig
GPG_TTY=not a tty
LESS_ADVANCED_PREPROCESSOR=no
PE_MPICH_DIR_CRAY_DEFAULT64=64
PE_ATP_MODULE_NAME=atp
PE_MPICH_GENCOMPS_CRAY=90
PE_FFTW_DEFAULT_TARGET_x86_64=x86_64
PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_x86_64=19.1
PE_PETSC_DEFAULT_GENCOMPS_CRAYCLANG_x86_skylake=90
COLORTERM=1
GCC_X86_64=/opt/gcc/8.1.0/snos
ASSEMBLER_X86_64=/opt/cray/pe/cce/11.0.0/binutils/x86_64/x86_64-pc-linux-gnu/bin/as
FPATH=:/opt/cray/pe/modules/3.2.11.4/init/sh_funcs/no_redirect:/opt/cray/pe/modules/3.2.11.4/init/sh_funcs/no_redirect:/opt/cray/pe/modules/3.2.11.4/init/sh_funcs/no_redirect
PE_TPSL_64_DEFAULT_GENCOMPS_CRAYCLANG_x86_skylake=90
PE_TPSL_64_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/tpsl/20.03.2/@PRGENV@64/@PE_TPSL_64_DEFAULT_GENCOMPS@/@PE_TPSL_64_DEFAULT_TARGET@/lib/pkgconfig
CRAY_PERFTOOLS_VERSION=20.10.0
PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAYCLANG_sandybridge=9.0
ROCR_VISIBLE_DEVICES=0
PE_PKGCONFIG__PRODUCTS=PE_ATP
PE_PETSC_DEFAULT_GENCOMPS_GNU_sandybridge=82
PE_NETCDF_HDF5PARALLEL_DEFAULT_REQUIRED_PRODUCTS=PE_HDF5_PARALLEL
SQUEUE_SORT=-t,e,S
ATP_CFLAGS=
JAVA_HOME=/usr/lib64/jvm/java
COMPILERRT_PATH_X86_64=/opt/cray/pe/cce/11.0.0/cce-clang/x86_64/lib/clang/11.0.0/lib/linux
PE_LIBSCI_GENCOMPILERS_CRAYCLANG_x86_64=9.0
PE_FFTW_DEFAULT_TARGET_x86_skylake=x86_skylake
PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_x86_skylake=19.1
SQUEUE_FORMAT=%.8i %.8u %.7a %.14j %.3t %9r %19S %.10M %.10L %.5D %.6C
SLURM_PROCID=0
APP2_STATE=20.10.0
PE_LIBSCI_ACC_DEFAULT_GENCOMPS_GNU_x86_64=81
LINKER_X86_64=/opt/cray/pe/cce/11.0.0/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld
PE_MPICH_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/mpt/7.7.16/gni/mpich-@PRGENV@@PE_MPICH_DEFAULT_DIR_DEFAULT64@/@PE_MPICH_DEFAULT_GENCOMPS@/lib/pkgconfig
SLURM_JOB_GID=31496
MACHTYPE=x86_64-suse-linux
PE_FFTW_DEFAULT_TARGET_broadwell=broadwell
XTPE_LINK_TYPE=dynamic
PE_PAPI_DEFAULT_ACCEL_LIBS=
PE_TRILINOS_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_64=9.0
SLURMD_NODENAME=nid02126
PE_LIBSCI_ACC_DEFAULT_GENCOMPS_CRAYCLANG_x86_64=90
PE_SMA_DEFAULT_COMPFLAG=
XALT_EXECUTABLE_TRACKING=yes
CRAY_CUDATOOLKIT_POST_LINK_OPTS=-L/usr/local/cuda-11.0/lib64 -L/usr/local/cuda-11.0/nvvm/lib64 -Wl,--as-needed,-lcupti,-lcudart,--no-as-needed -lcuda
PE_PKGCONFIG_PRODUCTS=PE_MPICH:PE_LIBSCI
PE_CRAY_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/pe/ga/5.3.0.10/CRAY/8.6/lib/pkgconfig
PE_PETSC_DEFAULT_GENCOMPS_CRAYCLANG_x86_64=90
CRAY_MPICH_BASEDIR=/opt/cray/pe/mpt/7.7.16/gni
PE_PETSC_DEFAULT_GENCOMPS_INTEL_sandybridge=191
PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_sandybridge=190
PE_LIBSCI_GENCOMPILERS_GNU_x86_64=8.1
MINICOM=-c on
DVS_VERSION=0.9.0
SLURM_TASKS_PER_NODE=1(x48)
APEX_PROFILE_OUTPUT=1
PAT_BUILD_PAPI_LIBDIR=/opt/cray/pe/papi/6.0.0.4/lib64
XALT_GPU_TRACKING=no
PE_MPICH_PKGCONFIG_VARIABLES=PE_MPICH_NV_LIBS_@accelerator@:PE_MPICH_ALTERNATE_LIBS_@multithreaded@:PE_MPICH_ALTERNATE_LIBS_@dpm@
PE_MPICH_PKGCONFIG_LIBS=mpich
PE_PARALLEL_NETCDF_DEFAULT_FIXED_PRGENV=PGI INTEL CRAYCLANG GNU
QT_SYSTEM_DIR=/usr/share/desktop-data
OSTYPE=linux
PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia60=nv60
PE_LEVEL=11.0
PE_MPICH_NV_LIBS=
PE_NETCDF_DEFAULT_REQUIRED_PRODUCTS=PE_HDF5
PE_FFTW2_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH
PE_PETSC_DEFAULT_GENCOMPS_GNU_x86_64=82
XDG_SESSION_ID=710289
PE_TPSL_DEFAULT_GENCOMPS_GNU_sandybridge=82
PE_TPSL_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI
SLURM_NNODES=48
USER=pdiehl
PAGER=less
MODULE_VERSION=3.2.11.4
CRAY_CXX_IPA_LIBS_AARCH64=/opt/cray/pe/cce/11.0.0/cce/aarch64/lib/libcray-c++-rts.a
CRAY_ALPS_POST_LINK_OPTS=-L/opt/cray/alps/6.6.59-7.0.2.1_3.85__g872a8d62.ari/lib64
PE_TPSL_DEFAULT_GENCOMPS_CRAYCLANG_x86_64=90
SHMEM_ABORT_ON_ERROR=1
PE_PKG_CONFIG_PATH=/opt/cray/pe/valgrind4hpc/2.8.1/lib/pkgconfig:/opt/cray/pe/cti/2.8.1/lib/pkgconfig:/opt/cray/pe/atp/3.8.1/lib/pkgconfig
PE_CRAYCLANG_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/pe/parallel-netcdf/1.12.1.0/CRAYCLANG/9.0/lib/pkgconfig:/opt/cray/pe/netcdf-hdf5parallel/4.7.4.0/CRAYCLANG/9.0/lib/pkgconfig:/opt/cray/pe/netcdf/4.7.4.0/CRAYCLANG/9.0/lib/pkgconfig:/opt/cray/pe/hdf5-parallel/1.12.0.0/CRAYCLANG/9.0/lib/pkgconfig:/opt/cray/pe/hdf5/1.12.0.0/CRAYCLANG/9.0/lib/pkgconfig
SLURM_NTASKS_PER_CORE=1
PE_MPICH_DEFAULT_GENCOMPILERS_CRAY=9.0
TOOLMODULES=apprentice:apprentice2:atp:chapel:cray-lgdb:craypat:craypkg-gen:cray-snplauncher:ddt:gdb:iobuf:papi:perftools:perftools-lite:stat:totalview:xt-craypat:xt-lgdb:xt-papi:xt-totalview
DVS_INCLUDE_OPTS=-I/opt/cray/dvs/2.12_2.2.177-7.0.2.1_13.5__g0b75e43d/include
PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia35=nv35
PE_LIBSCI_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH
PE_LIBSCI_GENCOMPILERS_INTEL_x86_64=16.0
PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_x86_64=8.2
CRAY_CPU_TARGET=haswell
PE_GA_DEFAULT_GENCOMPS_GNU=82 73
PE_INTEL_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/pe/parallel-netcdf/1.12.1.0/INTEL/19.1/lib/pkgconfig:/opt/cray/pe/netcdf-hdf5parallel/4.7.4.0/INTEL/19.1/lib/pkgconfig:/opt/cray/pe/netcdf/4.7.4.0/INTEL/19.1/lib/pkgconfig:/opt/cray/pe/mpt/7.7.16/gni/mpich-INTEL/16.0/lib/pkgconfig:/opt/cray/pe/hdf5-parallel/1.12.0.0/INTEL/19.1/lib/pkgconfig:/opt/cray/pe/hdf5/1.12.0.0/INTEL/19.1/lib/pkgconfig:/opt/cray/pe/ga/5.3.0.10/INTEL/18.0/lib/pkgconfig
KSH_AUTOLOAD=1
PE_LIBSCI_VOLATILE_PRGENV=CRAYCLANG GNU INTEL
PE_SMA_DEFAULT_PKGCONFIG_VARIABLES=PE_SMA_COMPFLAG_@prgenv@
PE_TPSL_64_DEFAULT_GENCOMPS_CRAYCLANG_sandybridge=90
PE_MPICH_GENCOMPILERS_PGI=20.1
SLURM_NTASKS_PER_NODE=1
PE_PETSC_DEFAULT_GENCOMPILERS_CRAYCLANG_sandybridge=9.0
PKGCONFIG_ENABLED=1
CRAY_UGNI_POST_LINK_OPTS=-L/opt/cray/ugni/6.0.14.0-7.0.2.1_3.77__ge78e5b0.ari/lib64
MORE=-sl
PE_MPICH_GENCOMPS_GNU=82 81 71
PE_PAPI_DEFAULT_ACCEL_LIBS_nvidia35=,-lcupti,-lcudart,-lcuda
CRAY_PERFTOOLS_PREFIX=/opt/cray/pe/perftools/20.10.0
PE_FORTRAN_PKGCONFIG_LIBS=mpichf90
PE_MPICH_DEFAULT_GENCOMPILERS_CRAYCLANG=9.0
PE_TRILINOS_DEFAULT_GENCOMPILERS_INTEL_x86_64=19.1
CRAY_CXX_IPA_LIBS=/opt/cray/pe/cce/11.0.0/cce/x86_64/lib/libcray-c++-rts.a
PE_TPSL_64_DEFAULT_GENCOMPS_CRAYCLANG_haswell=90
PE_MPICH_GENCOMPILERS_CRAY=9.0
CRAY_LIBSCI_BASE_DIR=/opt/cray/pe/libsci/20.09.1
PE_NETCDF_HDF5PARALLEL_DEFAULT_FIXED_PRGENV=GNU CRAYCLANG PGI INTEL
PWD=/scratch/snx3000/pdiehl/runs
TARGETMODULES=craype-abudhabi:craype-abudhabi-cu:craype-accel-host:craype-accel-nvidia20:craype-accel-nvidia30:craype-accel-nvidia35:craype-barcelona:craype-broadwell:craype-haswell:craype-hugepages128K:craype-hugepages128M:craype-hugepages16M:craype-hugepages256M:craype-hugepages2M:craype-hugepages32M:craype-hugepages4M:craype-hugepages512K:craype-hugepages512M:craype-hugepages64M:craype-hugepages8M:craype-intel-knc:craype-interlagos:craype-interlagos-cu:craype-istanbul:craype-ivybridge:craype-mc12:craype-mc8:craype-mic-knl:craype-network-aries:craype-network-gemini:craype-network-infiniband:craype-network-none:craype-network-seastar:craype-sandybridge:craype-shanghai:craype-target-compute_node:craype-target-local_host:craype-target-native:craype-xeon:xtpe-barcelona:xtpe-interlagos:xtpe-interlagos-cu:xtpe-istanbul:xtpe-mc12:xtpe-mc8:xtpe-network-gemini:xtpe-network-seastar:xtpe-shanghai:xtpe-target-native:xtpe-xeon
PE_MPICH_NV_LIBS_nvidia20=-lcudart
SLURM_JOB_NODELIST=nid0[2126-2141,7188-7219]
HOME=/users/pdiehl
SLURM_CLUSTER_NAME=daint
CRAY_PMI_INCLUDE_OPTS=-I/opt/cray/pe/pmi/5.0.17/include
PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_sandybridge=19.0
CRAYLIBS_AARCH64=/opt/cray/pe/cce/11.0.0/cce/aarch64/lib
PE_TPSL_64_DEFAULT_GENCOMPILERS_GNU_haswell=8.2
PE_PETSC_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_skylake=9.0
PELOCAL_PRGENV=true
PE_PETSC_DEFAULT_GENCOMPS_CRAYCLANG_haswell=90
PE_TPSL_DEFAULT_GENCOMPS_CRAYCLANG_sandybridge=90
PE_TPSL_64_DEFAULT_GENCOMPS_GNU_sandybridge=82
CMAKE_PREFIX_PATH=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10
PE_LIBSCI_REQUIRED_PRODUCTS=PE_MPICH
craype_already_loaded=0
ATP_HOME=/opt/cray/pe/atp/3.8.1/alps
SLURM_NODELIST=nid0[2126-2141,7188-7219]
HOST=daint102
SSH_CLIENT=148.187.1.7 53552 22
ALT_LINKER=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/bin/ld
PE_PETSC_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI:PE_HDF5_PARALLEL:PE_TPSL
APEX_SCREEN_OUTPUT=1
CRAY_PRGENVCRAY=loaded
CUDA_HOME=/usr/local/cuda-11.0
SINFO_FORMAT=%9P %5a %8s %.10l %.6c %.6z %.7D %10T %N
PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_haswell=19.0
XNLSPATH=/usr/share/X11/nls
PE_MPICH_FORTRAN_PKGCONFIG_LIBS=mpichf90
CPATH=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/include
SLURM_NTASKS=48
PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_x86_64=19.0
PE_LIBSCI_ACC_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/libsci_acc/20.10.1/@PRGENV@/@PE_LIBSCI_ACC_DEFAULT_GENCOMPS@/@PE_LIBSCI_ACC_DEFAULT_TARGET@/lib/pkgconfig
PE_LIBSCI_PKGCONFIG_LIBS=libsci_mpi:libsci
PE_MPICH_DEFAULT_GENCOMPS_CRAY=90
PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_haswell=19.0
XALT_TRANSMISSION_STYLE=curl
APEX_ENABLE_CUDA=1
SDK_HOME=/usr/lib64/jvm/java
PE_LIBSCI_OMP_REQUIRES_openmp=_mp
KRB5CCNAME=FILE:/tmp/krb5cc_25723_1HEERn
SLURM_JOB_CPUS_PER_NODE=24(x48)
CRAY_CUDATOOLKIT_DIR=/usr/local/cuda-11.0
CRAYLIBS_X86_64=/opt/cray/pe/cce/11.0.0/cce/x86_64/lib
XDG_DATA_DIRS=/usr/share
CUDA_CACHE_PATH=/scratch/snx3000/pdiehl/.nv/ComputeCache
PE_CXX_PKGCONFIG_LIBS=mpichcxx
SLURM_TOPOLOGY_ADDR=s21.s5.nid02126
CRAY_BINUTILS_ROOT=/opt/cray/pe/cce/11.0.0/binutils/x86_64/x86_64-pc-linux-gnu/../
PE_SMA_DEFAULT_DIR_CRAYCLANG_DEFAULT64=64
PROJECT=/project/d69/pdiehl
cce_already_loaded=0
PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_haswell=190
CRAY_RCA_POST_LINK_OPTS=-L/opt/cray/rca/2.2.20-7.0.2.1_2.93__g8e3fb5b.ari/lib64 -lrca
PE_MPICH_DEFAULT_GENCOMPS_PGI=201
PE_MPICH_MODULE_NAME=cray-mpich
LIBGL_DEBUG=quiet
CRAY_LIBSCI_DIR=/opt/cray/pe/libsci/20.09.1
NLSPATH=/opt/cray/pe/cce/11.0.0/cce/x86_64/share/nls/En/%N.cat
SLURM_WORKING_CLUSTER=daint:daintsl01:6817:9216:108
PE_LIBSCI_DEFAULT_OMP_REQUIRES=
PE_LIBSCI_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/libsci/20.09.1/@PRGENV@/@PE_LIBSCI_GENCOMPS@/@PE_LIBSCI_TARGET@/lib/pkgconfig
JDK_HOME=/usr/lib64/jvm/java
PE_MPICH_TARGET_VAR_nvidia35=-lcudart
ATP_VERSION=3.8.1
PE_TPSL_DEFAULT_GENCOMPILERS_GNU_sandybridge=8.2
COMPILER_PATH=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/bin
LIBSCI_VERSION=20.09.1
SLURM_JOB_NAME=apex-octo
CRAY_UDREG_INCLUDE_OPTS=-I/opt/cray/udreg/2.3.2-7.0.2.1_2.52__g8175d3d.ari/include
PROFILEREAD=true
TMPDIR=/tmp
PE_TRILINOS_DEFAULT_VOLATILE_PRGENV=CRAYCLANG GNU INTEL
PE_MPICH_DIR_PGI_DEFAULT64=64
EBROOTXALT=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10
LIBRARY_PATH=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/lib64:/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/lib
PERFTOOLS_VERSION=20.10.0
PE_MPICH_DEFAULT_GENCOMPILERS_GNU=8.2 8.1 7.1
SLURM_JOB_GPUS=0
PE_LIBSCI_DEFAULT_OMP_REQUIRES_openmp=_mp
CRAY_BINUTILS_VERSION=/opt/cray/pe/cce/11.0.0
CRAY_CCE_CLANGSHARE=/opt/cray/pe/cce/11.0.0/cce-clang/x86_64/share
PE_PKGCONFIG_LIBS=cray-cudatoolkit:mpich:libAtpSigHandler:cray-rca:libsci_mpi:libsci
CRAY_PMI_PREFIX=/opt/cray/pe/pmi/5.0.17
SLURM_JOBID=36590600
RCLOCAL_PRGENV=true
PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_skylake=82
USERMODULES=acml:alps:apprentice:apprentice2:atp:blcr:cce:chapel:cray-ccdb:cray-fftw:cray-ga:cray-hdf5:cray-hdf5-parallel:cray-lgdb:cray-libsci:cray-libsci_acc:cray-mpich:cray-mpich2:cray-mpich-compat:cray-netcdf:cray-netcdf-hdf5parallel:cray-parallel-netcdf:craypat:craype:cray-petsc:cray-petsc-complex:craypkg-gen:cray-shmem:cray-snplauncher:cray-tpsl:cray-trilinos:cudatoolkit:ddt:fftw:ga:gcc:hdf5:hdf5-parallel:intel:iobuf:java:lgdb:libfast:libsci_acc:mpich1:netcdf:netcdf-hdf5parallel:netcdf-nofsync:netcdf-nofsync-hdf5parallel:ntk:onesided:papi:parallel-netcdf:pathscale:perftools:perftools-lite:petsc:petsc-complex:pgi:pmi:PrgEnv-cray:PrgEnv-gnu:PrgEnv-intel:PrgEnv-pathscale:PrgEnv-pgi:stat:totalview:tpsl:trilinos:xt-asyncpe:xt-craypat:xt-lgdb:xt-libsci:xt-mpich2:xt-mpt:xt-papi:xt-shmem:xt-totalview
PAT_REPORT_PRUNE_NAME=_cray$mt_execute_,_cray$mt_start_,_cray$mt_kmpc_fork,__cray_hwpc_,f_cray_hwpc_,cstart,hip_impl::,hipLaunchKernelGGL,__pat_,pat_region_,PAT_,OMP.slave_loop,slave_entry,_new_slave_entry,_thread_pool_slave_entry,THREAD_POOL_join,__libc_start_main,_start,__start,start_thread,__wrap_,UPC_ADIO_,_upc_,upc_,__caf_,__pgas_,syscall,__device_stub,__cray_acc_hw,_ZZ,.omp_outlined.
SLURM_CONF=/etc/opt/slurm/slurm.conf
PE_PETSC_DEFAULT_GENCOMPILERS_GNU_sandybridge=8.2
PE_MPICH_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/mpt/7.7.16/gni/mpich-@PRGENV@@PE_MPICH_DIR_DEFAULT64@/@PE_MPICH_GENCOMPS@/lib/pkgconfig
LOADEDMODULES=modules/3.2.11.4:cge/3.2.1463_r03f4dfb_fe3.3.0_2019062614:craype-network-aries:cce/11.0.0:craype/2.7.3:cray-libsci/20.09.1:udreg/2.3.2-7.0.2.1_2.52__g8175d3d.ari:ugni/6.0.14.0-7.0.2.1_3.77__ge78e5b0.ari:pmi/5.0.17:dmapp/7.1.1-7.0.2.1_2.98__g38cf134.ari:gni-headers/5.0.12.0-7.0.2.1_2.34__g3b1768f.ari:xpmem/2.2.20-7.0.2.1_2.72__g87eb960.ari:job/2.2.4-7.0.2.1_2.86__g36b56f4.ari:dvs/2.12_2.2.177-7.0.2.1_13.5__g0b75e43d:alps/6.6.59-7.0.2.1_3.85__g872a8d62.ari:rca/2.2.20-7.0.2.1_2.93__g8e3fb5b.ari:atp/3.8.1:perftools-base/20.10.0:PrgEnv-cray/6.0.9:cray-mpich/7.7.16:slurm/20.11.7-7:craype-haswell:xalt/2.8.10:daint-gpu/20.11:cudatoolkit/11.0.2_3.38-8.1__g5b73779:Base-opts/2.4.142-7.0.2.1_2.72__g8f27585.ari
PE_TPSL_DEFAULT_GENCOMPILERS_INTEL_x86_64=19.0
PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_GNU_x86_64=8.1
CRAYPE_NETWORK_TARGET=aries
CRAY_CUDATOOLKIT_INCLUDE_OPTS=-I/usr/local/cuda-11.0/include -I/usr/local/cuda-11.0/nvvm/include -I/usr/local/cuda-11.0/extras/Debugger/include
PE_GA_DEFAULT_FIXED_PRGENV=CRAY PGI INTEL
INCLUDE_PATH_AARCH64=/opt/cray/pe/cce/11.0.0/cce/aarch64/include/craylibs
CRAY_ALPS_INCLUDE_OPTS=-I/opt/cray/alps/6.6.59-7.0.2.1_3.85__g872a8d62.ari/include
CRAY_PE_USE_CLANG=/opt/cray/pe/cce/11.0.0/cce-clang/x86_64/bin/clang
CRAY_UGNI_INCLUDE_OPTS=-I/opt/cray/ugni/6.0.14.0-7.0.2.1_3.77__ge78e5b0.ari/include
PE_MPICH_DEFAULT_GENCOMPILERS_PGI=20.1
PE_TPSL_DEFAULT_GENCOMPS_INTEL_haswell=190
SCRATCH=/scratch/snx3000/pdiehl
GCC_AARCH64=/opt/gcc-cross-aarch64/8.1.0/aarch64
RCLOCAL_BASEOPTS=true
LIBRARYMODULES=acml:alps:cray-dwarf:cray-fftw:cray-ga:cray-hdf5:cray-hdf5-parallel:cray-libsci:cray-libsci_acc:cray-mpich:cray-mpich2:cray-mpich-abi:cray-netcdf:cray-netcdf-hdf5parallel:cray-parallel-netcdf:cray-petsc:cray-petsc-complex:cray-shmem:cray-tpsl:cray-trilinos:cudatoolkit:fftw:ga:hdf5:hdf5-parallel:iobuf:libfast:netcdf:netcdf-hdf5parallel:ntk:onesided:papi:petsc:petsc-complex:pmi:tpsl:trilinos:xt-libsci:xt-mpich2:xt-mpt:xt-papi
SLURM_NODE_ALIASES=(null)
PE_MPICH_ALTERNATE_LIBS_dpm=_dpm
SLURM_JOB_QOS=normal
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.node
PE_FFTW_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/fftw/3.3.8.8/@PE_FFTW_DEFAULT_TARGET@/lib/pkgconfig
PE_TPSL_DEFAULT_GENCOMPS_INTEL_x86_64=190
APEX_ENABLE=1
SSH_TTY=/dev/pts/59
PE_TRILINOS_DEFAULT_GENCOMPS_INTEL_x86_64=191
MPICH_ABORT_ON_ERROR=1
PE_PAPI_DEFAULT_PKGCONFIG_VARIABLES=PE_PAPI_ACCEL_LIBS_@accelerator@ PE_PAPI_ACCEL_FAMILY_LIBS_@accelerator_family@
FROM_HEADER=
PE_LIBSCI_DEFAULT_GENCOMPILERS_INTEL_x86_64=16.0
PE_LIBSCI_GENCOMPS_INTEL_x86_64=160
CRAY_MPICH_ROOTDIR=/opt/cray/pe/mpt/7.7.16
ALPS_APP_PE=0
MAIL=/var/mail/pdiehl
PE_MPICH_DEFAULT_DIR_PGI_DEFAULT64=64
CRAY_CCE_SHARE=/opt/cray/pe/cce/11.0.0/cce/x86_64/share
PE_HDF5_PARALLEL_DEFAULT_FIXED_PRGENV=GNU CRAYCLANG PGI INTEL
PE_MPICH_VOLATILE_PRGENV=PGI GNU CRAYCLANG CRAY
PE_PETSC_DEFAULT_GENCOMPS_CRAYCLANG_sandybridge=90
PE_INTEL_FIXED_PKGCONFIG_PATH=/opt/cray/pe/mpt/7.7.16/gni/mpich-INTEL/16.0/lib/pkgconfig
SLURM_CPUS_ON_NODE=24
CRAY_BINUTILS_BIN_AARCH64=/opt/cray/pe/cce/11.0.0/binutils/cross/x86_64-aarch64/aarch64-linux-gnu/bin
PE_HDF5_DEFAULT_FIXED_PRGENV=GNU CRAYCLANG PGI INTEL
PE_MPICH_ALTERNATE_LIBS_multithreaded=_mt
PE_TPSL_DEFAULT_GENCOMPS_INTEL_sandybridge=190
XALT_SCALAR_SAMPLING=no
PE_TPSL_DEFAULT_GENCOMPS_GNU_x86_64=82
PE_LIBSCI_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_64=9.0
PE_LIBSCI_GENCOMPS_CRAYCLANG_x86_64=90
SLURM_JOB_NUM_NODES=48
PE_PGI_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/pe/parallel-netcdf/1.12.1.0/PGI/20.1/lib/pkgconfig:/opt/cray/pe/netcdf-hdf5parallel/4.7.4.0/PGI/20.1/lib/pkgconfig:/opt/cray/pe/netcdf/4.7.4.0/PGI/20.1/lib/pkgconfig:/opt/cray/pe/hdf5-parallel/1.12.0.0/PGI/20.1/lib/pkgconfig:/opt/cray/pe/hdf5/1.12.0.0/PGI/20.1/lib/pkgconfig:/opt/cray/pe/ga/5.3.0.10/PGI/17.10/lib/pkgconfig
PE_PETSC_DEFAULT_GENCOMPILERS_GNU_x86_skylake=8.2
PE_MPICH_GENCOMPS_PGI=201
SLURM_MEM_PER_NODE=61000
PE_FFTW_DEFAULT_TARGET_haswell=haswell
PE_GA_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/ga/5.3.0.10/@PRGENV@/@PE_GA_DEFAULT_GENCOMPS@/lib/pkgconfig
PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_haswell=19.1
LESSKEY=/etc/lesskey.bin
CRAY_SITE_LIST_DIR=/etc/opt/cray/pe/modules
SHELL=/usr/local/bin/bash
TERM=xterm-256color
PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAYCLANG_haswell=9.0
PE_LIBSCI_ACC_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_LIBSCI
PE_MPICH_GENCOMPILERS_CRAYCLANG=9.0
CRAY_BINUTILS_ROOT_AARCH64=/opt/cray/pe/cce/11.0.0/binutils/cross/x86_64-aarch64/aarch64-linux-gnu/../
CRAY_LIBSCI_PREFIX=/opt/cray/pe/libsci/20.09.1/CRAYCLANG/9.0/x86_64
PE_MPICH_FIXED_PRGENV=INTEL
PE_TPSL_64_DEFAULT_GENCOMPS_GNU_x86_64=82
PMI_NO_FORK=1
PE_LIBSCI_ACC_DEFAULT_NV_SUFFIX_nvidia20=nv20
EBDEVELXALT=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/easybuild/xalt-2.8.10-easybuild-devel
SLURM_JOB_UID=25723
CRAY_LIBSCI_VERSION=20.09.1
PE_TPSL_DEFAULT_VOLATILE_PRGENV=CRAYCLANG CRAYCLANG64 GNU GNU64 INTEL INTEL64
XCURSOR_THEME=DMZ
LS_OPTIONS=-N --color=none -T 0
CRAYLMD_LICENSE_FILE=/opt/cray/pe/cce/cce.lic
SLURM_JOB_PARTITION=normal
PE_MPICH_DEFAULT_GENCOMPS_GNU=82 81 71
PE_LIBSCI_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/libsci/20.09.1/@PRGENV@/@PE_LIBSCI_DEFAULT_GENCOMPS@/@PE_LIBSCI_DEFAULT_TARGET@/lib/pkgconfig
SLURM_TIME_FORMAT=relative
CRAY_CXX_IPA_LIBS_X86_64=/opt/cray/pe/cce/11.0.0/cce/x86_64/lib/libcray-c++-rts.a
CRAY_BINUTILS_BIN_X86_64=/opt/cray/pe/cce/11.0.0/binutils/x86_64/bin
PE_LIBSCI_DEFAULT_PKGCONFIG_VARIABLES=PE_LIBSCI_DEFAULT_OMP_REQUIRES_@openmp@:PE_SCI_EXT_LIBPATH:PE_SCI_EXT_LIBNAME
CC_X86_64=/opt/cray/pe/cce/11.0.0/cce/x86_64
PE_SMA_DEFAULT_COMPFLAG_GNU=-fcray-pointer
CRAY_XPMEM_POST_LINK_OPTS=-L/opt/cray/xpmem/2.2.20-7.0.2.1_2.72__g87eb960.ari/lib64
FORTRAN_SYSTEM_MODULE_NAMES=ftn_lib_definitions
PE_PETSC_DEFAULT_GENCOMPS_GNU_haswell=82
PE_SMA_DEFAULT_VOLATILE_PKGCONFIG_PATH=/opt/cray/pe/mpt/7.7.16/gni/sma@PE_SMA_DEFAULT_DIR_DEFAULT64@/lib64/pkgconfig
PE_LIBSCI_ACC_DEFAULT_VOLATILE_PRGENV=CRAYCLANG GNU
PE_GNU_DEFAULT_FIXED_PKGCONFIG_PATH=/opt/cray/pe/parallel-netcdf/1.12.1.0/GNU/8.2/lib/pkgconfig:/opt/cray/pe/netcdf-hdf5parallel/4.7.4.0/GNU/8.2/lib/pkgconfig:/opt/cray/pe/netcdf/4.7.4.0/GNU/8.2/lib/pkgconfig:/opt/cray/pe/hdf5-parallel/1.12.0.0/GNU/8.2/lib/pkgconfig:/opt/cray/pe/hdf5/1.12.0.0/GNU/8.2/lib/pkgconfig
PE_LIBSCI_ACC_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_64=9.0
TMOUT=259200
CRAY_CUDATOOLKIT_PREFIX=/usr/local/cuda-11.0
PE_MPICH_GENCOMPILERS_GNU=8.2 8.1 7.1
SLURM_JOB_USER=pdiehl
CUDA_VISIBLE_DEVICES=0
CRAY_PE_CCE_VARIANT=CC=Clang:FTN=Classic
CRAY_DMAPP_POST_LINK_OPTS=-L/opt/cray/dmapp/7.1.1-7.0.2.1_2.98__g38cf134.ari/lib64
SLURM_NPROCS=48
PE_LIBSCI_DEFAULT_GENCOMPS_INTEL_x86_64=160
CRAY_CC_VERSION=11.0.0
CRAY_MPICH2_VER=7.7.16
PE_GA_DEFAULT_GENCOMPILERS_GNU=8.2 7.3
SHLVL=2
PE_PETSC_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_64=9.0
CUDATOOLKIT_HOME=/usr/local/cuda-11.0
SLURM_SUBMIT_HOST=daint102
G_FILENAME_ENCODING=@locale,UTF-8,ISO-8859-15,CP1252
CRAY_LIBSCI_PREFIX_DIR=/opt/cray/pe/libsci/20.09.1/CRAYCLANG/9.0/x86_64
PE_TRILINOS_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH:PE_HDF5_PARALLEL:PE_NETCDF_HDF5PARALLEL:PE_LIBSCI:PE_TPSL
SLURM_JOB_ACCOUNT=d69
CRAY_CRAYPE_VERSION=2.7.3
PE_PAPI_DEFAULT_ACCELL_FAMILY_LIBS=
CRAY_BINUTILS_BIN=/opt/cray/pe/cce/11.0.0/binutils/x86_64/bin
CRAY_CUDATOOLKIT_VERSION=11.0.2_3.38-8.1__g5b73779
MANPATH=/usr/local/cuda-11.0/doc/man:/opt/cray/pe/mpt/7.7.16/gni/man/mpich:/opt/cray/pe/perftools/20.10.0/man:/opt/cray/pe/papi/6.0.0.4/share/pdoc/man:/opt/cray/pe/atp/3.8.1/share/man:/opt/cray/alps/6.6.59-7.0.2.1_3.85__g872a8d62.ari/man:/opt/cray/pe/pmi/5.0.17/man:/opt/cray/pe/libsci/20.09.1/man:/opt/cray/pe/man/csmlversion:/opt/cray/pe/craype/2.7.3/man:/opt/cray/pe/cce/11.0.0/cce-clang/x86_64/share/man:/opt/cray/pe/cce/11.0.0/man:/apps/cscs/daint/share/man:/opt/cray/cge/3.2.1463_r03f4dfb_fe3.3.0_2019062614/man:/opt/cray/pe/modules/3.2.11.4/share/man:/usr/local/man:/usr/local/share/man:/usr/share/man:/usr/man:/opt/cray/share/man:/opt/cray/pe/man
PE_FFTW_DEFAULT_TARGET_mic_knl=mic_knl
CRAY_FTN_VERSION=11.0.0
PE_MPICH_DEFAULT_GENCOMPS_CRAYCLANG=90
PE_TPSL_64_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_64=9.0
PE_PETSC_DEFAULT_GENCOMPS_INTEL_haswell=191
ALPS_APP_ID=18446744052271305736
MPICH_DIR=/opt/cray/pe/mpt/7.7.16/gni/mpich-crayclang/9.0
ATP_CFLAGS_GNU_FORTRAN=-fno-backtrace
PE_FFTW_DEFAULT_TARGET_sandybridge=sandybridge
PE_PETSC_DEFAULT_GENCOMPILERS_INTEL_sandybridge=19.1
MODULEPATH=/apps/daint/UES/jenkins/7.0.UP02-20.11/gpu/easybuild/tools/modules/all:/apps/daint/UES/jenkins/7.0.UP02-20.11/gpu/easybuild/modules/all:/apps/daint/modulefiles:/apps/daint/system/modulefiles:/apps/daint/UES/easybuild/modulefiles:/apps/daint/UES/reframe:/apps/common/system/modulefiles:/opt/cray/pe/perftools/20.10.0/modulefiles:/opt/cray/ari/modulefiles:/opt/cray/pe/craype-targets/default/modulefiles:/opt/cray/pe/modulefiles:/opt/cray/modulefiles:/opt/modulefiles
CRAY_MPICH_DIR=/opt/cray/pe/mpt/7.7.16/gni/mpich-crayclang/9.0
PE_PKGCONFIG_PRODUCTS_DEFAULT=PE_PAPI
SLURM_GTIDS=0
LOGNAME=pdiehl
DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/25723/bus
PE_MPICH_DEFAULT_FIXED_PRGENV=INTEL
MPICH_MAX_THREAD_SAFETY=multiple
CRAY_PMI_VERSION=5.0.17
CRAY_MPICH_VERSION=7.7.16
PE_MPICH_NV_LIBS_nvidia60=-lcudart
XDG_RUNTIME_DIR=/run/user/25723
PE_PETSC_DEFAULT_GENCOMPS_INTEL_x86_64=191
MODULE_VERSION_STACK=3.2.11.4
PE_TPSL_DEFAULT_GENCOMPS_INTEL_x86_skylake=190
EBVERSIONXALT=2.8.10
PE_PETSC_DEFAULT_GENCOMPS_INTEL_x86_skylake=191
ALLINEA_QUEUE_DLL=/opt/cray/pe/mpt/7.7.16/gni/mpich-crayclang/9.0/lib/libtvmpich.so.3.0.1
JRE_HOME=/usr/lib64/jvm/java/jre
PE_TPSL_DEFAULT_GENCOMPILERS_CRAYCLANG_x86_64=9.0
PE_LIBSCI_PKGCONFIG_VARIABLES=PE_LIBSCI_OMP_REQUIRES_@openmp@:PE_SCI_EXT_LIBPATH:PE_SCI_EXT_LIBNAME
XDG_CONFIG_DIRS=/etc/xdg
PE_MPICH_DEFAULT_VOLATILE_PRGENV=PGI GNU CRAYCLANG CRAY
PE_MPICH_TARGET_VAR_nvidia20=-lcudart
PATH=/usr/local/cuda-11.0/bin:/usr/local/cuda-11.0/libnvvp:/opt/nvidia/nsight-compute/2020.1.1/:/opt/nvidia/nsight-systems/2020.3.2//bin:/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/sbin:/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/bin:/opt/cray/pe/mpt/7.7.16/gni/bin:/opt/cray/pe/perftools/20.10.0/bin:/opt/cray/pe/papi/6.0.0.4/bin:/opt/cray/rca/2.2.20-7.0.2.1_2.93__g8e3fb5b.ari/bin:/opt/cray/alps/6.6.59-7.0.2.1_3.85__g872a8d62.ari/sbin:/opt/cray/alps/default/bin:/opt/cray/pe/craype/2.7.3/bin:/opt/cray/pe/cce/11.0.0/cce-clang/x86_64/bin:/opt/cray/pe/cce/11.0.0/binutils/x86_64/x86_64-pc-linux-gnu/bin:/opt/cray/pe/cce/11.0.0/binutils/cross/x86_64-aarch64/aarch64-linux-gnu/../bin:/opt/cray/pe/cce/11.0.0/utils/x86_64/bin:/apps/cscs/daint/bin:/apps/daint/system/bin:/apps/common/system/bin:/opt/cray/cge/3.2.1463_r03f4dfb_fe3.3.0_2019062614/bin:/opt/cray/pe/modules/3.2.11.4/bin:/users/pdiehl/bin:/usr/local/bin:/usr/bin:/bin:/usr/lib/mit/bin:/usr/lib/mit/sbin:/opt/cray/pe/bin:/users/pdiehl/.local/bin:/users/pdiehl/bin:/users/pdiehl/.local/bin:/users/pdiehl/bin:/scratch/snx3000/pdiehl/tau2/craycnl/bin
SLURM_JOB_ID=36590600
JAVA_BINDIR=/usr/lib64/jvm/java/bin
PE_TPSL_64_DEFAULT_VOLATILE_PRGENV=CRAYCLANG CRAYCLANG64 GNU GNU64 INTEL INTEL64
_LMFILES_=/opt/cray/pe/modulefiles/modules/3.2.11.4:/opt/cray/modulefiles/cge/3.2.1463_r03f4dfb_fe3.3.0_2019062614:/opt/cray/pe/craype-targets/default/modulefiles/craype-network-aries:/opt/cray/pe/modulefiles/cce/11.0.0:/opt/cray/pe/modulefiles/craype/2.7.3:/opt/cray/pe/modulefiles/cray-libsci/20.09.1:/opt/cray/ari/modulefiles/udreg/2.3.2-7.0.2.1_2.52__g8175d3d.ari:/opt/cray/ari/modulefiles/ugni/6.0.14.0-7.0.2.1_3.77__ge78e5b0.ari:/opt/cray/pe/modulefiles/pmi/5.0.17:/opt/cray/ari/modulefiles/dmapp/7.1.1-7.0.2.1_2.98__g38cf134.ari:/opt/cray/ari/modulefiles/gni-headers/5.0.12.0-7.0.2.1_2.34__g3b1768f.ari:/opt/cray/ari/modulefiles/xpmem/2.2.20-7.0.2.1_2.72__g87eb960.ari:/opt/cray/ari/modulefiles/job/2.2.4-7.0.2.1_2.86__g36b56f4.ari:/opt/cray/ari/modulefiles/dvs/2.12_2.2.177-7.0.2.1_13.5__g0b75e43d:/opt/cray/ari/modulefiles/alps/6.6.59-7.0.2.1_3.85__g872a8d62.ari:/opt/cray/ari/modulefiles/rca/2.2.20-7.0.2.1_2.93__g8e3fb5b.ari:/opt/cray/pe/modulefiles/atp/3.8.1:/opt/cray/pe/modulefiles/perftools-base/20.10.0:/opt/cray/pe/modulefiles/PrgEnv-cray/6.0.9:/opt/cray/pe/modulefiles/cray-mpich/7.7.16:/opt/modulefiles/slurm/20.11.7-7:/opt/cray/pe/craype-targets/default/modulefiles/craype-haswell:/apps/daint/UES/easybuild/modulefiles/xalt/2.8.10:/apps/daint/UES/easybuild/modulefiles/daint-gpu/20.11:/opt/cray/modulefiles/cudatoolkit/11.0.2_3.38-8.1__g5b73779:/opt/modulefiles/Base-opts/2.4.142-7.0.2.1_2.72__g8f27585.ari
PE_NETCDF_DEFAULT_FIXED_PRGENV=GNU CRAYCLANG PGI INTEL
PE_PETSC_DEFAULT_GENCOMPILERS_GNU_haswell=8.2
MODULESHOME=/opt/cray/pe/modules/3.2.11.4
PKG_CONFIG_PATH=/usr/local/cuda-11.0/11.0.2_3.38-8.1__g5b73779/lib64/pkgconfig:/opt/cray/rca/2.2.20-7.0.2.1_2.93__g8e3fb5b.ari/lib64/pkgconfig:/opt/cray/alps/6.6.59-7.0.2.1_3.85__g872a8d62.ari/lib64/pkgconfig:/opt/cray/xpmem/2.2.20-7.0.2.1_2.72__g87eb960.ari/lib64/pkgconfig:/opt/cray/gni-headers/5.0.12.0-7.0.2.1_2.34__g3b1768f.ari/lib64/pkgconfig:/opt/cray/dmapp/7.1.1-7.0.2.1_2.98__g38cf134.ari/lib64/pkgconfig:/opt/cray/pe/pmi/5.0.17/lib64/pkgconfig:/opt/cray/ugni/6.0.14.0-7.0.2.1_3.77__ge78e5b0.ari/lib64/pkgconfig:/opt/cray/udreg/2.3.2-7.0.2.1_2.52__g8175d3d.ari/lib64/pkgconfig:/opt/cray/pe/craype/2.7.3/pkg-config:/opt/cray/pe/iobuf/2.0.10/lib/pkgconfig:/opt/cray/pe/fftw/2.1.5.9/lib/pkgconfig:/opt/cray/cge/3.2.1463_r03f4dfb_fe3.3.0_2019062614/lib/pkgconfig:/opt/cray/pe/atp/3.8.1/lib/pkgconfig
LIBSCI_BASE_DIR=/opt/cray/pe/libsci/20.09.1
INFOPATH=/opt/cray/cge/3.2.1463_r03f4dfb_fe3.3.0_2019062614/info
G_BROKEN_FILENAMES=1
XALT_ETC_DIR=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/etc
HISTSIZE=1000
CRAYPE_DIR=/opt/cray/pe/craype/2.7.3
PE_GA_DEFAULT_VOLATILE_PRGENV=GNU
LD_PRELOAD=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/lib64/libxalt_init.so
CMAKE_LIBRARY_PATH=/apps/daint/UES/xalt/xalt2/software/xalt/2.8.10/lib64
PE_MPICH_GENCOMPS_CRAYCLANG=90
PE_TRILINOS_DEFAULT_GENCOMPILERS_GNU_x86_64=8.2
PE_HDF5_PARALLEL_DEFAULT_REQUIRED_PRODUCTS=PE_MPICH
OFFLOAD_INIT=on_start
PE_MPICH_DIR_CRAYCLANG_DEFAULT64=64
PE_PETSC_DEFAULT_GENCOMPILERS_GNU_x86_64=8.2
PE_PKGCONFIG_DEFAULT_PRODUCTS=PE_TRILINOS:PE_TPSL_64:PE_TPSL:PE_PETSC:PE_PARALLEL_NETCDF:PE_NETCDF_HDF5PARALLEL:PE_NETCDF:PE_MPICH:PE_LIBSCI_ACC:PE_LIBSCI:PE_HDF5_PARALLEL:PE_HDF5:PE_GA:PE_FFTW2:PE_FFTW
CPU=x86_64
SSH_SENDS_LOCALE=yes
CRAYPE_VERSION=2.7.3
INCLUDE_PATH_X86_64=/opt/cray/pe/cce/11.0.0/cce-clang/x86_64/lib/clang/11.0.0/include:/opt/cray/pe/cce/11.0.0/cce/x86_64/include/craylibs
CRAY_PMI_POST_LINK_OPTS=-L/opt/cray/pe/pmi/5.0.17/lib64
SLURM_LOCALID=0
CRAY_MPICH_PREFIX=/opt/cray/pe/mpt/7.7.16/gni/mpich-crayclang/9.0
CRAY_GNI_HEADERS_INCLUDE_OPTS=-I/opt/cray/gni-headers/5.0.12.0-7.0.2.1_2.34__g3b1768f.ari/include
CVS_RSH=ssh
GPU_DEVICE_ORDINAL=0
LESSOPEN=lessopen.sh %s
CRAYPAT_OPTS_EXECUTABLE=libexec64/opts
PE_TPSL_64_DEFAULT_GENCOMPS_INTEL_x86_64=190
PE_FFTW_DEFAULT_TARGET_x86_cascadelake=x86_cascadelake
PE_TPSL_DEFAULT_GENCOMPILERS_CRAYCLANG_haswell=9.0
CRAY_XPMEM_INCLUDE_OPTS=-I/opt/cray/xpmem/2.2.20-7.0.2.1_2.72__g87eb960.ari/include
PE_LIBSCI_DEFAULT_GENCOMPILERS_GNU_x86_64=8.1
PE_LIBSCI_GENCOMPS_GNU_x86_64=81
PE_TPSL_64_DEFAULT_GENCOMPILERS_INTEL_x86_skylake=19.0
BASH_FUNC_module%%=() {  eval `/opt/cray/pe/modules/3.2.11.4/bin/modulecmd bash $*`
}
_=/usr/bin/printenv
diehlpk commented 2 years ago

@hkaiser Would you mind checking if HPX parses the correct arguments from the environment?

stale[bot] commented 2 years ago

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

hkaiser commented 2 years ago

IIUC, this has been fixed on Daint.