Install GITRm on the DIII-D omega cluster

jguterl commented 1 month ago

documentation : https://github.com/jguterl/pyGITRM/wiki/Install-gitrm-on-GA_Omega-cluster

Add that documentation when working and fixed to the main git wiki https://github.com/DIII-D/GITRmProject Ideally, create a bash script and add it to the git repo

jguterl commented 1 month ago

#!/bin/bash
module purge
#git clone --branch env-gcc-11 git@github.com:GAmfe/css-omega-modules.git 
#module use ${PWD}/css-omega-modules
module load env/gcc8.x

# INSTALL KOKKOS
#KOKKOS -> 3.7.01 (git checkout #VERSION)

# Insert your GITRm installation folder

export ROOT=/home/user/GITRm_install_folder
export BUILD=build_OmegaGA
export KK=$ROOT/${BUILD}-kokkos/install
export PP=$ROOT/${BUILD}-pumipic/install
export ENGPAR=$ROOT/${BUILD}-engpar/install
export OH=$ROOT/${BUILD}-omegah/install
export H5=$ROOT/${BUILD}-hdf5
export CAB=$ROOT/${BUILD}-cabana/install
export H5_INCLUDE_=${H5}/include/hdf5.h
export OMPI_CXX=$ROOT/kokkos/bin/nvcc_wrapper
export GITRM=$ROOT/${BUILD}-gitrm/install
export OMPI_CC=gcc
export CMAKE_PREFIX_PATH=$PP:$KK:$OH:$CAB:$ENGPAR:$PP:$OMPI_CXX:$CMAKE_PREFIX_PATH
export MPICC=/fusion/usc/c8/opt/env/gcc-8.3.1/bin/mpicc
cd $ROOT
if [ ! -d $H5 ]
then
mkdir -p $H5
wget https://support.hdfgroup.org/ftp/HDF5/current/src/CMake-hdf5-1.10.5.tar.gz
tar -xvzf CMake-hdf5-1.10.5.tar.gz
cd CMake-hdf5-1.10.5/hdf5-1.10.5/
rm -r CMakeCache.txt CMakeFiles/
./configure --enable-cxx --prefix=$H5
#cd hdf5-1.4.4
#./configure --prefix=$H5 --enable-cxx --share
make -j 24 install
echo "\n---------- HDF5 install completed -------------\n"
cd $ROOT

fi 
module load nvidia_hpc_sdk/20.11
mkdir -p $ROOT
cd $ROOT

if [ ! -d $KK ]
then
mkdir -p $KK
git clone https://github.com/kokkos/kokkos.git
cd kokkos
# git checkout 3.7.01
git checkout 3.7.01
cd ..

cd ${KK%%install}
cmake ../kokkos -DCMAKE_CXX_COMPILER=$ROOT/kokkos/bin/nvcc_wrapper -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=off -DKokkos_ENABLE_CUDA=on -DKokkos_ENABLE_CUDA_LAMBDA=on -DKokkos_ENABLE_DEBUG=on-DKokkos_ENABLE_PROFILING=on -DCMAKE_INSTALL_PREFIX=$KK
make -j 8 install
cd ..
fi 
echo "\n---------- kokkos install completed -------------\n"
# omega_h
if [ ! -d $OH ]
then
git clone https://github.com/SCOREC/omega_h.git
cd omega_h
git checkout e1be29b09848166ed17899de88af7851b94f7ba5
[ -d $OH ] && rm -rf ${OH%%install}
mkdir -p $OH
cd ${OH%%install}
cmake ../omega_h \
  -DCMAKE_INSTALL_PREFIX=$OH \
  -DBUILD_SHARED_LIBS=OFF \
  -DOmega_h_USE_Kokkos=ON \
  -DOmega_h_USE_CUDA=on \
  -DOmega_h_CUDA_ARCH=70 \
  -DOmega_h_USE_MPI=on  \
  -DBUILD_TESTING=on  \
  -DCMAKE_CXX_COMPILER=$ROOT/kokkos/bin/nvcc_wrapper \
  -DMPIEXEC_EXECUTABLE=/fusion/usc/c8/opt/env/gcc-8.3.1/bin/mpiexec \
  -DKokkos_PREFIX=$KK/lib64/cmake
make VERBOSE=1 -j8 install
cd ..
fi

# engpar
if [ ! -d $ENGPAR ]
then
git clone https://github.com/SCOREC/EnGPar.git engpar
[ -d $ENGPAR ] && rm -rf ${ENGPAR%%install}
mkdir -p $ENGPAR
cd ${ENGPAR%%install}
cmake ../engpar \
  -DCMAKE_INSTALL_PREFIX=$ENGPAR \
  -DCMAKE_C_COMPILER="mpicc" \
  -DCMAKE_CXX_COMPILER="mpicxx" \
  -DCMAKE_CXX_FLAGS="-std=c++11" \
  -DENABLE_PARMETIS=OFF \
  -DENABLE_PUMI=OFF \
  -DIS_TESTING=OFF
make install -j8 
cd ..
fi
echo "\n---------- ENGPAR install completed -------------\n"

if [ ! -d $CAB ]
then
git clone https://github.com/ECP-copa/Cabana.git cabana
cd cabana
git checkout 6d490ad170283cffac9cd8543a24d1dbfe889ba1
cd ..
mkdir -p $CAB
cd ${CAB%%install}
cmake -S $ROOT/cabana -B ${CAB%%install}\
  -DCMAKE_BUILD_TYPE=RelWithDebInfo \
  -DCMAKE_CXX_COMPILER=$ROOT/kokkos/bin/nvcc_wrapper \
  -DCMAKE_PREFIX_PATH=$KK \
  -DCMAKE_INSTALL_PREFIX=$CAB
make -j 24 install
cd ..
fi
echo "\n---------- Cabana install completed -------------\n"
cd $ROOT

if [ ! -d $PP ]
then

git clone https://github.com/SCOREC/pumi-pic.git
cd pumi-pic
git checkout ac/dps
git checkout d6a53c534f416b9e3f37643fecb95855adbb789a
sed -i -e 's!git@github.com:!https://github.com/!g' .gitmodules #use ssh
git submodule init 
git submodule update # download test data
cd $ROOT

mkdir -p $PP
cd ${PP%%install}
cmake ../pumi-pic \
  -DCMAKE_CXX_COMPILER=mpicxx  \
  -DIS_TESTING=ON  \
  -DPS_IS_TESTING=ON  \
  -DTEST_DATA_DIR=$ROOT/pumi-pic/pumipic-data  \
  -DOmega_h_PREFIX=$OH  \
  -DKokkos_PREFIX=$KK  \
  -DEnGPar_PREFIX=$ENGPAR  \
  -DCabana_INSTALL_DIR=$CAB \
  -DCabana_LIBRARY=$CAB/lib64 \
  -DENABLE_CABANA=on \
  -DCMAKE_INSTALL_PREFIX=$PP

make -j8 install
cd ..
fi

cd $ROOT

git clone git@github.com:DIII-D/gitrm.git
cd gitrm
git pull
sed -i "s,<hdf5.h>,\"${H5_INCLUDE_}\",g" src/GitrmInputOutput.hpp
sed -i "s,<hdf5.h>,\"${H5_INCLUDE_}\",g" src/GitrmSpectroscopy.cpp
cd ..
cd $ROOT
mkdir -p $GITRM
cd ${GITRM%%install}
cmake -S $ROOT/gitrm -B${GITRM%%install} \
 -DCMAKE_CXX_COMPILER=$ROOT/kokkos/bin/nvcc_wrapper \
 -DCMAKE_INSTALL_PREFIX=$GITRM \
 -DGITRm_USE_NETCDF=0 \
 -DGITRm_USE_HDF5=1 \
 -DHDF5_INCLUDE_DIRS=$H5/include \
 -DHDF5_ROOT=$H5 \
 -DHDF5_PREFIX=$H5 \
 -DHDF5_PATH=$H5/lib \
 -DHDF5_DIR=$H5 \
 -DHDF5_ROOT=$H5
make -j8
cd $ROOT

LC-GA commented 1 month ago

The working version of the Bash script for GITRm installation is now both in @jguterl 's previous comment and in https://github.com/jguterl/pyGITRM/wiki/Install-gitrm-on-GA_Omega-cluster.

LC-GA commented 1 month ago

Do you know how to set milestones?

Install GITRm
create wiki
...

DIII-D / GITRmProject

Install GITRm on the DIII-D omega cluster #2