update for LUMI environment

scripts/env.crusher.sh

@@ -0,0 +1,20 @@
+# modules
+module reset
+
+module load craype-accel-amd-gfx90a
+module load PrgEnv-amd
+module load amd/5.2.0
+module load rocm/5.2.0
+module load cray-mpich/8.1.17
+module load openblas/0.3.17-omp
+
+#
+# env
+#
+export LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}"
+# enable GPU aware MPI
+export MPICH_GPU_SUPPORT_ENABLED=1
+# to work around the OFI registration cache issue for > 8 nodes
+export FI_MR_CACHE_MAX_COUNT=0
+#export MPICH_SMP_SINGLE_COPY_MODE=NONE # does not work
+export MPICH_RANK_REORDER_DISPLAY=1

scripts/env.lumi.sh

@@ -0,0 +1,21 @@
+# Any other commands must follow the #SBATCH directives
+module load LUMI/22.06 partition/G
+#module load rocm/5.1.4
+
+module use /project/project_462000075/paklui/modulefiles
+module load rocm/5.3.0-10584
+#module load rocm/5.3.0-10619
+#module load openblas/0.3.17-omp
+#module load cce/14.0.2
+#module load cray-libsci/22.08.1.1
+#module load cray-mpich/8.1.18
+#module load craype/2.7.17
+
+#
+# env
+#
+# enable GPU aware MPI
+export MPICH_GPU_SUPPORT_ENABLED=1
+# to work around the OFI registration cache issue for > 8 nodes
+export FI_MR_CACHE_MAX_COUNT=0
+export MPICH_RANK_REORDER_DISPLAY=1

scripts/run_hpl.slurm

@@ -10,7 +10,8 @@
 #SBATCH --ntasks-per-node=8
 #SBATCH --gpus-per-node=8
 
-source ../env/env.crusher.sh
+#source ../env/env.crusher.sh
+source ../env/env.lumi.sh
 
 NP=$SLURM_NPROCS
 NODES=$SLURM_NNODES
@@ -34,10 +35,6 @@ CMD+="-o $LOG -e $LOG "
 #CMD+="${HOME}/mpich_bind.sh "
 CMD+="$EXE"
 
-#export MPICH_SMP_SINGLE_COPY_MODE=NONE # does not work
-export FI_MR_CACHE_MAX_COUNT=0
-export MPICH_RANK_REORDER_DISPLAY=1
-
 echo $CMD >> $LOG
 echo $CMD 2>&1 | tee -a $LOG
      $CMD 2>&1 | tee -a $LOG