From eeff79f9b495c79a41435edaf2955047ada7a34e Mon Sep 17 00:00:00 2001
From: Vicky Tsang <vicky.w.tsang@gmail.com>
Date: Thu, 12 May 2022 23:27:58 +0200
Subject: [PATCH] add readme for rocm. clean up run script

---
 README.Rocm                  | 25 +++++++++++++++++++++++++
 scripts/runHPL_singleNode.sh |  4 ++--
 2 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 README.Rocm

diff --git a/README.Rocm b/README.Rocm
new file mode 100644
index 0000000..03f9b46
--- /dev/null
+++ b/README.Rocm
@@ -0,0 +1,25 @@
+System requirements:
+ROCM 4.3 or above
+OpenMPI v4.0.3
+UCX v1.8.0
+
+
+Environment:
+export MPI_DIR=<MPI install directory>
+export BLAS_DIR=<OpenBLAS base directory>
+
+export PATH=$MPI_DIR/bin:$PATH
+export LD_LIBRARY_PATH=$MPI_DIR/lib:$BLAS_DIR/lib:$LD_LIBRARY_PATH
+
+export C_INCLUDE_PATH=$MPI_DIR/include
+export CPLUS_INCLUDE_PATH=$MPI_DIR/include
+
+Build HPL-GPU for ROCm
+
+$ git clone https://github.com/reger-men/HPL_GPU.git
+$ cd HPL-GPU
+$ mkdir build && cd build
+$ cmake ..
+$ make -j
+
+
diff --git a/scripts/runHPL_singleNode.sh b/scripts/runHPL_singleNode.sh
index 27482ba..e82eeec 100644
--- a/scripts/runHPL_singleNode.sh
+++ b/scripts/runHPL_singleNode.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-MPI_DIR=$HOME/ompi/bin
+MPI_DIR=$MPI_HOME/bin
 num_cpu_cores=16
 num_process=1
 
@@ -9,5 +9,5 @@ export OMP_NUM_THREADS=${num_cpu_cores}
 export LD_LIBRARY_PATH=openblas:$LD_LIBRARY_PATH
 
 # ./xhpl
-HSA_ENABLE_SDMA=1 ${MPI_DIR}/mpirun  -mca btl '^openib' --mca pml ucx -x UCX_RNDV_PIPELINE_SEND_THRESH=256k -x UCX_RNDV_FRAG_SIZE=rocm:4m --allow-run-as-root -np ${num_process} --map-by node:PE=${num_cpu_cores} --bind-to core:overload-allowed --report-bindings ./xhplhip
+HSA_ENABLE_SDMA=1 ${MPI_DIR}/mpirun --allow-run-as-root -np ${num_process} --map-by node:PE=${num_cpu_cores} --bind-to core:overload-allowed --report-bindings ./xhplhip
 grep --color "e+" HPL.out