adds github action kernel tests (with GPU HIP test)

danielpeter · Nov 19, 2024 · 54a6643 · 54a6643
1 parent 247a673
commit 54a6643
Show file tree

Hide file tree

Showing 4 changed files with 232 additions and 3 deletions.
diff --git a/.github/scripts/run_build.sh b/.github/scripts/run_build.sh
@@ -70,6 +70,16 @@ else
   hdf=()
 fi
 
+## HIP
+if [ "${HIP}" == "true" ]; then
+  echo
+  echo "enabling HIP"
+  echo
+  hip=(--with-hip HIPCC=g++ HIP_FLAGS="-O2 -g -std=c++17" HIP_PLATFORM=cpu HIP_INC=./external_libs/ROCm-HIP-CPU/include HIP_LIBS="-ltbb -lpthread -lstdc++")
+else
+  hip=()
+fi
+
 ## special testflags
 if [ "${TESTFLAGS}" == "check-mcmodel-medium" ]; then
   # note: this is a work-around as using the 'env:' parameter in the workflow 'CI.yml' with TESTFLAGS: FLAGS_CHECK=".."
@@ -94,6 +104,7 @@ set -- ${TESTFLAGS}
 "${adios[@]}" \
 "${netcdf[@]}" \
 "${hdf[@]}" \
+"${hip[@]}" \
 "${petsc[@]}" \
 "${flags[@]}" \
 FC=gfortran MPIFC=mpif90 CC=gcc "$@"
@@ -112,10 +123,12 @@ sed -i "s:IMAIN .*:IMAIN = ISTANDARD_OUTPUT:" setup/constants.h
 # compilation
 echo
 echo "clean:"
+echo
 make clean
 
 echo
 echo "compilation:"
+echo
 make -j4 all
 
 # checks

diff --git a/.github/scripts/run_install.sh b/.github/scripts/run_install.sh
@@ -65,6 +65,18 @@ if [ "${PETSC}" == "true" ]; then
   echo; echo "done PETSc"; echo
 fi
 
+## HIP
+if [ "${HIP}" == "true" ]; then
+  echo
+  echo "HIP additionals installation:"
+  echo
+  sudo apt-get install -yq --no-install-recommends libtbb-dev
+fi
+
+# checks exit code
+if [[ $? -ne 0 ]]; then exit 1; fi
+echo
+
 # python3 pip upgrade might complain: "ERROR: launchpadlib 1.10.13 requires testresources"
 sudo apt-get install -yq --no-install-recommends python3-testresources
 # checks exit code

diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh
@@ -22,14 +22,85 @@ echo
 
 # bash function for checking seismogram output with reference solutions
 my_test(){
-  echo "testing seismograms:"
+  echo "######################################################################################################################"
+  echo "testing seismograms"
   ln -s $WORKDIR/utils/scripts/compare_seismogram_correlations.py
   ./compare_seismogram_correlations.py REF_SEIS/ OUTPUT_FILES/
   if [[ $? -ne 0 ]]; then exit 1; fi
   ./compare_seismogram_correlations.py REF_SEIS/ OUTPUT_FILES/ | grep min/max | cut -d \| -f 3 | awk '{print "correlation:",$1; if ($1 < 0.999 ){print $1,"failed"; exit 1;}else{ print $1,"good"; exit 0;}}'
   if [[ $? -ne 0 ]]; then exit 1; fi
+  echo "######################################################################################################################"
 }
 
+my_kernel_test(){
+  # kernel value test - checks rho/kappa/mu kernel value outputs
+  echo "######################################################################################################################"
+  echo "testing kernel values"
+  file_ref=REF_KERNEL/output_solver.txt
+  file_out=output.log        # captures the OUTPUT_FILES/output_solver.txt when running solver since IMAIN was set to standard out
+  if [ ! -e $file_ref ]; then echo "Please check if file $file_ref exists..."; ls -alR ./; exit 1; fi
+  if [ ! -e $file_out ]; then echo "Please check if file $file_out exists..."; ls -alR ./; exit 1; fi
+  # gets reference expected kernel values from REF_KERNEL/ folder
+  RHO=`grep -E 'maximum value of rho[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '`
+  KAPPA=`grep -E 'maximum value of kappa[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '`
+  MU=`grep -E 'maximum value of mu[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '`
+  KAPPAV=`grep -E 'maximum value of kappav[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '`
+  MUV=`grep -E 'maximum value of muv[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '`
+
+  # need at least rho & kappa (for acoustic kernels)
+  if [ "$RHO" == "" ]; then
+    echo "  missing reference kernel values: RHO=$RHO | KAPPA=$KAPPA KAPPAV=$KAPPAV | MU=$MU MUV=$MUV"
+    echo
+    exit 1
+  else
+    echo "  reference kernel values: RHO=$RHO | KAPPA=$KAPPA KAPPAV=$KAPPAV | MU=$MU MUV=$MUV"
+  fi
+  # compares with test output - using a relative tolerance of 0.001 (1 promille) with respect to expected value
+  # final test result
+  PASSED=0
+  # checks rho kernel value
+  if [ "$RHO" != "" ]; then
+    VAL=`grep -E 'maximum value of rho[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '`
+    echo "kernel rho   : $VAL"
+    echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print "  value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print "  failed"; exit 1;}else{print "  good"; exit 0;} }' ex=$RHO val=$VAL
+    if [[ $? -ne 0 ]]; then PASSED=1; fi
+  fi
+  # checks kappa kernel value
+  if [ "$KAPPA" != "" ]; then
+    VAL=`grep -E 'maximum value of kappa[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '`
+    echo "kernel kappa : $VAL"
+    echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print "  value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print "  failed"; exit 1;}else{print "  good"; exit 0;} }' ex=$KAPPA val=$VAL
+    if [[ $? -ne 0 ]]; then PASSED=1; fi
+  fi
+  # checks kappav kernel value (if anisotropic kernels)
+  if [ "$KAPPAV" != "" ]; then
+    VAL=`grep -E 'maximum value of kappav[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '`
+    echo "kernel kappav: $VAL"
+    echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print "  value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print "  failed"; exit 1;}else{print "  good"; exit 0;} }' ex=$KAPPAV val=$VAL
+    if [[ $? -ne 0 ]]; then PASSED=1; fi
+  fi
+  # checks mu kernel value
+  if [ "$MU" != "" ]; then
+    VAL=`grep -E 'maximum value of mu[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '`
+    echo "kernel mu    : $VAL"
+    echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print "  value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print "  failed"; exit 1;}else{print "  good"; exit 0;} }' ex=$MU val=$VAL
+    if [[ $? -ne 0 ]]; then PASSED=1; fi
+  fi
+  # checks muv kernel value (if anisotropic kernels)
+  if [ "$MUV" != "" ]; then
+    VAL=`grep -E 'maximum value of muv[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '`
+    echo "kernel muv   : $VAL"
+    echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print "  value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print "  failed"; exit 1;}else{print "  good"; exit 0;} }' ex=$MUV val=$VAL
+    if [[ $? -ne 0 ]]; then PASSED=1; fi
+  fi
+  # overall pass
+  if [[ $PASSED -ne 0 ]]; then
+    echo "testing kernel values: failed"; exit 1;
+  else
+    echo "testing kernel values: all good"
+  fi
+  echo "######################################################################################################################"
+}
 # test example
 cd $dir
 
@@ -68,7 +139,7 @@ fi
 # hdf5 i/o example
 if [ "${HDF5}" == "true" ]; then
   echo
-  echo "HDF5 enabled test run"
+  echo "test run w/ HDF5"
   echo
   sed -i "s:^HDF5_ENABLED .*:HDF5_ENABLED    = .true.:" DATA/Par_file
   #sed -i "s:^HDF5_FOR_MOVIES .*:HDF5_FOR_MOVIES    = .true.:" DATA/Par_file
@@ -80,13 +151,24 @@ fi
 # adios
 if [ "${ADIOS2}" == "true" ]; then
   # turns on ADIOS
+  echo "turning on ADIOS"
   sed -i "s:^ADIOS_ENABLED .*:ADIOS_ENABLED = .true.:" DATA/Par_file
 fi
 
+## GPU
+if [ "${GPU}" == "true" ]; then
+  # turns on GPU
+  echo "turning on GPU"
+  sed -i "s:^GPU_MODE .*:GPU_MODE = .true.:" DATA/Par_file
+fi
+
+# save Par_file state
+cp -v DATA/Par_file DATA/Par_file.bak
+
 # use kernel script
 if [ "${RUN_KERNEL}" == "true" ]; then
   # use kernel script
-  ./run_this_example.kernel.sh
+  ./run_this_example_kernel.sh | tee output.log
 else
   # default script
   ./run_this_example.sh
@@ -107,9 +189,48 @@ if [ "${DEBUG}" == "true" ] || [ "${FULL_GRAVITY}" == "true" ] || [ "${RUN_KERNE
 else
   my_test
 fi
+# checks exit code
+if [[ $? -ne 0 ]]; then exit 1; fi
+
+# kernel test
+if [ "${RUN_KERNEL}" == "true" ]; then
+  # check kernel values
+  my_kernel_test
+  # checks exit code
+  if [[ $? -ne 0 ]]; then exit 1; fi
+  # clean up
+  rm -rf OUTPUT_FILES/ SEM/ output.log
+
+  # re-run kernel test w/ UNDO_ATT
+  UNDO_ATT=`grep ^UNDO_ATTENUATION DATA/Par_file | cut -d = -f 2 | tr -d ' '`
+  if [[ ${UNDO_ATT} == *"false"* ]]; then
+    echo
+    echo "*****************************************"
+    echo "run kernel w/ UNDO_ATTENUATION"
+    echo "*****************************************"
+    echo
+
+    # turns on UNDO_ATTENUATION
+    echo "turning on UNDO_ATTENUATION"
+    sed -i "s:^UNDO_ATTENUATION .*:UNDO_ATTENUATION = .true.:" DATA/Par_file
+
+    # use kernel script
+    ./run_this_example_kernel.sh | tee output.log
+    # checks exit code
+    if [[ $? -ne 0 ]]; then exit 1; fi
+    # kernel test
+    my_kernel_test
+    # checks exit code
+    if [[ $? -ne 0 ]]; then exit 1; fi
+  fi
+fi
+
+# restore original Par_file
+cp -v DATA/Par_file.bak DATA/Par_file
 
 # cleanup
 rm -rf OUTPUT_FILES* DATABASES_MPI*
+if [ -e SEM ]; then rm -rf SEM/; fi
 
 echo
 echo "all good"

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -575,3 +575,86 @@ jobs:
         HDF5: true
       run: ./.github/scripts/run_tests.sh
       shell: bash
+
+  linuxTest_12:
+    name: Test 12 - regional_EMC_model netCDF
+    runs-on: ubuntu-latest
+    needs: [linuxCheck]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install packages
+      env:
+        NETCDF: true
+      run: ./.github/scripts/run_install.sh
+      shell: bash
+
+    - name: Run build
+      env:
+        NETCDF: true
+      run: ./.github/scripts/run_build.sh
+      shell: bash
+
+    - name: Run test
+      env:
+        TESTDIR: EXAMPLES/regional_EMC_model
+      run: ./.github/scripts/run_tests.sh
+      shell: bash
+
+  linuxTest_13:
+    name: Test 13 - regional_EMC_model kernel netCDF
+    runs-on: ubuntu-latest
+    needs: [linuxCheck]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install packages
+      env:
+        NETCDF: true
+      run: ./.github/scripts/run_install.sh
+      shell: bash
+
+    - name: Run build
+      env:
+        NETCDF: true
+      run: ./.github/scripts/run_build.sh
+      shell: bash
+
+    - name: Run test kernel
+      env:
+        TESTDIR: EXAMPLES/regional_EMC_model
+        RUN_KERNEL: true
+      run: ./.github/scripts/run_tests.sh
+      shell: bash
+
+  linuxTest_14:
+    name: Test 14 - regional_EMC_model kernel netCDF GPU HIP
+    runs-on: ubuntu-latest
+    needs: [linuxCheck]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install packages
+      env:
+        NETCDF: true
+        HIP: true
+      run: ./.github/scripts/run_install.sh
+      shell: bash
+
+    - name: Run build
+      env:
+        NETCDF: true
+        HIP: true
+      run: ./.github/scripts/run_build.sh
+      shell: bash
+
+    - name: Run test kernel
+      env:
+        TESTDIR: EXAMPLES/regional_EMC_model
+        RUN_KERNEL: true
+        GPU: true
+      run: ./.github/scripts/run_tests.sh
+      shell: bash