From 54a6643d59ebbfd6ea446275b06a5a2b313c2dcd Mon Sep 17 00:00:00 2001 From: Daniel Peter Date: Tue, 19 Nov 2024 10:59:24 +0100 Subject: [PATCH] adds github action kernel tests (with GPU HIP test) --- .github/scripts/run_build.sh | 13 ++++ .github/scripts/run_install.sh | 12 ++++ .github/scripts/run_tests.sh | 127 ++++++++++++++++++++++++++++++++- .github/workflows/CI.yml | 83 +++++++++++++++++++++ 4 files changed, 232 insertions(+), 3 deletions(-) diff --git a/.github/scripts/run_build.sh b/.github/scripts/run_build.sh index ebb922b2f..9da46e565 100755 --- a/.github/scripts/run_build.sh +++ b/.github/scripts/run_build.sh @@ -70,6 +70,16 @@ else hdf=() fi +## HIP +if [ "${HIP}" == "true" ]; then + echo + echo "enabling HIP" + echo + hip=(--with-hip HIPCC=g++ HIP_FLAGS="-O2 -g -std=c++17" HIP_PLATFORM=cpu HIP_INC=./external_libs/ROCm-HIP-CPU/include HIP_LIBS="-ltbb -lpthread -lstdc++") +else + hip=() +fi + ## special testflags if [ "${TESTFLAGS}" == "check-mcmodel-medium" ]; then # note: this is a work-around as using the 'env:' parameter in the workflow 'CI.yml' with TESTFLAGS: FLAGS_CHECK=".." @@ -94,6 +104,7 @@ set -- ${TESTFLAGS} "${adios[@]}" \ "${netcdf[@]}" \ "${hdf[@]}" \ +"${hip[@]}" \ "${petsc[@]}" \ "${flags[@]}" \ FC=gfortran MPIFC=mpif90 CC=gcc "$@" @@ -112,10 +123,12 @@ sed -i "s:IMAIN .*:IMAIN = ISTANDARD_OUTPUT:" setup/constants.h # compilation echo echo "clean:" +echo make clean echo echo "compilation:" +echo make -j4 all # checks diff --git a/.github/scripts/run_install.sh b/.github/scripts/run_install.sh index 7614c9d7d..f24ffe996 100755 --- a/.github/scripts/run_install.sh +++ b/.github/scripts/run_install.sh @@ -65,6 +65,18 @@ if [ "${PETSC}" == "true" ]; then echo; echo "done PETSc"; echo fi +## HIP +if [ "${HIP}" == "true" ]; then + echo + echo "HIP additionals installation:" + echo + sudo apt-get install -yq --no-install-recommends libtbb-dev +fi + +# checks exit code +if [[ $? -ne 0 ]]; then exit 1; fi +echo + # python3 pip upgrade might complain: "ERROR: launchpadlib 1.10.13 requires testresources" sudo apt-get install -yq --no-install-recommends python3-testresources # checks exit code diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh index b20501549..ed53be74d 100755 --- a/.github/scripts/run_tests.sh +++ b/.github/scripts/run_tests.sh @@ -22,14 +22,85 @@ echo # bash function for checking seismogram output with reference solutions my_test(){ - echo "testing seismograms:" + echo "######################################################################################################################" + echo "testing seismograms" ln -s $WORKDIR/utils/scripts/compare_seismogram_correlations.py ./compare_seismogram_correlations.py REF_SEIS/ OUTPUT_FILES/ if [[ $? -ne 0 ]]; then exit 1; fi ./compare_seismogram_correlations.py REF_SEIS/ OUTPUT_FILES/ | grep min/max | cut -d \| -f 3 | awk '{print "correlation:",$1; if ($1 < 0.999 ){print $1,"failed"; exit 1;}else{ print $1,"good"; exit 0;}}' if [[ $? -ne 0 ]]; then exit 1; fi + echo "######################################################################################################################" } +my_kernel_test(){ + # kernel value test - checks rho/kappa/mu kernel value outputs + echo "######################################################################################################################" + echo "testing kernel values" + file_ref=REF_KERNEL/output_solver.txt + file_out=output.log # captures the OUTPUT_FILES/output_solver.txt when running solver since IMAIN was set to standard out + if [ ! -e $file_ref ]; then echo "Please check if file $file_ref exists..."; ls -alR ./; exit 1; fi + if [ ! -e $file_out ]; then echo "Please check if file $file_out exists..."; ls -alR ./; exit 1; fi + # gets reference expected kernel values from REF_KERNEL/ folder + RHO=`grep -E 'maximum value of rho[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '` + KAPPA=`grep -E 'maximum value of kappa[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '` + MU=`grep -E 'maximum value of mu[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '` + KAPPAV=`grep -E 'maximum value of kappav[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '` + MUV=`grep -E 'maximum value of muv[[:space:]]+kernel' $file_ref | cut -d = -f 2 | tr -d ' '` + + # need at least rho & kappa (for acoustic kernels) + if [ "$RHO" == "" ]; then + echo " missing reference kernel values: RHO=$RHO | KAPPA=$KAPPA KAPPAV=$KAPPAV | MU=$MU MUV=$MUV" + echo + exit 1 + else + echo " reference kernel values: RHO=$RHO | KAPPA=$KAPPA KAPPAV=$KAPPAV | MU=$MU MUV=$MUV" + fi + # compares with test output - using a relative tolerance of 0.001 (1 promille) with respect to expected value + # final test result + PASSED=0 + # checks rho kernel value + if [ "$RHO" != "" ]; then + VAL=`grep -E 'maximum value of rho[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '` + echo "kernel rho : $VAL" + echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print " value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print " failed"; exit 1;}else{print " good"; exit 0;} }' ex=$RHO val=$VAL + if [[ $? -ne 0 ]]; then PASSED=1; fi + fi + # checks kappa kernel value + if [ "$KAPPA" != "" ]; then + VAL=`grep -E 'maximum value of kappa[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '` + echo "kernel kappa : $VAL" + echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print " value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print " failed"; exit 1;}else{print " good"; exit 0;} }' ex=$KAPPA val=$VAL + if [[ $? -ne 0 ]]; then PASSED=1; fi + fi + # checks kappav kernel value (if anisotropic kernels) + if [ "$KAPPAV" != "" ]; then + VAL=`grep -E 'maximum value of kappav[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '` + echo "kernel kappav: $VAL" + echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print " value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print " failed"; exit 1;}else{print " good"; exit 0;} }' ex=$KAPPAV val=$VAL + if [[ $? -ne 0 ]]; then PASSED=1; fi + fi + # checks mu kernel value + if [ "$MU" != "" ]; then + VAL=`grep -E 'maximum value of mu[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '` + echo "kernel mu : $VAL" + echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print " value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print " failed"; exit 1;}else{print " good"; exit 0;} }' ex=$MU val=$VAL + if [[ $? -ne 0 ]]; then PASSED=1; fi + fi + # checks muv kernel value (if anisotropic kernels) + if [ "$MUV" != "" ]; then + VAL=`grep -E 'maximum value of muv[[:space:]]+kernel' $file_out | cut -d = -f 2 | tr -d ' '` + echo "kernel muv : $VAL" + echo "" | awk '{diff=ex-val;diff_abs=(diff >= 0)? diff:-diff;diff_rel=diff_abs/ex;print " value: expected = "ex" gotten = "val" - difference absolute = "diff_abs" relative = "diff_rel; if (diff_rel>0.001){print " failed"; exit 1;}else{print " good"; exit 0;} }' ex=$MUV val=$VAL + if [[ $? -ne 0 ]]; then PASSED=1; fi + fi + # overall pass + if [[ $PASSED -ne 0 ]]; then + echo "testing kernel values: failed"; exit 1; + else + echo "testing kernel values: all good" + fi + echo "######################################################################################################################" +} # test example cd $dir @@ -68,7 +139,7 @@ fi # hdf5 i/o example if [ "${HDF5}" == "true" ]; then echo - echo "HDF5 enabled test run" + echo "test run w/ HDF5" echo sed -i "s:^HDF5_ENABLED .*:HDF5_ENABLED = .true.:" DATA/Par_file #sed -i "s:^HDF5_FOR_MOVIES .*:HDF5_FOR_MOVIES = .true.:" DATA/Par_file @@ -80,13 +151,24 @@ fi # adios if [ "${ADIOS2}" == "true" ]; then # turns on ADIOS + echo "turning on ADIOS" sed -i "s:^ADIOS_ENABLED .*:ADIOS_ENABLED = .true.:" DATA/Par_file fi +## GPU +if [ "${GPU}" == "true" ]; then + # turns on GPU + echo "turning on GPU" + sed -i "s:^GPU_MODE .*:GPU_MODE = .true.:" DATA/Par_file +fi + +# save Par_file state +cp -v DATA/Par_file DATA/Par_file.bak + # use kernel script if [ "${RUN_KERNEL}" == "true" ]; then # use kernel script - ./run_this_example.kernel.sh + ./run_this_example_kernel.sh | tee output.log else # default script ./run_this_example.sh @@ -107,9 +189,48 @@ if [ "${DEBUG}" == "true" ] || [ "${FULL_GRAVITY}" == "true" ] || [ "${RUN_KERNE else my_test fi +# checks exit code +if [[ $? -ne 0 ]]; then exit 1; fi + +# kernel test +if [ "${RUN_KERNEL}" == "true" ]; then + # check kernel values + my_kernel_test + # checks exit code + if [[ $? -ne 0 ]]; then exit 1; fi + # clean up + rm -rf OUTPUT_FILES/ SEM/ output.log + + # re-run kernel test w/ UNDO_ATT + UNDO_ATT=`grep ^UNDO_ATTENUATION DATA/Par_file | cut -d = -f 2 | tr -d ' '` + if [[ ${UNDO_ATT} == *"false"* ]]; then + echo + echo "*****************************************" + echo "run kernel w/ UNDO_ATTENUATION" + echo "*****************************************" + echo + + # turns on UNDO_ATTENUATION + echo "turning on UNDO_ATTENUATION" + sed -i "s:^UNDO_ATTENUATION .*:UNDO_ATTENUATION = .true.:" DATA/Par_file + + # use kernel script + ./run_this_example_kernel.sh | tee output.log + # checks exit code + if [[ $? -ne 0 ]]; then exit 1; fi + # kernel test + my_kernel_test + # checks exit code + if [[ $? -ne 0 ]]; then exit 1; fi + fi +fi + +# restore original Par_file +cp -v DATA/Par_file.bak DATA/Par_file # cleanup rm -rf OUTPUT_FILES* DATABASES_MPI* +if [ -e SEM ]; then rm -rf SEM/; fi echo echo "all good" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0234d4042..5c06ebd1e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -575,3 +575,86 @@ jobs: HDF5: true run: ./.github/scripts/run_tests.sh shell: bash + + linuxTest_12: + name: Test 12 - regional_EMC_model netCDF + runs-on: ubuntu-latest + needs: [linuxCheck] + + steps: + - uses: actions/checkout@v4 + + - name: Install packages + env: + NETCDF: true + run: ./.github/scripts/run_install.sh + shell: bash + + - name: Run build + env: + NETCDF: true + run: ./.github/scripts/run_build.sh + shell: bash + + - name: Run test + env: + TESTDIR: EXAMPLES/regional_EMC_model + run: ./.github/scripts/run_tests.sh + shell: bash + + linuxTest_13: + name: Test 13 - regional_EMC_model kernel netCDF + runs-on: ubuntu-latest + needs: [linuxCheck] + + steps: + - uses: actions/checkout@v4 + + - name: Install packages + env: + NETCDF: true + run: ./.github/scripts/run_install.sh + shell: bash + + - name: Run build + env: + NETCDF: true + run: ./.github/scripts/run_build.sh + shell: bash + + - name: Run test kernel + env: + TESTDIR: EXAMPLES/regional_EMC_model + RUN_KERNEL: true + run: ./.github/scripts/run_tests.sh + shell: bash + + linuxTest_14: + name: Test 14 - regional_EMC_model kernel netCDF GPU HIP + runs-on: ubuntu-latest + needs: [linuxCheck] + + steps: + - uses: actions/checkout@v4 + + - name: Install packages + env: + NETCDF: true + HIP: true + run: ./.github/scripts/run_install.sh + shell: bash + + - name: Run build + env: + NETCDF: true + HIP: true + run: ./.github/scripts/run_build.sh + shell: bash + + - name: Run test kernel + env: + TESTDIR: EXAMPLES/regional_EMC_model + RUN_KERNEL: true + GPU: true + run: ./.github/scripts/run_tests.sh + shell: bash