hip.yml

name: ERF CI (hip)

on:
  push:
    # branches: [development]
    paths-ignore:
      - Docs
      - README.rst
      - license.txt

  pull_request:
    branches: [development]

concurrency:
  group: ${{ github.ref }}-${{ github.head_ref }}-hip
  cancel-in-progress: true

jobs:
  # MPI_C is broken since HIP 4.1
  # https://gitlab.kitware.com/cmake/cmake/-/issues/21968
  # https://github.com/ROCm-Developer-Tools/HIP/issues/2246
  Build-And-Test-HIP:
    name: HIP ROCm GFortran@9.3 C++17 [tests]
    runs-on: ubuntu-20.04
    # Have to have -Wno-deprecated-declarations due to deprecated atomicAddNoRet
    # Have to have -Wno-gnu-zero-variadic-macro-arguments to avoid
    #    amrex/Src/Base/AMReX_GpuLaunchGlobal.H:15:5: error: must specify at least one argument for '...' parameter of variadic macro [-Werror,-Wgnu-zero-variadic-macro-arguments]
    #        __launch_bounds__(amrex_launch_bounds_max_threads)
    #        ^
    #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:178:71: note: expanded from macro '__launch_bounds__'
    #        select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__)
    #                                                                          ^
    #    /opt/rocm-4.1.1/hip/include/hip/hcc_detail/hip_runtime.h:176:9: note: macro 'select_impl_' defined here
    #    #define select_impl_(_1, _2, impl_, ...) impl_
    # NOTE: -Werror was removed because ERF specifically had a lot of warnings. It will be a separate task to go through and fix them all...
    env: {CXXFLAGS: "-Wall -Wextra -Wpedantic -Wnull-dereference -Wfloat-conversion -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments -Wno-pass-failed"}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Dependencies
      run: Submodules/AMReX/.github/workflows/dependencies/dependencies_hip.sh

    - name: Install CCache
      run: Submodules/AMReX/.github/workflows/dependencies/dependencies_ccache.sh

    - name: Set Up Cache
      uses: actions/cache@v4
      with:
        path: ~/.cache/ccache
        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
        restore-keys: |
             ccache-${{ github.workflow }}-${{ github.job }}-git-

    - name: Build & Install
      run: |
        export CCACHE_COMPRESS=1
        export CCACHE_COMPRESSLEVEL=10
        export CCACHE_MAXSIZE=300M
        ccache -z

        source /etc/profile.d/rocm.sh
        hipcc --version
        which clang
        which clang++

        # "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3
        #   https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
        #   https://github.com/open-mpi/ompi/issues/9317
        export LDFLAGS="-lopen-pal"

        cmake \
          -B${{runner.workspace}}/ERF/build-${{matrix.os}} \
          -DCMAKE_VERBOSE_MAKEFILE=ON                   \
          -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \
          -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DERF_DIM:STRING=3 \
          -DERF_ENABLE_MPI:BOOL=ON \
          -DERF_ENABLE_HIP:BOOL=ON \
          -DAMReX_AMD_ARCH=gfx908 \
          -DERF_ENABLE_TESTS:BOOL=ON \
          -DERF_ENABLE_ALL_WARNINGS:BOOL=ON \
          -DERF_ENABLE_FCOMPARE:BOOL=ON \
          -DCODECOV:BOOL=ON \
          -DCMAKE_C_COMPILER=$(which clang)             \
          -DCMAKE_CXX_COMPILER=$(which clang++)         \
          -DCMAKE_CXX_STANDARD=17 \
          ${{github.workspace}};
        # ${{matrix.mpipreflags}} \

        # for some reason this cmake command fails to build the code,
        # and we need to use the make command that follows instead ...
        # cmake --build ${{runner.workspace}}/ERF/build-${{matrix.os}} --parallel ${{env.NPROCS}};

        pushd ${{runner.workspace}}/ERF/build-${{matrix.os}};
        # make -j ${{env.NPROCS}};
        make -j 2;

        ccache -s
        du -hs ~/.cache/ccache

    # - name: Regression Tests
    #   run: |
    #     ctest -L regression -VV
    #   working-directory: ${{runner.workspace}}/ERF/build-${{matrix.os}}

    # - name: Generate coverage report
    #   working-directory: ${{runner.workspace}}/ERF/build-${{matrix.os}}
    #   run: |
    #     find . -type f -name '*.gcno' -path "**Source**" -exec gcov -pb {} +
    #     cd ..
    #     gcovr -g -k -r . --xml regressioncov.xml  # -v

    # - name: Upload coverage to Codecov
    #   uses: codecov/codecov-action@v4
    #   with:
    #     dry_run: false
    #     # token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
    #     files: ./regressioncov.xml # optional
    #     flags: regtests # optional
    #     # name: codecov-umbrella # optional
    #     fail_ci_if_error: true # optional (default = false)
    #     verbose: true # optional (default = false)
    #     directory: ${{runner.workspace}}/ERF

    - name: Success artifacts
      uses: actions/upload-artifact@v4
      if: success()
      with:
        name: build-and-test
        path: |
          ${{runner.workspace}}/ERF/regressioncov.xml
    - name: Failing test artifacts
      uses: actions/upload-artifact@v4
      if: failure()
      with:
        name: build-and-test
        path: |
          ${{runner.workspace}}/ERF/regressioncov.xml

  save_pr_number:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - name: Save PR number
        env:
          PR_NUMBER: ${{ github.event.number }}
        run: |
          echo $PR_NUMBER > pr_number.txt
      - uses: actions/upload-artifact@v4
        with:
          name: pr_number
          path: pr_number.txt
          retention-days: 1