tests.yml

# Copyright (C) The DDC development team, see COPYRIGHT.md file
#
# SPDX-License-Identifier: MIT

name: test
on:
  push: { branches: [ main ] }
  pull_request:
concurrency:
  group: ${{ github.workflow }}-${{ github.ref == github.ref_protected && github.run_id || github.event.pull_request.number || github.ref }}
  cancel-in-progress: true
jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: DoozyX/clang-format-lint-action@v0.18
      with:
        source: 'benchmarks/ examples/ include/ddc/ install_test/ tests/'
        exclude: ''
        extensions: 'hpp,cpp'
        clangFormatVersion: 18
    - name: Find trailing whitespaces
      run: find . -not -path './vendor/*' -type f \( -name '*.[c|h]pp' -o -name 'CMakeLists.txt' -o -name '*.cmake' -o -name '*.md' -o -name '*.py' \) -exec ./bin/trailing_spaces --Werror '{}' '+'
    - name: REUSE Compliance Check
      if: always()
      uses: fsfe/reuse-action@v4

  cmake-format:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Install python3 virtual environments
      run: |
        sudo apt-get update
        sudo apt-get --yes install python3-pip python3-venv
    - name: Install gersemi
      run: |
        python3 -m venv ./gersemi
        . ./gersemi/bin/activate
        python3 -m pip install gersemi==0.17
    - name: Run gersemi
      run: |
        . ./gersemi/bin/activate
        gersemi --check cmake/DDCVendorConfiguration.cmake
        find CMakeLists.txt tests/ examples/ benchmarks/ -name 'CMakeLists.txt' -exec gersemi --check '{}' '+'

  id_repo:
    runs-on: ubuntu-latest
    steps:
    - name: Identify repository
      id: identify_repo
      run: |
        echo "in_base_repo=${{ (github.event_name == 'push' && github.repository == 'CExA-project/ddc') || github.event.pull_request.head.repo.full_name == 'CExA-project/ddc' }}" >> "$GITHUB_OUTPUT"
    outputs: { in_base_repo: '${{ steps.identify_repo.outputs.in_base_repo }}' }

  docker-build:
    strategy:
      fail-fast: false
      matrix:
        image: ['oldest', 'latest']
        backend: ['cpu', 'cuda', 'hip']
    needs: id_repo
    runs-on: ubuntu-latest
    steps:
    - name: Free Disk Space (Ubuntu)
      uses: jlumbroso/free-disk-space@v1.3.1
      with: { tool-cache: true, large-packages: false }
    - name: Checkout built branch
      uses: actions/checkout@v4
    - name: Login to GitHub Container Registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    - name: Build
      run: |
        DOCKER_BUILDKIT=1 docker build \
          --build-arg BACKEND=${{matrix.backend}} \
          --build-arg BUILDKIT_INLINE_CACHE=1 \
          --cache-from ghcr.io/cexa-project/ddc/${{matrix.image}}_${{matrix.backend}} \
          -t ghcr.io/cexa-project/ddc/${{matrix.image}}_${{matrix.backend}} \
          -t ghcr.io/cexa-project/ddc/${{matrix.image}}_${{matrix.backend}}:${GITHUB_SHA:0:7} \
          docker/${{matrix.image}}
    - name: Publish image for current SHA
      if: needs.id_repo.outputs.in_base_repo == 'true'
      run: |
        docker push ghcr.io/cexa-project/ddc/${{matrix.image}}_${{matrix.backend}}:${GITHUB_SHA:0:7}
    - name: Publish latest (default) image
      if: github.event_name == 'push' && github.ref_name == 'main' && needs.id_repo.outputs.in_base_repo == 'true'
      run: |
        docker push ghcr.io/cexa-project/ddc/${{matrix.image}}_${{matrix.backend}}
    - name: Create image tarball
      if: needs.id_repo.outputs.in_base_repo == 'false'
      run: |
        docker save ghcr.io/cexa-project/ddc/${{matrix.image}}_${{matrix.backend}}:${GITHUB_SHA:0:7} > ${{matrix.image}}_${{matrix.backend}}.tar
    - name: Generate docker artifact from image
      if: needs.id_repo.outputs.in_base_repo == 'false'
      uses: actions/upload-artifact@v4
      with:
        name: ${{matrix.image}}_${{matrix.backend}}-artifact
        path: ${{matrix.image}}_${{matrix.backend}}.tar
        retention-days: 1

  test:
    strategy:
      fail-fast: false
      matrix:
        image: ['oldest', 'latest']
        backend: ['cpu-gcc', 'cpu-clang', 'cuda', 'hip']
        cxx_version:  ['17', '20', '23']
        cmake_build_type: ['Debug', 'Release']
        exclude:
        - image: 'oldest' # nvcc 11 only supports C++-17
          backend: 'cuda'
          cxx_version: '20'
        - image: 'oldest' # nvcc 11 only supports C++-17
          backend: 'cuda'
          cxx_version: '23'
        - image: 'latest' # nvcc 12 only supports C++-20
          backend: 'cuda'
          cxx_version: '23'
    runs-on: ubuntu-20.04
    needs: [docker-build, id_repo]
    steps:
    - name: Free Disk Space (Ubuntu)
      uses: jlumbroso/free-disk-space@v1.3.1
      with: { tool-cache: true, large-packages: false }
    - name: Checkout built branch
      uses: actions/checkout@v4
      with: { submodules: recursive }
    - name: Collect image artifact
      if: needs.id_repo.outputs.in_base_repo == 'false'
      uses: actions/download-artifact@v4
      with:
        name: |
          ${{matrix.image}}_${{fromJSON('{"cpu-gcc":"cpu","cpu-clang":"cpu"}')[matrix.backend] || matrix.backend}}-artifact
    - name: Load image artifact into docker
      if: needs.id_repo.outputs.in_base_repo == 'false'
      run: |
        docker load < ${{matrix.image}}_${{fromJSON('{"gcc":"cpu","clang":"cpu"}')[matrix.backend] || matrix.backend}}.tar
        rm ${{matrix.image}}_${{fromJSON('{"cpu-gcc":"cpu","cpu-clang":"cpu"}')[matrix.backend] || matrix.backend}}.tar
    - name: Test
      id: test
      run: |
        cat<<-'EOF' > run.sh
          set -xe
          git config --global --add safe.directory '*'

          export benchmark_ROOT=$PWD/opt/benchmark
          export DDC_ROOT=$PWD/opt/ddc
          export GTest_ROOT=$PWD/opt/gtest
          export Kokkos_ROOT=$PWD/opt/kokkos
          export KokkosFFT_ROOT=$PWD/opt/kokkos-fft
          export KokkosKernels_ROOT=$PWD/opt/kokkos-kernels

          case "${{matrix.backend}}" in
          'cuda')
            export CC=${CUDA_GCC}
            export CXX=${CUDA_GXX}
            EXTRA_CMAKE_FLAGS="-DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_CUDA_CONSTEXPR=ON -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON -DKokkos_ARCH_AMPERE80=ON"
          ;;
          'hip')
            export CC=hipcc
            export CXX=hipcc
            # We extend the CMAKE_PREFIX_PATH with the content of _ROOT variables to support
            # rocm 5.7. (likely due to a cmake_minimum_required(VERSION 3.3))
            export CMAKE_PREFIX_PATH="/opt/rocm:$benchmark_ROOT:$DDC_ROOT:$GTest_ROOT:$Kokkos_ROOT:$KokkosFFT_ROOT:$KokkosKernels_ROOT"
            EXTRA_CMAKE_FLAGS="-DKokkos_ENABLE_HIP=ON -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE=ON -DKokkos_ENABLE_ROCTHRUST=OFF -DKokkos_ARCH_AMD_GFX90A=ON"
          ;;
          'cpu-clang')
            export CC=clang
            export CXX=clang++
            DDC_CMAKE_CXX_FLAGS="-Wextra-semi -Wextra-semi-stmt -Wold-style-cast"
          ;;
          'cpu-gcc')
            export CC=gcc
            export CXX=g++
            DDC_CMAKE_CXX_FLAGS="-Wextra-semi -Wold-style-cast"
            if [ 'xDebug' = 'x${{matrix.cmake_build_type}}' ]
            then
              DDC_CMAKE_CXX_FLAGS="${DDC_CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=address"
            fi
          ;;
          esac

          cmake \
            -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \
            -DCMAKE_CXX_STANDARD=${{matrix.cxx_version}} \
            -DBENCHMARK_ENABLE_GTEST_TESTS=OFF \
            -DBENCHMARK_ENABLE_TESTING=OFF \
            -DBENCHMARK_INSTALL_DOCS=OFF \
            -DBENCHMARK_USE_BUNDLED_GTEST=OFF \
            -B build \
            -S /src/vendor/benchmark
          cmake --build build --parallel 2
          cmake --install build --prefix $benchmark_ROOT
          rm -rf build

          cmake \
            -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \
            -DCMAKE_CXX_STANDARD=${{matrix.cxx_version}} \
            -B build \
            -S /src/vendor/googletest
          cmake --build build --parallel 2
          cmake --install build --prefix $GTest_ROOT
          rm -rf build

          cmake \
            -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \
            -DCMAKE_CXX_STANDARD=${{matrix.cxx_version}} \
            -DKokkos_ENABLE_DEPRECATED_CODE_4=OFF \
            -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF \
            -DKokkos_ENABLE_SERIAL=ON \
            ${EXTRA_CMAKE_FLAGS} \
            -B build \
            -S /src/vendor/kokkos
          cmake --build build --parallel 2
          cmake --install build --prefix $Kokkos_ROOT
          rm -rf build

          cmake \
            -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \
            -DCMAKE_CXX_STANDARD=${{matrix.cxx_version}} \
            -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON \
            -B build \
            -S /src/vendor/kokkos-fft
          cmake --build build --parallel 2
          cmake --install build --prefix $KokkosFFT_ROOT
          rm -rf build

          cmake \
            -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \
            -DCMAKE_CXX_STANDARD=${{matrix.cxx_version}} \
            -DKokkosKernels_ADD_DEFAULT_ETI=OFF \
            -DKokkosKernels_ENABLE_ALL_COMPONENTS=OFF \
            -DKokkosKernels_ENABLE_COMPONENT_BLAS=ON \
            -DKokkosKernels_ENABLE_COMPONENT_BATCHED=ON \
            -DKokkosKernels_ENABLE_COMPONENT_LAPACK=OFF \
            -DKokkosKernels_ENABLE_TPL_BLAS=OFF \
            -DKokkosKernels_ENABLE_TPL_CUSOLVER=OFF \
            -DKokkosKernels_ENABLE_TPL_LAPACK=OFF \
            -B build \
            -S /src/vendor/kokkos-kernels
          cmake --build build --parallel 2
          cmake --install build --prefix $KokkosKernels_ROOT
          rm -rf build

          cmake \
            -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \
            -DCMAKE_CXX_FLAGS="\
              -Werror -Wall -Wextra -Wpedantic -Wno-sign-compare -pedantic-errors \
              ${DDC_CMAKE_CXX_FLAGS}" \
            -DCMAKE_CXX_STANDARD=${{matrix.cxx_version}} \
            -DDDC_BUILD_BENCHMARKS=ON \
            -DDDC_benchmark_DEPENDENCY_POLICY=INSTALLED \
            -DDDC_GTest_DEPENDENCY_POLICY=INSTALLED \
            -DDDC_Kokkos_DEPENDENCY_POLICY=INSTALLED \
            -DDDC_KokkosFFT_DEPENDENCY_POLICY=INSTALLED \
            -B build \
            -S /src
          cmake --build build --parallel 2

          case "${{matrix.backend}}" in
          'cpu-'*)
            ctest --test-dir build --parallel 2 --output-on-failure --timeout 10 --output-junit tests.xml
            ./build/examples/heat_equation
            ./build/examples/uniform_heat_equation
            ./build/examples/non_uniform_heat_equation
            ./build/examples/heat_equation_spectral
            ./build/examples/game_of_life
            ./build/examples/characteristics_advection
          ;;
          esac

          cmake --install build --prefix $DDC_ROOT
          rm -rf build
          cmake \
            -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \
            -B build \
            -S /src/install_test
          cmake --build build --parallel 2

        EOF
        docker run \
          --cidfile='docker.cid' \
          -v ${PWD}:/src:ro \
          ghcr.io/cexa-project/ddc/${{matrix.image}}_${{fromJSON('{"cpu-gcc":"cpu","cpu-clang":"cpu"}')[matrix.backend] || matrix.backend}}:${GITHUB_SHA:0:7} \
          bash /src/run.sh
        if docker cp "$(cat docker.cid)":/data/tests.xml /home/runner/work/ddc/ddc/tests.xml
        then echo "with_report=true" >> "$GITHUB_OUTPUT"
        else echo "with_report=false" >> "$GITHUB_OUTPUT"
        fi
    - name: Publish Test Report
      uses: mikepenz/action-junit-report@v4
      if: ( success() || failure() ) && steps.test.outputs.with_report == 'true' # always run even if the previous step fails
      with:
        report_paths: '/home/runner/work/ddc/ddc/tests.xml'

  test-with-sanitizer-undefined:
    runs-on: ubuntu-latest
    needs: [docker-build, id_repo]
    steps:
    - name: Free Disk Space (Ubuntu)
      uses: jlumbroso/free-disk-space@v1.3.1
      with: { tool-cache: true, large-packages: false }
    - name: Checkout built branch
      uses: actions/checkout@v4
      with: { submodules: recursive }
    - name: Collect image artifact
      if: needs.id_repo.outputs.in_base_repo == 'false'
      uses: actions/download-artifact@v4
      with:
        name: oldest_cpu-artifact
    - name: Load image artifact into docker
      if: needs.id_repo.outputs.in_base_repo == 'false'
      run: |
        docker load < oldest_cpu.tar
        rm oldest_cpu.tar
    - name: Test
      id: test
      run: |
        cat<<-'UBSAN_EOF' > ubsan.supp
          vptr:ginkgo/core/solver/solver_base.hpp
        UBSAN_EOF
        cat<<-'EOF' > run.sh
          set -xe
          git config --global --add safe.directory '*'
          export CC=clang
          export CXX=clang++
          cmake \
            -DDDC_BUILD_BENCHMARKS=ON \
            -DCMAKE_BUILD_TYPE=Debug \
            -DCMAKE_CXX_STANDARD=17 \
            -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF \
            -DKokkos_ENABLE_DEPRECATED_CODE_4=OFF \
            -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF \
            -DCMAKE_CXX_FLAGS="-fsanitize=undefined -fno-omit-frame-pointer" \
            -B build \
            -S /src
          cmake --build build --parallel 2
          export UBSAN_OPTIONS=print_stacktrace=1,halt_on_error=1,suppressions=/src/ubsan.supp
          ctest --test-dir build --parallel 2 --output-on-failure --timeout 10 --output-junit tests.xml
          ./build/examples/heat_equation
          ./build/examples/uniform_heat_equation
          ./build/examples/non_uniform_heat_equation
          ./build/examples/heat_equation_spectral
          ./build/examples/game_of_life
          ./build/examples/characteristics_advection
        EOF
        docker run \
          --cidfile='docker.cid' \
          -v ${PWD}:/src:ro \
          ghcr.io/cexa-project/ddc/oldest_cpu:${GITHUB_SHA:0:7} \
          bash /src/run.sh
        if docker cp "$(cat docker.cid)":/data/tests.xml /home/runner/work/ddc/ddc/tests.xml
        then echo "with_report=true" >> "$GITHUB_OUTPUT"
        else echo "with_report=false" >> "$GITHUB_OUTPUT"
        fi
    - name: Publish Test Report
      uses: mikepenz/action-junit-report@v4
      if: ( success() || failure() ) && steps.test.outputs.with_report == 'true' # always run even if the previous step fails
      with:
        report_paths: '/home/runner/work/ddc/ddc/tests.xml'

  clang-tidy-test:
    runs-on: ubuntu-latest
    needs: [docker-build, id_repo]
    steps:
    - name: Free Disk Space (Ubuntu)
      uses: jlumbroso/free-disk-space@v1.3.1
      with: { tool-cache: true, large-packages: false }
    - name: Checkout built branch
      uses: actions/checkout@v4
      with: { submodules: recursive }
    - name: Collect image artifact
      if: needs.id_repo.outputs.in_base_repo == 'false'
      uses: actions/download-artifact@v4
      with: { name: 'oldest_cpu-artifact' }
    - name: Load image artifact into docker
      if: needs.id_repo.outputs.in_base_repo == 'false'
      run: |
        docker load < oldest_cpu-artifact.tar
        rm oldest_cpu-artifact.tar
    - name: clang-tidy
      run: |
        cat<<-'EOF' > run.sh
        set -xe
        git config --global --add safe.directory '*'
        cmake \
        -DCMAKE_CXX_COMPILER=clang++ \
        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
        -DDDC_BUILD_BENCHMARKS=ON \
        -DBUILD_TESTING=ON \
        -DCMAKE_BUILD_TYPE=Release \
        -DCMAKE_CXX_STANDARD=17 \
        -DKokkos_ENABLE_DEPRECATED_CODE_4=OFF \
        -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF \
        -B build \
        -S /src
        find /src/benchmarks /src/examples /src/tests -name '*.cpp' -exec clang-tidy -p build -header-filter="(/src/include/ddc/.*|/src/tests/.*)" '{}' '+'
        EOF
        docker run \
          -v ${PWD}:/src:ro ghcr.io/cexa-project/ddc/latest_cpu:${GITHUB_SHA:0:7} \
          bash /src/run.sh