.gitlab-ci.yml

# MIT License
#
# Copyright (c) 2017-2022 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

variables:
  GIT_SUBMODULE_STRATEGY: normal # Fetch submodules before job start (non-recursively)
  ROCM_PATH: /opt/rocm

include:
  - project: amd/ci-templates
    ref: main
    file:
      - /defaults.yaml
      - /deps-cmake.yaml
      - /deps-rocm.yaml
      - /deps-nvcc.yaml
      - /gpus-rocm.yaml
      - /gpus-nvcc.yaml
      - /rules.yaml

stages:
  - lint
  - build
  - test
  - benchmark

clang-format:
  extends:
    - .deps:rocm
  stage: lint
  needs: []
  tags:
    - rocm-build
  variables:
    CLANG_FORMAT: "/opt/rocm/llvm/bin/clang-format"
    GIT_CLANG_FORMAT: "/opt/rocm/llvm/bin/git-clang-format"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
  script:
    - cd $CI_PROJECT_DIR
    - git config --global --add safe.directory $CI_PROJECT_DIR
    - scripts/code-format/check-format.sh $CI_MERGE_REQUEST_DIFF_BASE_SHA --binary "$CLANG_FORMAT"

.rocm:cmake-minimum:
  variables:
    COMPILER: hipcc
  extends:
    - .deps:rocm
    - .deps:cmake-minimum
  before_script:
    - !reference [".deps:rocm", before_script]
    - !reference [".deps:cmake-minimum", before_script]

.rocm:cmake-latest:
  variables:
    COMPILER: hipcc
  extends:
    - .deps:rocm
    - .deps:cmake-latest
  before_script:
    - !reference [".deps:rocm", before_script]
    - !reference [".deps:cmake-latest", before_script]

.nvcc:cmake-minimum:
  variables:
    COMPILER: c++
  extends:
    - .deps:nvcc
    - .deps:cmake-minimum
  before_script:
    - !reference [".deps:nvcc", before_script]
    - !reference [".deps:cmake-minimum", before_script]

.nvcc:cmake-latest:
  variables:
    COMPILER: c++
  extends:
    - .deps:nvcc
    - .deps:cmake-latest
  before_script:
    - !reference [".deps:nvcc", before_script]
    - !reference [".deps:cmake-latest", before_script]

.deps:hip-cpu-extend:
  before_script:
    # C++20 standard library headers are required for HIP-CPU
    # Clang-15 is required to build hip-cpu, as rocm clang 5.4 and gcc all crash.
    - $SUDO_CMD apt-get update -qq
    - $SUDO_CMD apt-get install -y -qq software-properties-common
    - $SUDO_CMD apt-add-repository -y ppa:ubuntu-toolchain-r/test
    - wget -q -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO_CMD apt-key add -
    - echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main' | $SUDO_CMD tee /etc/apt/sources.list.d/llvm.list
    - $SUDO_CMD apt-get update -qq
    - $SUDO_CMD apt-get install -y -qq g++-11 clang-15

.hip-cpu:cmake-minimum:
  variables:
    COMPILER: clang++-15
  extends:
    - .deps:rocm
    - .deps:cmake-minimum
    - .deps:hip-cpu-extend
  before_script:
    - !reference [".deps:rocm", before_script]
    - !reference [".deps:cmake-minimum", before_script]
    - !reference [".deps:hip-cpu-extend", before_script]

.hip-cpu:cmake-latest:
  variables:
    COMPILER: clang++-15
  extends:
    - .deps:rocm
    - .deps:cmake-latest
    - .deps:hip-cpu-extend
  before_script:
    - !reference [".deps:rocm", before_script]
    - !reference [".deps:cmake-latest", before_script]
    - !reference [".deps:hip-cpu-extend", before_script]

.rocm:build:
  stage: build
  extends:
    - .gpus:rocm-gpus
    - .rules:build
  needs: []
  script:
    - cmake
      -S $CI_PROJECT_DIR
      -B $CI_PROJECT_DIR/build
      -G Ninja
      -D CMAKE_CXX_COMPILER=${COMPILER}
      -D BUILD_TEST=ON
      -D BUILD_BENCHMARK=ON
      -D BUILD_FORTRAN_WRAPPER=OFF
      -D DEPENDENCIES_FORCE_DOWNLOAD=ON
      -D BUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
      -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
      -D GPU_TARGETS=${GPU_TARGETS}
    - cmake --build $CI_PROJECT_DIR/build
    - cmake --build $CI_PROJECT_DIR/build --target package

.nvcc:build:
  stage: build
  extends:
    - .gpus:nvcc-gpus
    - .rules:build
  needs: []
  script:
    - cmake
      -S $CI_PROJECT_DIR
      -B $CI_PROJECT_DIR/build
      -G Ninja
      -D CMAKE_CXX_COMPILER=${COMPILER}
      -D BUILD_TEST=ON
      -D BUILD_BENCHMARK=ON
      -D BUILD_FORTRAN_WRAPPER=OFF
      -D DEPENDENCIES_FORCE_DOWNLOAD=ON
      -D BUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
      -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
      -D NVGPU_TARGETS=${GPU_TARGETS}
    - cmake --build $CI_PROJECT_DIR/build
    - cmake --build $CI_PROJECT_DIR/build --target package

.hip-cpu:build:
  stage: build
  extends:
    - .rules:build
  needs: []
  script:
    - cmake
      -S $CI_PROJECT_DIR
      -B $CI_PROJECT_DIR/build
      -G Ninja
      -D CMAKE_CXX_COMPILER=${COMPILER}
      -D BUILD_TEST=ON
      -D BUILD_BENCHMARK=ON
      -D BUILD_FORTRAN_WRAPPER=OFF
      -D DEPENDENCIES_FORCE_DOWNLOAD=ON
      -D BUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
      -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
      -D USE_HIP_CPU=ON
      -D BUILD_HIPRAND=OFF
      -D CMAKE_INSTALL_PREFIX=${ROCRAND_DIR}
    - cmake --build $CI_PROJECT_DIR/build
    - cmake --build $CI_PROJECT_DIR/build --target package

# Build output is too big to upload in the case of the static library builds
.save-artifacts:
  artifacts:
    paths:
      - $CI_PROJECT_DIR/build/library/
      - $CI_PROJECT_DIR/build/test/test_*
      - $CI_PROJECT_DIR/build/test/CTestTestfile.cmake
      - $CI_PROJECT_DIR/build/benchmark/benchmark_*
      - $CI_PROJECT_DIR/build/deps/googlebenchmark/
      - $CI_PROJECT_DIR/build/deps/hip-cpu/
      - $CI_PROJECT_DIR/build/deps/tbb/
      - $CI_PROJECT_DIR/build/tbb-src/
      - $CI_PROJECT_DIR/build/CMakeCache.txt
      - $CI_PROJECT_DIR/build/CTestTestfile.cmake
      - $CI_PROJECT_DIR/build/*.deb
      - $CI_PROJECT_DIR/build/*.zip
    expire_in: 2 weeks

build:rocm-cmake-minimum:
  tags:
    - rocm-build
  needs: []
  extends:
    - .rocm:cmake-minimum
    - .rocm:build
    - .save-artifacts
  variables:
    BUILD_SHARED_LIBS: "ON"

build:rocm-static-cmake-minimum:
  tags:
    - rocm-build
  needs: []
  extends:
    - .rocm:cmake-minimum
    - .rocm:build
  variables:
    BUILD_SHARED_LIBS: "OFF"

build:rocm-cmake-latest:
  tags:
    - rocm-build
  needs: []
  extends:
    - .rocm:cmake-latest
    - .rocm:build
  variables:
    BUILD_SHARED_LIBS: "ON"

build:rocm-static-cmake-latest:
  tags:
    - rocm-build
  needs: []
  extends:
    - .rocm:cmake-latest
    - .rocm:build
  variables:
    BUILD_SHARED_LIBS: "OFF"

build:nvcc-cmake-minimum:
  tags:
    - nvcc-build
  needs: []
  extends:
    - .nvcc:cmake-minimum
    - .nvcc:build
    - .save-artifacts
  variables:
    BUILD_SHARED_LIBS: "ON"

build:nvcc-static-cmake-minimum:
  tags:
    - nvcc-build
  needs: []
  extends:
    - .nvcc:cmake-minimum
    - .nvcc:build
  variables:
    BUILD_SHARED_LIBS: "OFF"

build:nvcc-cmake-latest:
  tags:
    - nvcc-build
  needs: []
  extends:
    - .nvcc:cmake-latest
    - .nvcc:build
  variables:
    BUILD_SHARED_LIBS: "ON"

build:nvcc-static-cmake-latest:
  tags:
    - nvcc-build
  needs: []
  extends:
    - .nvcc:cmake-latest
    - .nvcc:build
  variables:
    BUILD_SHARED_LIBS: "OFF"

build-cpu:cmake-minimum:
  tags:
    - rocm-build
  needs: []
  extends:
    - .hip-cpu:cmake-minimum
    - .hip-cpu:build
    - .save-artifacts
  variables:
    BUILD_SHARED_LIBS: "ON"

build-cpu:cmake-latest:
  tags:
    - rocm-build
  needs: []
  extends:
    - .hip-cpu:cmake-latest
    - .hip-cpu:build
    - .save-artifacts
  variables:
    BUILD_SHARED_LIBS: "ON"

.test:ctest:
  extends:
    - .rules:test
  stage: test
  script:
    - cd $CI_PROJECT_DIR/build
    - ctest --output-on-failure

test:rocm:
  needs:
    - build:rocm-cmake-minimum
  extends:
    - .rocm:cmake-minimum
    - .test:ctest
    - .gpus:rocm

test:nvcc:
  needs:
    - build:nvcc-cmake-minimum
  extends:
    - .nvcc:cmake-minimum
    - .test:ctest
    - .gpus:nvcc
  before_script:
    # This is only needed because of the legacy before_script in .gpus:nvcc
    - !reference [".nvcc:cmake-minimum", before_script]

test:hip-cpu:
  needs:
    - build-cpu:cmake-minimum
  extends:
    - .hip-cpu:cmake-minimum
    - .test:ctest
  tags:
    - rocm-build

.benchmark:
  extends:
    # rocRAND benchmarks are short, we can afford to run them automatically
    - .rules:test
  stage: benchmark
  script:
    - cd $CI_PROJECT_DIR/build
    - ./benchmark/benchmark_rocrand_host_api --trials 5
    - ./benchmark/benchmark_rocrand_kernel --dis all --engine all --trials 5

benchmark:rocm:
  needs:
    - build:rocm-cmake-minimum
  extends:
    - .rocm:cmake-minimum
    - .benchmark
    - .gpus:rocm

benchmark:nvcc:
  needs:
    - build:nvcc-cmake-minimum
  extends:
    - .nvcc:cmake-minimum
    - .benchmark
    - .gpus:nvcc

benchmark:hip-cpu:
  needs:
    - build-cpu:cmake-minimum
  extends:
    - .hip-cpu:cmake-minimum
    - .benchmark
  tags:
    - rocm-build
  timeout: 8h

.test:python:
  stage: test
  extends:
    - .rules:test
  before_script:
    - $SUDO_CMD apt-get update -qq
    - $SUDO_CMD apt-get install -y -qq python3 python3-pip python3-numpy
    - $SUDO_CMD apt-get install -y -qq wget
    - pip3 install setuptools
    - export ROCRAND_PATH=$CI_PROJECT_DIR/build/library/
  script:
    # rocRAND Wrapper with Python 3
    - pip3 --version
    - cd $CI_PROJECT_DIR/python/rocrand
    - python3 setup.py test
    - pip3 install . --user
    - python3 tests/rocrand_test.py
    - pip3 uninstall --yes rocrand

test:rocm-python:
  tags:
    - rocm
  extends:
    - .rocm:cmake-minimum
    - .test:python
  needs:
    - build:rocm-cmake-minimum

test:nvcc-python:
  tags:
    - nvcc
  extends:
    - .nvcc:cmake-minimum
    - .test:python
  needs:
    - build:nvcc-cmake-minimum

.test:package:
  stage: test
  extends:
    - .rules:test
  script:
    - cd $CI_PROJECT_DIR/build
    - $SUDO_CMD dpkg -i rocrand*.deb
    - cmake
      -S $CI_PROJECT_DIR/test/package/
      -B $CI_PROJECT_DIR/build_package_test
      -D CMAKE_CXX_COMPILER=${COMPILER}
    - cmake --build $CI_PROJECT_DIR/build_package_test
    - cd $CI_PROJECT_DIR/build_package_test
    - ctest --output-on-failure
    - $SUDO_CMD dpkg -r rocrand-benchmarks rocrand-clients rocrand-dev rocrand-tests rocrand

.test-cpu:package:
  stage: test
  extends:
    - .rules:test
  script:
    - cd $CI_PROJECT_DIR/build
    - $SUDO_CMD dpkg -i rocrand*.deb
    - cmake
      -S $CI_PROJECT_DIR/test/package/
      -B $CI_PROJECT_DIR/build_package_test
      -D CMAKE_CXX_COMPILER=${COMPILER}
      -D USE_HIP_CPU=ON
      -D BUILD_HIPRAND=OFF
      -D TBB_DIR=$CI_PROJECT_DIR/build/tbb-src/cmake
      -D hip_cpu_rt_DIR=$CI_PROJECT_DIR/build/deps/hip-cpu/share/hip_cpu_rt/cmake
      -D rocrand_DIR=${ROCRAND_DIR}
    - cmake --build $CI_PROJECT_DIR/build_package_test
    - cd $CI_PROJECT_DIR/build_package_test
    - ctest --output-on-failure
    - $SUDO_CMD dpkg -r rocrand-benchmarks rocrand-clients rocrand-dev rocrand-tests rocrand

test:rocm-package:
  tags:
    - rocm
  extends:
    - .rocm:cmake-minimum
    - .test:package
  needs:
    - build:rocm-cmake-minimum

test:nvcc-package:
  tags:
    - nvcc
  extends:
    - .nvcc:cmake-minimum
    - .test:package
  needs:
    - build:nvcc-cmake-minimum

test:hip-cpu-package:
  tags:
    - rocm
  extends:
    - .hip-cpu:cmake-minimum
    - .test-cpu:package
  needs:
    - build-cpu:cmake-minimum

.test:install:
  stage: test
  extends:
    - .rules:test
  script:
    - cmake
      -S $CI_PROJECT_DIR
      -B $CI_PROJECT_DIR/build_only_install
      -D BUILD_TEST=OFF
      -D BUILD_FORTRAN_WRAPPER=OFF
      -D BUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
      -D CMAKE_CXX_COMPILER=${COMPILER}
    - $SUDO_CMD cmake --build $CI_PROJECT_DIR/build_only_install --target install
    - cmake
      -S $CI_PROJECT_DIR/test/package/
      -B $CI_PROJECT_DIR/install_test
      -D CMAKE_CXX_COMPILER=${COMPILER}
    - cmake --build $CI_PROJECT_DIR/install_test
    - cd $CI_PROJECT_DIR/install_test
    - ctest --output-on-failure

.test-cpu:install:
  stage: test
  extends:
    - .rules:test
  script:
    - cmake
      -S $CI_PROJECT_DIR
      -B $CI_PROJECT_DIR/build_only_install
      -D BUILD_TEST=OFF
      -D BUILD_FORTRAN_WRAPPER=OFF
      -D BUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF
      -D CMAKE_CXX_COMPILER=${COMPILER}
      -D USE_HIP_CPU=ON
      -D BUILD_HIPRAND=OFF
      -D DEPENDENCIES_FORCE_DOWNLOAD=ON
      -D CMAKE_INSTALL_PREFIX=${ROCRAND_DIR}
    - $SUDO_CMD cmake --build $CI_PROJECT_DIR/build_only_install --target install
    - cmake
      -S $CI_PROJECT_DIR/test/package/
      -B $CI_PROJECT_DIR/install_test
      -D CMAKE_CXX_COMPILER=${COMPILER}
      -D USE_HIP_CPU=ON
      -D TBB_DIR=$CI_PROJECT_DIR/build_only_install/tbb-src/cmake
      -D hip_cpu_rt_DIR=$CI_PROJECT_DIR/build_only_install/deps/hip-cpu/share/hip_cpu_rt/cmake
      -D rocrand_DIR=${ROCRAND_DIR}
    - cmake --build $CI_PROJECT_DIR/install_test
    - cd $CI_PROJECT_DIR/install_test
    - ctest --output-on-failure

test:rocm-install:
  tags:
    - rocm
  extends:
    - .rocm:cmake-minimum
    - .test:install
  needs: []

test:nvcc-install:
  tags:
    - nvcc
  extends:
    - .nvcc:cmake-minimum
    - .test:install
  needs: []

test:hip-cpu-install:
  variables:
    ROCRAND_DIR: $CI_PROJECT_DIR/rocRAND
  tags:
    - rocm
  extends:
    - .hip-cpu:cmake-minimum
    - .test-cpu:install
  needs: []

test:doc:
  stage: test
  extends:
    - .rules:test
  image: sphinxdoc/sphinx-latexpdf:latest
  needs: []
  before_script:
    - apt-get update
    - apt-get install -y -qq doxygen
    - pip3 install --user -r $CI_PROJECT_DIR/docs/requirements.txt
  script:
    - $CI_PROJECT_DIR/docs/run_doc.sh
  artifacts:
    paths:
      - $CI_PROJECT_DIR/docs/_build
      - $CI_PROJECT_DIR/docs/docBin
    expire_in: 1 week

.test:parity:
  stage: test
  extends:
    - .rules:test
  script:
    - cd $CI_PROJECT_DIR/build
    - $SUDO_CMD dpkg -i rocrand*.deb
    - cmake
      -S $CI_PROJECT_DIR/test/parity
      -B $CI_PROJECT_DIR/build_parity_test
      -D CMAKE_CXX_COMPILER=${COMPILER}
    - cmake --build $CI_PROJECT_DIR/build_parity_test
    - $CI_PROJECT_DIR/build_parity_test/test_rocrand_parity | tee $CI_PROJECT_DIR/build_parity_test/parity-test-results.csv
    - $SUDO_CMD dpkg -r rocrand-benchmarks rocrand-clients rocrand-dev rocrand-tests rocrand
  artifacts:
    paths:
      - $CI_PROJECT_DIR/build_parity_test/parity-test-results.csv
    expire_in: 2 weeks

test:nvcc-parity:
  tags:
    - nvcc
  extends:
    - .nvcc:cmake-minimum
    - .test:parity
    - .gpus:nvcc
  needs:
    - build:nvcc-cmake-minimum
  before_script:
    - !reference [".nvcc:cmake-minimum", before_script]