diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0fd46d224..620d4da42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ stages: .build_script: script: #Use pre-existing allocation if any - - JOBID=$(if [[ "$SYS_TYPE" == "toss_3_x86_64_ib" ]]; then squeue -h --name=${PROJECT_ALLOC_NAME} --format=%A; fi) + - JOBID=$(if [[ $SYS_TYPE = toss_* ]]; then squeue -h --name=${PROJECT_ALLOC_NAME} --format=%A; fi) - ASSIGN_ID=$(if [[ -n "${JOBID}" ]]; then echo "--jobid=${JOBID}"; fi) #BUILD + TEST - echo -e "section_start:$(date +%s):build_and_test\r\e[0K @@ -35,3 +35,4 @@ stages: include: - local: .gitlab/build_quartz.yml - local: .gitlab/build_lassen.yml + - local: .gitlab/build_corona.yml diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh index cd6130187..b65ed3c94 100755 --- a/.gitlab/build_and_test.sh +++ b/.gitlab/build_and_test.sh @@ -14,7 +14,7 @@ mkdir $BUILD_DIR cd $BUILD_DIR cmake -C ../host-configs/llnl/$SYS_TYPE/$HOST_CONFIG ../tests/internal -make -j8 +make VERBOSE=1 -j8 ctest -DCTEST_OUTPUT_ON_FAILURE=1 --no-compress-output -T Test -VV xsltproc -o junit.xml ../tests/ctest-to-junit.xsl Testing/*/Test.xml diff --git a/.gitlab/build_corona.yml b/.gitlab/build_corona.yml new file mode 100644 index 000000000..ee19a551a --- /dev/null +++ b/.gitlab/build_corona.yml @@ -0,0 +1,51 @@ +#### +# This is the share configuration of jobs for corona +.on_corona: + tags: + - shell + - corona + rules: + - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_CORONA == "OFF"' #run except if ... + when: never + - if: '$CI_JOB_NAME =~ /corona_release/' + when: always + - when: on_success + +#### +# In pre-build phase, allocate a node for builds +corona_allocate: + variables: + GIT_STRATEGY: none + extends: [.on_corona] + stage: allocate + script: + - salloc -p pbatch -N 1 -t 10 --no-shell --job-name=${PROJECT_ALLOC_NAME} --mpibind=off + needs: [] + +#### +# In post-build phase, deallocate resources +# Note : make sure this is run even on build phase failure +corona_release: + variables: + GIT_STRATEGY: none + extends: [.on_corona] + stage: release + script: + - export JOBID=$(squeue -h --name=${PROJECT_ALLOC_NAME} --format=%A) + - if [[ -n "${JOBID}" ]]; then scancel ${JOBID}; fi + +#### +# Template +.build_on_corona: + stage: build + variables: + ALLOC_COMMAND: "srun -p pbatch -t 10 -N 1 ${ASSIGN_ID} --interactive" + extends: [.build_script, .on_corona] + needs: [corona_allocate] + +#### +# Build jobs +corona-rocm_4_5_2_hip: + variables: + HOST_CONFIG: "rocm@4.5.2_hip.cmake" + extends: [.build_on_corona] diff --git a/.gitlab/build_quartz.yml b/.gitlab/build_quartz.yml index 6ba15e031..b9e49288f 100644 --- a/.gitlab/build_quartz.yml +++ b/.gitlab/build_quartz.yml @@ -7,7 +7,7 @@ rules: - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_QUARTZ == "OFF"' #run except if ... when: never - - if: '$CI_JOB_NAME =~ /release_resources/' + - if: '$CI_JOB_NAME =~ /quartz_release/' when: always - when: on_success @@ -39,8 +39,9 @@ quartz_release: .build_on_quartz: stage: build variables: - ALLOC_COMMAND: "srun -p pdebug -t 30 -N 1 ${ASSIGN_ID} --interactive" + ALLOC_COMMAND: "srun -p pdebug -t 10 -N 1 ${ASSIGN_ID} --interactive" extends: [.build_script, .on_quartz] + needs: [quartz_allocate] #### # Build jobs diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 35edd4614..60519a26e 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -9,6 +9,22 @@ The project release numbers follow [Semantic Versioning](http://semver.org/spec/ ## [Unreleased] - Release date yyyy-mm-dd +## [Version 0.5.1] - Release date 2022-04-22 + +### Added +- Added support for C++20. Note: XL does not support C++20. + While PGI has C++20 support, it is currently disabled (A BLT fatal error will occur). +- BLT_CXX_STD now sets CMAKE_HIP_STANDARD, in CMake 3.21+, similar to CMAKE_CUDA_STANDARD. + +### Fixed +- Removed hard-coded -std=c++11 from various places related to CUDA flags. This now honors + CMAKE_CUDA_STANDARD if set otherwise falls back on BLT_CXX_STD or CMAKE_CXX_STANDARD. +- Removed extra HIP offload flags that were being added as generator expressions as opposed to simple + flags. + +### Removed +- Removed support for deprecated HCC. + ## [Version 0.5.0] - Release date 2022-03-07 ### Added @@ -235,7 +251,8 @@ The project release numbers follow [Semantic Versioning](http://semver.org/spec/ -[Unreleased]: https://github.com/LLNL/blt/compare/v0.5.0...develop +[Unreleased]: https://github.com/LLNL/blt/compare/v0.5.1...develop +[Version 0.5.1]: https://github.com/LLNL/blt/compare/v0.5.0...v0.5.1 [Version 0.5.0]: https://github.com/LLNL/blt/compare/v0.4.1...v0.5.0 [Version 0.4.1]: https://github.com/LLNL/blt/compare/v0.4.0...v0.4.1 [Version 0.4.0]: https://github.com/LLNL/blt/compare/v0.3.6...v0.4.0 diff --git a/SetupBLT.cmake b/SetupBLT.cmake index 2a6db6610..460c8d75c 100644 --- a/SetupBLT.cmake +++ b/SetupBLT.cmake @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (BSD-3-Clause) if (NOT BLT_LOADED) - set(BLT_VERSION "0.5.0" CACHE STRING "") + set(BLT_VERSION "0.5.1" CACHE STRING "") mark_as_advanced(BLT_VERSION) message(STATUS "BLT Version: ${BLT_VERSION}") @@ -33,6 +33,13 @@ if (NOT BLT_LOADED) # Show CMake info right out of the gate ################################ message(STATUS "CMake Version: ${CMAKE_VERSION}") + get_property(_is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if (_is_multi_config) + message(STATUS "CMake Configuration Types: ${CMAKE_CONFIGURATION_TYPES}") + else() + message(STATUS "CMake Build Type: ${CMAKE_BUILD_TYPE}") + endif() + unset(_is_multi_config) if(${CMAKE_VERSION} VERSION_LESS 3.8.0) message("*************************************") @@ -95,7 +102,6 @@ if (NOT BLT_LOADED) set_property(GLOBAL PROPERTY USE_FOLDERS ON) endif() - ################################ # Enable cmake compilation database feature ################################ @@ -106,6 +112,11 @@ if (NOT BLT_LOADED) ################################ include(${BLT_ROOT_DIR}/cmake/BLTMacros.cmake) + ################################ + # Setup compiler standards early + ################################ + include(${BLT_ROOT_DIR}/cmake/SetupCompilerStandards.cmake) + ################################ # Standard TPL support ################################ @@ -212,3 +223,4 @@ if (NOT BLT_LOADED) endif() endif() # only load BLT once! + diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 722e3e439..6d5d78547 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -958,7 +958,6 @@ endmacro(blt_add_benchmark) ## DEFAULT defaultFlag (optional) ## GNU gnuFlag (optional) ## CLANG clangFlag (optional) -## HCC hccFlag (optional) ## INTEL intelFlag (optional) ## INTELLLVM intelLLVMFlag (optional) ## XL xlFlag (optional) @@ -972,7 +971,7 @@ endmacro(blt_add_benchmark) macro(blt_append_custom_compiler_flag) set(options) - set(singleValueArgs FLAGS_VAR DEFAULT GNU CLANG HCC PGI INTEL INTELLLVM XL MSVC MSVC_INTEL CRAY) + set(singleValueArgs FLAGS_VAR DEFAULT GNU CLANG PGI INTEL INTELLLVM XL MSVC MSVC_INTEL CRAY) set(multiValueArgs) # Parse the arguments diff --git a/cmake/BLTOptions.cmake b/cmake/BLTOptions.cmake index 564c06590..ec4594537 100644 --- a/cmake/BLTOptions.cmake +++ b/cmake/BLTOptions.cmake @@ -59,7 +59,6 @@ mark_as_advanced(ENABLE_CLANG_CUDA) set(BLT_CLANG_CUDA_ARCH "sm_30" CACHE STRING "Compute architecture to use when generating CUDA code with Clang") mark_as_advanced(BLT_CLANG_CUDA_ARCH) option(ENABLE_HIP "Enable HIP support" OFF) -option(ENABLE_HCC "Enable HCC support" OFF) set(CMAKE_HIP_ARCHITECTURES "gfx900" CACHE STRING "gfx architecture to use when generating HIP/ROCm code") #------------------------------------------------------------------------------ diff --git a/cmake/SetupCompilerOptions.cmake b/cmake/SetupCompilerOptions.cmake index 5fd3c1652..2af48cd8d 100644 --- a/cmake/SetupCompilerOptions.cmake +++ b/cmake/SetupCompilerOptions.cmake @@ -134,54 +134,26 @@ endif() # all targets that use -Compiler ########################################## -########################################## -# Support extra flags for the C compiler. -########################################## if(BLT_C_FLAGS) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${BLT_C_FLAGS}") message(STATUS "Updated CMAKE_C_FLAGS to \"${CMAKE_C_FLAGS}\"") endif() -############################################# -# Support extra flags for the C++ compiler. -############################################# if(BLT_CXX_FLAGS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BLT_CXX_FLAGS}") message(STATUS "Updated CMAKE_CXX_FLAGS to \"${CMAKE_CXX_FLAGS}\"") endif() -################################################ -# Support extra flags for the Fortran compiler. -################################################ if(ENABLE_FORTRAN AND BLT_FORTRAN_FLAGS) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${BLT_FORTRAN_FLAGS}") - message(STATUS "Updated CMAKE_Fortran_FLAGS to \"${CMAKE_Fortran_FLAGS}\"") + message(STATUS "Updated CMAKE_Fortran_FLAGS to \"${CMAKE_Fortran_FLAGS}\"") endif() - -############################################################ -# Map Legacy FindCUDA variables to native cmake variables -# Note - we are intentionally breaking the semicolon delimited -# list that FindCUDA demanded of CUDA_NVCC_FLAGS so users -# are forced to clean up their host configs. -############################################################ -if (ENABLE_CUDA) - if (BLT_CUDA_FLAGS) - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${BLT_CUDA_FLAGS}") - endif() - # quirk of ordering means that one needs to define -std=c++11 in CMAKE_CUDA_FLAGS if - # --expt-extended-lambda is being used so cmake can get past the compiler check, - # but the CMAKE_CUDA_STANDARD stuff adds another definition in which breaks things. - # So we rip it out here, but it ends up being inserted in the final build rule by cmake. - if (CMAKE_CUDA_FLAGS) - STRING(REPLACE "-std=c++11" " " CMAKE_CUDA_FLAGS ${CMAKE_CUDA_FLAGS} ) - endif() +if(ENABLE_CUDA AND BLT_CUDA_FLAGS) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} ${BLT_CUDA_FLAGS}") + message(STATUS "Updated CMAKE_CUDA_FLAGS to \"${CMAKE_CUDA_FLAGS}\"") endif() - -################################################ -# Support extra linker flags -################################################ if(BLT_EXE_LINKER_FLAGS) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${BLT_EXE_LINKER_FLAGS}") message(STATUS "Updated CMAKE_EXE_LINKER_FLAGS to \"${CMAKE_EXE_LINKER_FLAGS}\"") @@ -253,61 +225,6 @@ if(BUILD_SHARED_LIBS) endif() endif() -################################ -# C++ Standard -################################ -if (NOT DEFINED CMAKE_CXX_EXTENSIONS) - message(STATUS "Setting CMAKE_CXX_EXTENSIONS to Off") - set( CMAKE_CXX_EXTENSIONS OFF ) -endif () -SET( CMAKE_CXX_STANDARD_REQUIRED ON ) - -set(BLT_CXX_STD "" CACHE STRING "Version of C++ standard") -set_property(CACHE BLT_CXX_STD PROPERTY STRINGS c++98 c++11 c++14 c++17) - -if (BLT_CXX_STD) - if( BLT_CXX_STD STREQUAL c++98 ) - set(CMAKE_CXX_STANDARD 98) - elseif( BLT_CXX_STD STREQUAL c++11 ) - set(CMAKE_CXX_STANDARD 11) - blt_append_custom_compiler_flag( - FLAGS_VAR CMAKE_CXX_FLAGS - DEFAULT " " - XL "-std=c++11" - PGI "--c++11") - elseif( BLT_CXX_STD STREQUAL c++14) - set(CMAKE_CXX_STANDARD 14) - blt_append_custom_compiler_flag( - FLAGS_VAR CMAKE_CXX_FLAGS - DEFAULT " " - XL "-std=c++14" - PGI "--c++14") - elseif( BLT_CXX_STD STREQUAL c++17) - # Error out on what does not support C++17 - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "XL") - message(FATAL_ERROR "XL does not support C++17.") - endif() - if (ENABLE_CUDA AND (NOT DEFINED CMAKE_CUDA_COMPILE_FEATURES OR (NOT "cuda_std_17" IN_LIST CMAKE_CUDA_COMPILE_FEATURES))) - message(FATAL_ERROR "CMake's CUDA_STANDARD does not support C++17.") - endif() - - set(CMAKE_CXX_STANDARD 17) - blt_append_custom_compiler_flag( - FLAGS_VAR CMAKE_CXX_FLAGS - DEFAULT " " - PGI "--c++17") - else() - message(FATAL_ERROR "${BLT_CXX_STD} is an invalid entry for BLT_CXX_STD. " - "Valid Options are ( c++98, c++11, c++14, c++17 )") - endif() - - if (ENABLE_CUDA) - set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD}) - endif() - - message(STATUS "Standard C++${CMAKE_CXX_STANDARD} selected") -endif() - ################################################################## # Additional compiler warnings and treatment of warnings as errors @@ -319,7 +236,6 @@ blt_append_custom_compiler_flag( CLANG "-Wall -Wextra" # Additional possibilities for clang include: # "-Wdocumentation -Wdeprecated -Weverything" - HCC "-Wall" PGI "-Minform=warn" MSVC "/W4" # Additional possibilities for visual studio include: @@ -336,7 +252,6 @@ blt_append_custom_compiler_flag( CLANG "-Wall -Wextra" # Additional possibilities for clang include: # "-Wdocumentation -Wdeprecated -Weverything" - HCC "-Wall" PGI "-Minform=warn" MSVC "/W4" # Additional possibilities for visual studio include: diff --git a/cmake/SetupCompilerStandards.cmake b/cmake/SetupCompilerStandards.cmake new file mode 100644 index 000000000..d8f70f3ac --- /dev/null +++ b/cmake/SetupCompilerStandards.cmake @@ -0,0 +1,73 @@ +# Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and +# other BLT Project Developers. See the top-level LICENSE file for details +# +# SPDX-License-Identifier: (BSD-3-Clause) + +if(NOT DEFINED CMAKE_CXX_EXTENSIONS) + message(STATUS "Setting CMAKE_CXX_EXTENSIONS to OFF") + set(CMAKE_CXX_EXTENSIONS OFF) +endif() +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(BLT_CXX_STD "" CACHE STRING "Version of C++ standard") +set_property(CACHE BLT_CXX_STD PROPERTY STRINGS c++98 c++11 c++14 c++17 c++20) + +if (BLT_CXX_STD) + if( BLT_CXX_STD STREQUAL c++98 ) + set(CMAKE_CXX_STANDARD 98) + elseif( BLT_CXX_STD STREQUAL c++11 ) + set(CMAKE_CXX_STANDARD 11) + blt_append_custom_compiler_flag( + FLAGS_VAR CMAKE_CXX_FLAGS + DEFAULT " " + XL "-std=c++11" + PGI "--c++11") + elseif( BLT_CXX_STD STREQUAL c++14) + set(CMAKE_CXX_STANDARD 14) + blt_append_custom_compiler_flag( + FLAGS_VAR CMAKE_CXX_FLAGS + DEFAULT " " + XL "-std=c++14" + PGI "--c++14") + elseif( BLT_CXX_STD STREQUAL c++17) + # Error out on what does not support C++17 + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "XL") + message(FATAL_ERROR "XL does not support C++17.") + endif() + set(CMAKE_CXX_STANDARD 17) + blt_append_custom_compiler_flag( + FLAGS_VAR CMAKE_CXX_FLAGS + DEFAULT " " + PGI "--c++17") + elseif( BLT_CXX_STD STREQUAL c++20) + # Error out on what does not support C++20 + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "XL") + message(FATAL_ERROR "XL does not support C++20.") + endif() + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI") + message(FATAL_ERROR "PGI is not yet supported using C++20.") + endif() + set(CMAKE_CXX_STANDARD 20) + else() + message(FATAL_ERROR "${BLT_CXX_STD} is an invalid entry for BLT_CXX_STD. " + "Valid Options are ( c++98, c++11, c++14, c++17, c++20 )") + endif() + + message(STATUS "Standard C++${CMAKE_CXX_STANDARD} selected") + + if (ENABLE_CUDA) + if (NOT DEFINED CMAKE_CUDA_STANDARD) + set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD}) + endif() + + message(STATUS "Standard C++${CMAKE_CUDA_STANDARD} selected for CUDA") + endif() + + if (ENABLE_HIP AND ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.21.0") + if (NOT DEFINED CMAKE_HIP_STANDARD) + set(CMAKE_HIP_STANDARD ${CMAKE_CXX_STANDARD}) + endif() + + message(STATUS "Standard C++${CMAKE_HIP_STANDARD} selected for HIP") + endif() +endif() diff --git a/cmake/thirdparty/FindROCm.cmake b/cmake/thirdparty/FindROCm.cmake deleted file mode 100644 index 2ca58d954..000000000 --- a/cmake/thirdparty/FindROCm.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and -# other BLT Project Developers. See the top-level LICENSE file for details -# -# SPDX-License-Identifier: (BSD-3-Clause) - -# Author: Chip Freitag @ Advanced Micro Devices, Inc. -# Date: February 14, 2018 - -find_path(ROCM_PATH - NAMES bin/hcc - PATHS - ENV ROCM_DIR - ${ROCM_ROOT_DIR} - /opt/rocm - DOC "Path to ROCm hcc executable") - - -if(ROCM_PATH) - message(STATUS "ROCM_PATH: ${ROCM_PATH}") - set(CMAKE_CXX_COMPILER_ID "HCC") - - set(ROCM_FOUND TRUE) - -else() - set(ROCM_FOUND FALSE) - message(WARNING "ROCm hcc executable not found") -endif() diff --git a/cmake/thirdparty/SetupCUDA.cmake b/cmake/thirdparty/SetupCUDA.cmake index 0c11219c8..7dc73d224 100644 --- a/cmake/thirdparty/SetupCUDA.cmake +++ b/cmake/thirdparty/SetupCUDA.cmake @@ -67,6 +67,17 @@ if (NOT ENABLE_CLANG_CUDA) enable_language(CUDA) endif () +if(CMAKE_CUDA_STANDARD STREQUAL "17") + if(NOT DEFINED CMAKE_CUDA_COMPILE_FEATURES OR (NOT "cuda_std_17" IN_LIST CMAKE_CUDA_COMPILE_FEATURES)) + message(FATAL_ERROR "CMake's CUDA_STANDARD does not support C++17.") + endif() +endif() + +if(CMAKE_CUDA_STANDARD STREQUAL "20") + if(NOT DEFINED CMAKE_CUDA_COMPILE_FEATURES OR (NOT "cuda_std_20" IN_LIST CMAKE_CUDA_COMPILE_FEATURES)) + message(FATAL_ERROR "CMake's CUDA_STANDARD does not support C++20.") + endif() +endif() ############################################################ # Map Legacy FindCUDA variables to native cmake variables @@ -82,8 +93,8 @@ if (CUDA_LINK_WITH_NVCC) # executable link dependencies so the device link fails if there are any missing CUDA library dependencies. Since # we are doing a link with the nvcc compiler, the device link step is unnecessary . # Frustratingly, nvcc-link errors out if you pass it an empty file, so we have to first compile the empty file. - set(CMAKE_CUDA_DEVICE_LINK_LIBRARY "touch .cu ; ${CMAKE_CUDA_COMPILER} -std=c++11 -dc .cu -o ") - set(CMAKE_CUDA_DEVICE_LINK_EXECUTABLE "touch .cu ; ${CMAKE_CUDA_COMPILER} -std=c++11 -dc .cu -o ") + set(CMAKE_CUDA_DEVICE_LINK_LIBRARY "touch .cu ; ${CMAKE_CUDA_COMPILER} -std=c++${CMAKE_CUDA_STANDARD} -dc .cu -o ") + set(CMAKE_CUDA_DEVICE_LINK_EXECUTABLE "touch .cu ; ${CMAKE_CUDA_COMPILER} -std=c++${CMAKE_CUDA_STANDARD} -dc .cu -o ") endif() # If CUDA_TOOLKIT_ROOT_DIR is not set, it should be set by find_package(CUDA) diff --git a/cmake/thirdparty/SetupHCC.cmake b/cmake/thirdparty/SetupHCC.cmake deleted file mode 100644 index 4897bbe64..000000000 --- a/cmake/thirdparty/SetupHCC.cmake +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and -# other BLT Project Developers. See the top-level LICENSE file for details -# -# SPDX-License-Identifier: (BSD-3-Clause) - -################################ -# ROCM -################################ - -if (ENABLE_HCC) - set (CMAKE_MODULE_PATH "${BLT_ROOT_DIR}/cmake/thirdparty;${CMAKE_MODULE_PATH}") - find_package(ROCm REQUIRED) - - if (ROCM_FOUND) - message(STATUS "ROCM Compile Flags: ${ROCM_CXX_COMPILE_FLAGS}") - message(STATUS "ROCM Include Path: ${ROCM_INCLUDE_PATH}") - message(STATUS "ROCM Link Flags: ${ROCM_CXX_LINK_FLAGS}") - message(STATUS "ROCM Libraries: ${ROCM_CXX_LIBRARIES}") - message(STATUS "ROCM Device Arch: ${ROCM_ARCH}") - - if (ENABLE_FORTRAN) - message(ERROR "ROCM does not support Fortran at this time") - endif() - else() - message(ERROR "ROCM Executable not found") - endif() -endif() - - - -# register ROCM with blt -blt_register_library(NAME rocm - INCLUDES ${ROCM_CXX_INCLUDE_PATH} - LIBRARIES ${ROCM_CXX_LIBRARIES} - COMPILE_FLAGS ${ROCM_CXX_COMPILE_FLAGS} - LINK_FLAGS ${ROCM_CXX_LINK_FLAGS} - DEFINES USE_ROCM) - - diff --git a/cmake/thirdparty/SetupHIP.cmake b/cmake/thirdparty/SetupHIP.cmake index 6261febf1..152342047 100644 --- a/cmake/thirdparty/SetupHIP.cmake +++ b/cmake/thirdparty/SetupHIP.cmake @@ -44,8 +44,14 @@ if(DEFINED AMDGPU_TARGETS) foreach(_target ${AMDGPU_TARGETS}) if (NOT "${CMAKE_HIP_ARCHITECTURES}" MATCHES "${_target}") - list(REMOVE_ITEM _hip_compile_options "--offload-arch=${_target}") - list(REMOVE_ITEM _hip_link_libs "--offload-arch=${_target}") + set(_flag "--offload-arch=${_target}") + set(_generator_compile_flag "$<$:SHELL:${_flag}>") + set(_generator_link_flag "$<$:${_flag}>") + + list(REMOVE_ITEM _hip_compile_options ${_generator_compile_flag}) + list(REMOVE_ITEM _hip_compile_options ${_flag}) + list(REMOVE_ITEM _hip_link_libs ${_generator_link_flag}) + list(REMOVE_ITEM _hip_link_libs ${_flag}) endif() endforeach() diff --git a/cmake/thirdparty/SetupThirdParty.cmake b/cmake/thirdparty/SetupThirdParty.cmake index 38c09ab1f..c8e6e27ed 100644 --- a/cmake/thirdparty/SetupThirdParty.cmake +++ b/cmake/thirdparty/SetupThirdParty.cmake @@ -56,14 +56,6 @@ if (ENABLE_HIP) include(${BLT_ROOT_DIR}/cmake/thirdparty/SetupHIP.cmake) endif() -#------------------------------------ -# HCC -#------------------------------------ -message(STATUS "HCC Support is ${ENABLE_HCC}") -if (ENABLE_HCC) - include(${BLT_ROOT_DIR}/cmake/thirdparty/SetupHCC.cmake) -endif() - #------------------------------------ # Documentation Packages diff --git a/docs/api/utility.rst b/docs/api/utility.rst index 72b048d4d..85753f2cf 100644 --- a/docs/api/utility.rst +++ b/docs/api/utility.rst @@ -55,7 +55,6 @@ blt_append_custom_compiler_flag DEFAULT defaultFlag (optional) GNU gnuFlag (optional) CLANG clangFlag (optional) - HCC hccFlag (optional) INTEL intelFlag (optional) INTELLLVM intelLLVMFlag (optional) XL xlFlag (optional) diff --git a/docs/tutorial/portable_compiler_flags.rst b/docs/tutorial/portable_compiler_flags.rst index 5ac671553..be3fc641a 100644 --- a/docs/tutorial/portable_compiler_flags.rst +++ b/docs/tutorial/portable_compiler_flags.rst @@ -27,7 +27,6 @@ which allows users to easily place a compiler dependent flag into a CMake variab * MSVC (Microsoft Visual Studio) * MSVC_INTEL (Intel toolchain in Microsoft Visual Studio) * PGI - * HCC (AMD GPU) Here is an example for setting the appropriate flag to treat warnings as errors: diff --git a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_link_with_nvcc.cmake b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_link_with_nvcc.cmake index 86f5dc0f3..be9a065d6 100644 --- a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_link_with_nvcc.cmake +++ b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_link_with_nvcc.cmake @@ -58,15 +58,16 @@ set(BLT_OPENMP_LINK_FLAGS "-Xlinker -rpath -Xlinker ${OMP_HOME}/lib -L${OMP_HOME #------------------------------------------------------------------------------ set(ENABLE_CUDA ON CACHE BOOL "") -set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-10.1.168" CACHE PATH "") +set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.1.1" CACHE PATH "") set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "") set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE PATH "") -set (_cuda_arch "sm_70") -set (CMAKE_CUDA_FLAGS "-restrict -arch ${_cuda_arch} -std=c++11 --expt-extended-lambda -G" CACHE STRING "" ) +set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "") +set(_cuda_arch "sm_${CMAKE_CUDA_ARCHITECTURES}") +set(CMAKE_CUDA_FLAGS "-restrict -arch ${_cuda_arch} --expt-extended-lambda -G" CACHE STRING "" ) -set (CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "" ) -set (CUDA_LINK_WITH_NVCC ON CACHE BOOL "") +set(CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "" ) +set(CUDA_LINK_WITH_NVCC ON CACHE BOOL "") # nvcc does not like gtest's 'pthreads' flag set(gtest_disable_pthreads ON CACHE BOOL "") diff --git a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++14-export-tpls.cmake b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++14-export-tpls.cmake index 310b9cbbf..8563e153c 100644 --- a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++14-export-tpls.cmake +++ b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++14-export-tpls.cmake @@ -48,7 +48,7 @@ set(MPI_CXX_COMPILER "${_MPI_BASE_DIR}/bin/mpicxx" CACHE PATH "") set(ENABLE_CUDA ON CACHE BOOL "") -set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.0.182" CACHE PATH "") +set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.1.1" CACHE PATH "") set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "") set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "") diff --git a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++17.cmake b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++17.cmake index 4627f67f8..b29b15af8 100644 --- a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++17.cmake +++ b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++17.cmake @@ -48,14 +48,14 @@ set(MPI_CXX_COMPILER "${_MPI_BASE_DIR}/bin/mpicxx" CACHE PATH "") set(ENABLE_CUDA ON CACHE BOOL "") -set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.0.182" CACHE PATH "") +set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.1.1" CACHE PATH "") set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "") set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "") set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "") set(_cuda_arch "sm_${CMAKE_CUDA_ARCHITECTURES}") -set(CMAKE_CUDA_FLAGS "-Xcompiler=--gcc-toolchain=${_GCC_DIR} -restrict -arch ${_cuda_arch} -std=c++17 --expt-extended-lambda -G" CACHE STRING "") +set(CMAKE_CUDA_FLAGS "-Xcompiler=--gcc-toolchain=${_GCC_DIR} -restrict -arch ${_cuda_arch} --expt-extended-lambda -G" CACHE STRING "") set(CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "" ) diff --git a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++17_no_separable.cmake b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++17_no_separable.cmake index b8f52a254..ccc031b5e 100644 --- a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++17_no_separable.cmake +++ b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_c++17_no_separable.cmake @@ -48,14 +48,14 @@ set(MPI_CXX_COMPILER "${_MPI_BASE_DIR}/bin/mpicxx" CACHE PATH "") set(ENABLE_CUDA ON CACHE BOOL "") -set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.0.182" CACHE PATH "") +set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.1.1" CACHE PATH "") set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "") set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "") set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "") set(_cuda_arch "sm_${CMAKE_CUDA_ARCHITECTURES}") -set(CMAKE_CUDA_FLAGS "-Xcompiler=--gcc-toolchain=${_GCC_DIR} -restrict -arch ${_cuda_arch} -std=c++17 --expt-extended-lambda -G" CACHE STRING "") +set(CMAKE_CUDA_FLAGS "-Xcompiler=--gcc-toolchain=${_GCC_DIR} -restrict -arch ${_cuda_arch} --expt-extended-lambda -G" CACHE STRING "") # nvcc does not like gtest's 'pthreads' flag set(gtest_disable_pthreads ON CACHE BOOL "") diff --git a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_xlf.cmake b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_xlf.cmake index 55234cb47..90f78288f 100644 --- a/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_xlf.cmake +++ b/host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@upstream_nvcc_xlf.cmake @@ -63,13 +63,13 @@ set(BLT_FORTRAN_FLAGS "-WF,-C!" CACHE PATH "Converts C-style comments to Fortran #_blt_tutorial_useful_cuda_flags_start set(ENABLE_CUDA ON CACHE BOOL "") -set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-10.1.168" CACHE PATH "") +set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.1.1" CACHE PATH "") set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "") set(CMAKE_CUDA_HOST_COMPILER "${MPI_CXX_COMPILER}" CACHE PATH "") set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "") set(_cuda_arch "sm_${CMAKE_CUDA_ARCHITECTURES}") -set(CMAKE_CUDA_FLAGS "-restrict -arch ${_cuda_arch} -std=c++11 --expt-extended-lambda -G" CACHE STRING "") +set(CMAKE_CUDA_FLAGS "-restrict -arch ${_cuda_arch} --expt-extended-lambda -G" CACHE STRING "") set(CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "" ) diff --git a/host-configs/llnl/blueos_3_ppc64le_ib_p9/pgi@20.4_nvcc.cmake b/host-configs/llnl/blueos_3_ppc64le_ib_p9/pgi@20.4_nvcc.cmake index 7820ecb0b..5750e57fc 100644 --- a/host-configs/llnl/blueos_3_ppc64le_ib_p9/pgi@20.4_nvcc.cmake +++ b/host-configs/llnl/blueos_3_ppc64le_ib_p9/pgi@20.4_nvcc.cmake @@ -15,7 +15,7 @@ set(ENABLE_OPENMP OFF CACHE BOOL "") set(ENABLE_CUDA ON CACHE BOOL "") -set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-10.1.243" CACHE PATH "") +set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.1.1" CACHE PATH "") set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "") set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE PATH "") diff --git a/host-configs/llnl/toss_4_x86_64_ib/rocm@4.5.2_hip.cmake b/host-configs/llnl/toss_4_x86_64_ib/rocm@4.5.2_hip.cmake new file mode 100644 index 000000000..8d17a68ee --- /dev/null +++ b/host-configs/llnl/toss_4_x86_64_ib/rocm@4.5.2_hip.cmake @@ -0,0 +1,43 @@ +# Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and +# other BLT Project Developers. See the top-level LICENSE file for details +# +# SPDX-License-Identifier: (BSD-3-Clause) + +#------------------------------------------------------------------------------ +# Example host-config file for the corona cluster at LLNL +#------------------------------------------------------------------------------ +# This file provides CMake with paths / details for: +# C,C++, & Fortran compilers + MPI & HIP +# using ROCM compilers +#------------------------------------------------------------------------------ + +#------------------------------------------------------------------------------ +# ROCM@4.5.2 compilers +#------------------------------------------------------------------------------ +set(ROCM_HOME "/usr/tce/packages/rocmcc-tce/rocmcc-4.5.2") +set(CMAKE_C_COMPILER "${ROCM_HOME}/bin/amdclang" CACHE PATH "") +set(CMAKE_CXX_COMPILER "${ROCM_HOME}/bin/hipcc" CACHE PATH "") + +# Fortran support +set(ENABLE_FORTRAN ON CACHE BOOL "") +set(CMAKE_Fortran_COMPILER "${ROCM_HOME}/bin/amdflang" CACHE PATH "") + +#------------------------------------------------------------------------------ +# MPI Support +#------------------------------------------------------------------------------ +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPI_HOME "/usr/tce/packages/mvapich2-tce/mvapich2-2.3.6-rocmcc-4.5.2/") +set(MPI_C_COMPILER "${MPI_HOME}/bin/mpicc" CACHE PATH "") +set(MPI_CXX_COMPILER "${MPI_HOME}/bin/mpicxx" CACHE PATH "") +set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpif90" CACHE PATH "") + +#------------------------------------------------------------------------------ +# HIP support +#------------------------------------------------------------------------------ +set(ENABLE_HIP ON CACHE BOOL "") +set(ROCM_PATH "/opt/rocm-4.5.2/" CACHE PATH "") +set(CMAKE_HIP_ARCHITECTURES "gfx906" CACHE STRING "") + +# Recommended link line when not using tce-wrapped compilers +# set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags -L/opt/rocm-4.5.2/hip/lib -L/opt/rocm-4.5.2/lib -L/opt/rocm-4.5.2/lib64 -Wl,-rpath,/opt/rocm-4.5.2/hip/lib:/opt/rocm-4.5.2/lib:/opt/rocm-4.5.2/lib64 -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr" CACHE STRING "") diff --git a/host-configs/llnl/toss_4_x86_64_ib_cray/cce@12.0.1_clang_hip.cmake b/host-configs/llnl/toss_4_x86_64_ib_cray/cce@13.0.1_clang_hip.cmake similarity index 78% rename from host-configs/llnl/toss_4_x86_64_ib_cray/cce@12.0.1_clang_hip.cmake rename to host-configs/llnl/toss_4_x86_64_ib_cray/cce@13.0.1_clang_hip.cmake index 1266694a1..c640b6cbb 100644 --- a/host-configs/llnl/toss_4_x86_64_ib_cray/cce@12.0.1_clang_hip.cmake +++ b/host-configs/llnl/toss_4_x86_64_ib_cray/cce@13.0.1_clang_hip.cmake @@ -12,10 +12,10 @@ #------------------------------------------------------------------------------ #------------------------------------------------------------------------------ -# HPE Cray cce@12.0.1 compilers +# HPE Cray cce@13.0.1 compilers #------------------------------------------------------------------------------ # _blt_tutorial_compiler_config_start -set(CCE_HOME "/usr/tce/packages/cce-tce/cce-12.0.1") +set(CCE_HOME "/usr/tce/packages/cce-tce/cce-13.0.1") set(CMAKE_C_COMPILER "${CCE_HOME}/bin/craycc" CACHE PATH "") set(CMAKE_CXX_COMPILER "${CCE_HOME}/bin/crayCC" CACHE PATH "") @@ -30,7 +30,7 @@ set(CMAKE_Fortran_COMPILER "${CCE_HOME}/bin/crayftn" CACHE PATH "") # _blt_tutorial_mpi_config_start set(ENABLE_MPI ON CACHE BOOL "") -set(MPI_HOME "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.7-cce-12.0.1/") +set(MPI_HOME "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.13-cce-13.0.1/") set(MPI_C_COMPILER "${MPI_HOME}/bin/mpicc" CACHE PATH "") set(MPI_CXX_COMPILER "${MPI_HOME}/bin/mpicxx" CACHE PATH "") @@ -44,9 +44,11 @@ set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpif90" CACHE PATH "") # _blt_tutorial_hip_config_start set(ENABLE_HIP ON CACHE BOOL "") -set(ROCM_PATH "/opt/rocm-4.2.0/" CACHE PATH "") -set(HIP_PLATFORM "clang" CACHE STRING "") +set(ROCM_PATH "/opt/rocm-4.5.2/" CACHE PATH "") set(CMAKE_HIP_ARCHITECTURES "gfx908" CACHE STRING "gfx architecture to use when generating HIP/ROCm code") +# Recommended link line when not using tce-wrapped compilers +# set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags -L/opt/rocm-4.5.2/hip/lib -L/opt/rocm-4.5.2/lib -L/opt/rocm-4.5.2/lib64 -Wl,-rpath,/opt/rocm-4.5.2/hip/lib:/opt/rocm-4.5.2/lib:/opt/rocm-4.5.2/lib64 -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr" CACHE STRING "") + # _blt_tutorial_hip_config_end diff --git a/host-configs/llnl/toss_4_x86_64_ib_cray/cce@12.0.3_clang_hip.cmake b/host-configs/llnl/toss_4_x86_64_ib_cray/cce@13.0.1_clang_hip_c++17.cmake similarity index 68% rename from host-configs/llnl/toss_4_x86_64_ib_cray/cce@12.0.3_clang_hip.cmake rename to host-configs/llnl/toss_4_x86_64_ib_cray/cce@13.0.1_clang_hip_c++17.cmake index 7d26ee4e0..224fd5d1d 100644 --- a/host-configs/llnl/toss_4_x86_64_ib_cray/cce@12.0.3_clang_hip.cmake +++ b/host-configs/llnl/toss_4_x86_64_ib_cray/cce@13.0.1_clang_hip_c++17.cmake @@ -1,10 +1,10 @@ -# Copyright (c) 2017-2021, Lawrence Livermore National Security, LLC and +# Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and # other BLT Project Developers. See the top-level LICENSE file for details # # SPDX-License-Identifier: (BSD-3-Clause) #------------------------------------------------------------------------------ -# Example host-config file for the rzwhamo cluster at LLNL +# Example host-config file for the rznevada cluster at LLNL #------------------------------------------------------------------------------ # This file provides CMake with paths / details for: # C,C++, & Fortran compilers + MPI & HIP @@ -12,9 +12,9 @@ #------------------------------------------------------------------------------ #------------------------------------------------------------------------------ -# HPE Cray cce@12.0.3 compilers +# HPE Cray cce@13.0.1 compilers #------------------------------------------------------------------------------ -set(CCE_HOME "/usr/tce/packages/cce-tce/cce-12.0.3") +set(CCE_HOME "/usr/tce/packages/cce-tce/cce-13.0.1") set(CMAKE_C_COMPILER "${CCE_HOME}/bin/craycc" CACHE PATH "") set(CMAKE_CXX_COMPILER "${CCE_HOME}/bin/crayCC" CACHE PATH "") @@ -22,12 +22,15 @@ set(CMAKE_CXX_COMPILER "${CCE_HOME}/bin/crayCC" CACHE PATH "") set(ENABLE_FORTRAN ON CACHE BOOL "") set(CMAKE_Fortran_COMPILER "${CCE_HOME}/bin/crayftn" CACHE PATH "") +# Set compiler standard for C++ and HIP +set(BLT_CXX_STD "c++17" CACHE STRING "") + #------------------------------------------------------------------------------ # MPI Support #------------------------------------------------------------------------------ set(ENABLE_MPI ON CACHE BOOL "") -set(MPI_HOME "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.8-cce-12.0.3/") +set(MPI_HOME "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.13-cce-13.0.1/") set(MPI_C_COMPILER "${MPI_HOME}/bin/mpicc" CACHE PATH "") set(MPI_CXX_COMPILER "${MPI_HOME}/bin/mpicxx" CACHE PATH "") @@ -39,6 +42,8 @@ set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpif90" CACHE PATH "") #------------------------------------------------------------------------------ set(ENABLE_HIP ON CACHE BOOL "") -set(ROCM_PATH "/opt/rocm-4.5.0/" CACHE PATH "") -set(HIP_PLATFORM "clang" CACHE STRING "") +set(ROCM_PATH "/opt/rocm-4.5.2/" CACHE PATH "") set(CMAKE_HIP_ARCHITECTURES "gfx908" CACHE STRING "gfx architecture to use when generating HIP/ROCm code") + +# Recommended link line when not using tce-wrapped compilers +# set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags -L/opt/rocm-4.5.2/hip/lib -L/opt/rocm-4.5.2/lib -L/opt/rocm-4.5.2/lib64 -Wl,-rpath,/opt/rocm-4.5.2/hip/lib:/opt/rocm-4.5.2/lib:/opt/rocm-4.5.2/lib64 -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr" CACHE STRING "") diff --git a/host-configs/llnl/toss_4_x86_64_ib_cray/rocm@4.2.0_hip.cmake b/host-configs/llnl/toss_4_x86_64_ib_cray/rocm@4.5.2_hip.cmake similarity index 69% rename from host-configs/llnl/toss_4_x86_64_ib_cray/rocm@4.2.0_hip.cmake rename to host-configs/llnl/toss_4_x86_64_ib_cray/rocm@4.5.2_hip.cmake index 3b330a46c..11a5b0978 100644 --- a/host-configs/llnl/toss_4_x86_64_ib_cray/rocm@4.2.0_hip.cmake +++ b/host-configs/llnl/toss_4_x86_64_ib_cray/rocm@4.5.2_hip.cmake @@ -12,23 +12,22 @@ #------------------------------------------------------------------------------ #------------------------------------------------------------------------------ -# ROCM@4.2.0 compilers +# ROCM@4.5.2 compilers #------------------------------------------------------------------------------ -set(ROCM_HOME "/usr/tce/packages/rocmcc-tce/rocmcc-4.2.0") +set(ROCM_HOME "/usr/tce/packages/rocmcc-tce/rocmcc-4.5.2-cce-13.0.1") set(CMAKE_C_COMPILER "${ROCM_HOME}/bin/amdclang" CACHE PATH "") set(CMAKE_CXX_COMPILER "${ROCM_HOME}/bin/hipcc" CACHE PATH "") # Fortran support -set(CCE_HOME "/usr/tce/packages/cce-tce/cce-12.0.1") set(ENABLE_FORTRAN ON CACHE BOOL "") -set(CMAKE_Fortran_COMPILER "${CCE_HOME}/bin/crayftn" CACHE PATH "") +set(CMAKE_Fortran_COMPILER "${ROCM_HOME}/bin/crayftn" CACHE PATH "") #------------------------------------------------------------------------------ # MPI Support #------------------------------------------------------------------------------ set(ENABLE_MPI ON CACHE BOOL "") -set(MPI_HOME "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.7-rocmcc-4.2.0") +set(MPI_HOME "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.13-rocmcc-4.5.2-cce-13.0.1") set(MPI_C_COMPILER "${MPI_HOME}/bin/mpicc" CACHE PATH "") set(MPI_CXX_COMPILER "${MPI_HOME}/bin/mpicxx" CACHE PATH "") set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpif90" CACHE PATH "") @@ -37,5 +36,8 @@ set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpif90" CACHE PATH "") # HIP support #------------------------------------------------------------------------------ set(ENABLE_HIP ON CACHE BOOL "") -set(ROCM_PATH "/opt/rocm-4.2.0/" CACHE PATH "") -set(CMAKE_CUDA_ARCHITECTURES "gfx908" CACHE STRING "") +set(ROCM_PATH "/opt/rocm-4.5.2/" CACHE PATH "") +set(CMAKE_HIP_ARCHITECTURES "gfx908" CACHE STRING "") + +# Recommended link line when not using tce-wrapped compilers +# set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags -L/opt/rocm-4.5.2/hip/lib -L/opt/rocm-4.5.2/lib -L/opt/rocm-4.5.2/lib64 -Wl,-rpath,/opt/rocm-4.5.2/hip/lib:/opt/rocm-4.5.2/lib:/opt/rocm-4.5.2/lib64 -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr" CACHE STRING "") diff --git a/host-configs/other/hcc.cmake b/host-configs/other/hcc.cmake deleted file mode 100644 index c3e2f564d..000000000 --- a/host-configs/other/hcc.cmake +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and -# other BLT Project Developers. See the top-level LICENSE file for details -# -# SPDX-License-Identifier: (BSD-3-Clause) - -# The CMake command line to use this host config file must have certain -# defines enabled. Here is an example script for an out-of-place build of the -# tutorial blank project template: -## hcc.sh -##!/bin/bash -#rm -rf build-hcc-release 2>/dev/null -#mkdir build-hcc-release && cd build-hcc-release - -#BLT_DIR=$(git rev-parse --show-toplevel) - -#cmake \ -# -DCMAKE_BUILD_TYPE=Release \ -# -DENABLE_HCC=ON -DBLT_SOURCE_DIR=${BLT_DIR} \ -# -DROCM_ARCH=gfx900 \ -# -C ${BLT_DIR}/host-configs/hcc.cmake \ -# -DCMAKE_INSTALL_PREFIX=../install-hcc-release \ -# "$@" \ -# ${BLT_DIR}/docs/tutorial/blank_project - - -########################################################### -# standard ROCm HCC compiler -########################################################### - -set(ENABLE_HCC ON CACHE BOOL "") -set(ENABLE_OPENMP OFF CACHE BOOL "") - -set(ROCM_ROOT_DIR "/opt/rocm" CACHE PATH "ROCm ROOT directory path") - -if ( IS_DIRECTORY "${ROCM_ROOT_DIR}/hcc/include" ) # this path only exists on older rocm installs - set(ROCM_INCLUDE_PATH "${ROCM_ROOT_DIR}/hcc/include;${ROCM_ROOT_DIR}/include" CACHE PATH "") -else() - set(ROCM_INCLUDE_PATH "${ROCM_ROOT_DIR}/include" CACHE PATH "") -endif() -set(ROCM_CXX_LIBRARIES "-L${ROCM_ROOT_DIR}/lib -lhc_am -lhip_hcc" CACHE STRING "") - -########################################################### -# specify the target architecture -# Default with ROCm 1.7 is gfx803 (Fiji) -# Other options: -# gfx700 Hawaii -# gfx803 Polaris (RX580) -# gfx900 Vega -# gfx901 -########################################################### -set(ROCM_ARCH_FLAG "-amdgpu-target=${ROCM_ARCH}" CACHE STRING "") - -########################################################### -# get compile/link flags from hcc-config -########################################################### -# execute_process(COMMAND ${ROCM_ROOT_DIR}/hcc/bin/hcc-config --cxxflags OUTPUT_VARIABLE ROCM_CXX_COMPILE_FLAGS) -# execute_process(COMMAND ${ROCM_ROOT_DIR}/hcc/bin/hcc-config --ldflags OUTPUT_VARIABLE ROCM_CXX_LINK_FLAGS) -set(ROCM_CXX_COMPILE_FLAGS "-hc;-std=c++amp") -set(ROCM_CXX_LINK_FLAGS "-hc -std=c++amp") - -set(ROCM_CXX_COMPILE_FLAGS "${ROCM_CXX_COMPILE_FLAGS};-Wno-unused-parameter;-DHCC_ENABLE_ACCELERATOR_PRINTF" CACHE STRING "") -set(ROCM_CXX_LINK_FLAGS "${ROCM_CXX_LINK_FLAGS} ${ROCM_ARCH_FLAG} ${ROCM_CXX_LIBRARIES}" CACHE STRING "") - -########################################################### -# set CMake cache variables -########################################################### -# if hcc does not exist, consider setting the compiler to "${ROCM_ROOT_DIR}/bin/hipcc" -set(CMAKE_CXX_COMPILER "${ROCM_ROOT_DIR}/hcc/bin/hcc" CACHE FILEPATH "ROCm HCC compiler") -# set(BLT_CXX_FLAGS "${ROCM_CXX_COMPILE_FLAGS}" CACHE STRING "HCC compiler flags") - -set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_COMPILER} ${ROCM_CXX_LINK_FLAGS} -o " CACHE STRING "HCC linker command line") diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index e69e0792b..3bcacf7d2 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -285,8 +285,6 @@ if(CLANGFORMAT_FOUND) ../smoke/blt_gbenchmark_smoke.cpp ../smoke/blt_gmock_smoke.cpp ../smoke/blt_gtest_smoke.cpp - ../smoke/blt_hcc_runtime_smoke.cpp - ../smoke/blt_hcc_smoke.cpp ../smoke/blt_hip_runtime_smoke.cpp ../smoke/blt_hip_gtest_smoke.cpp ../smoke/blt_hip_smoke.cpp diff --git a/tests/internal/unit/CMakeLists.txt b/tests/internal/unit/CMakeLists.txt index bd4f8b8c6..edcad9ae2 100644 --- a/tests/internal/unit/CMakeLists.txt +++ b/tests/internal/unit/CMakeLists.txt @@ -117,11 +117,12 @@ macro(compare_source_lists) list(SORT sorted_a_list) list(SORT sorted_b_list) - foreach(i RANGE ${a_len}) + math(EXPR _range_stop "${a_len} - 1") + foreach(i RANGE 0 ${_range_stop}) set(a_item) set(b_item) - list(GET sorted_a_list i a_item) - list(GET sorted_b_list i b_item) + list(GET sorted_a_list ${i} a_item) + list(GET sorted_b_list ${i} b_item) if(NOT a_item STREQUAL b_item) message(FATAL_ERROR "Split source test failed. ${a_item} != ${b_item}") diff --git a/tests/smoke/CMakeLists.txt b/tests/smoke/CMakeLists.txt index 265f06976..266461591 100644 --- a/tests/smoke/CMakeLists.txt +++ b/tests/smoke/CMakeLists.txt @@ -185,30 +185,6 @@ if (ENABLE_CUDA) endif() endif() -################ -# HCC test -################ -if (ENABLE_HCC) - blt_add_executable(NAME blt_hcc_smoke - SOURCES blt_hcc_smoke.cpp - OUTPUT_DIR ${TEST_OUTPUT_DIRECTORY} - DEPENDS_ON rocm - FOLDER blt/tests ) - - blt_add_test(NAME blt_hcc_smoke - COMMAND blt_hcc_smoke) - - blt_add_executable(NAME blt_hcc_runtime_smoke - SOURCES blt_hcc_runtime_smoke.cpp - OUTPUT_DIR ${TEST_OUTPUT_DIRECTORY} - DEPENDS_ON rocm - FOLDER blt/tests ) - - blt_add_test(NAME blt_hcc_runtime_smoke - COMMAND blt_hcc_runtime_smoke) - - -endif() ################ # HIP test diff --git a/tests/smoke/blt_hcc_runtime_smoke.cpp b/tests/smoke/blt_hcc_runtime_smoke.cpp deleted file mode 100644 index f9eea6e1f..000000000 --- a/tests/smoke/blt_hcc_runtime_smoke.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and -// other BLT Project Developers. See the top-level LICENSE file for details -// -// SPDX-License-Identifier: (BSD-3-Clause) - -//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// -// Note: This is a ROCm example from AMD: -//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// - -//----------------------------------------------------------------------------- -// -// file: blt_hcc_runtime_smoke.cpp -// -//----------------------------------------------------------------------------- - -#include -#include -#include "hc.hpp" - -int main() -{ - using namespace hc; - accelerator acc; - std::vector accv = acc.get_all() ; - - std::cout << "Found " << accv.size() << " accelerators." << std::endl; - std::cout << std::endl; - - unsigned long idefault = 0; - for(unsigned long i=0; i< accv.size(); i++) - { - accelerator a = accv[i]; - std::cout << "Accelerator " << i << ": " ; - std::wcout << a.get_device_path() << L" : " << a.get_description(); - std::cout << " : " << (a.get_version()>>16) << "." - << (a.get_version()&0xff); - std::cout << std::endl; - if (a == acc) { idefault = i; } - } - - std::cout << std::endl; - std::cout << "Default Accelerator " << ": " << idefault << " : " ; - std::wcout << acc.get_device_path() << std::endl; - - return 0; -} - diff --git a/tests/smoke/blt_hcc_smoke.cpp b/tests/smoke/blt_hcc_smoke.cpp deleted file mode 100644 index fdae9d013..000000000 --- a/tests/smoke/blt_hcc_smoke.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and -// other BLT Project Developers. See the top-level LICENSE file for details -// -// SPDX-License-Identifier: (BSD-3-Clause) - -//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// -// Note: This is a ROCM Hello world example from AMD: -//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// - -//----------------------------------------------------------------------------- -// -// file: blt_hcc_smoke.cpp -// -//----------------------------------------------------------------------------- - -#include -#include - -int main() -{ - hc::parallel_for_each(hc::extent<1>(1), []() [[hc]] - { - hc::printf("Accelerator: Hello World!\n"); - }).wait(); - - return 0; -} - - diff --git a/tests/smoke/blt_hip_gtest_smoke.cpp b/tests/smoke/blt_hip_gtest_smoke.cpp index d8ccb0435..76172e76c 100644 --- a/tests/smoke/blt_hip_gtest_smoke.cpp +++ b/tests/smoke/blt_hip_gtest_smoke.cpp @@ -30,6 +30,9 @@ TEST(blt_hip_gtest_smoke,basic_assert_example) int num_threads = STR_LENGTH; int num_blocks = 1; hipLaunchKernelGGL((hello), dim3(num_blocks), dim3(num_threads),0,0); - hipDeviceSynchronize(); + if(hipSuccess != hipDeviceSynchronize()) + { + std::cout << "ERROR: hipDeviceSynchronize failed!" << std::endl; + } EXPECT_TRUE( true ); } diff --git a/tests/smoke/blt_hip_runtime_smoke.cpp b/tests/smoke/blt_hip_runtime_smoke.cpp index 65a1e771a..a7ca79fc6 100644 --- a/tests/smoke/blt_hip_runtime_smoke.cpp +++ b/tests/smoke/blt_hip_runtime_smoke.cpp @@ -17,11 +17,18 @@ int main() { int nDevices; - hipGetDeviceCount(&nDevices); + if(hipSuccess != hipGetDeviceCount(&nDevices)) + { + std::cout << "ERROR: hipGetDeviceCount failed!" << std::endl; + } + for (int i = 0; i < nDevices; i++) { hipDeviceProp_t prop; - hipGetDeviceProperties(&prop, i); + if(hipSuccess != hipGetDeviceProperties(&prop, i)) + { + std::cout << "ERROR: hipGetDeviceProperties failed!" << std::endl; + } printf("Device Number: %d\n", i); printf(" Device name: %s\n", prop.name); printf(" Memory Clock Rate (KHz): %d\n", diff --git a/tests/smoke/blt_hip_smoke.cpp b/tests/smoke/blt_hip_smoke.cpp index 3a0cd7f2c..b94eb152b 100644 --- a/tests/smoke/blt_hip_smoke.cpp +++ b/tests/smoke/blt_hip_smoke.cpp @@ -26,7 +26,10 @@ int main() int num_threads = STR_LENGTH; int num_blocks = 1; hipLaunchKernelGGL((hello), dim3(num_blocks), dim3(num_threads),0,0); - hipDeviceSynchronize(); + if(hipSuccess != hipDeviceSynchronize()) + { + std::cout << "ERROR: hipDeviceSynchronize failed!" << std::endl; + } return 0; }