From 7a303c94d5c2ee2000a2475932241b95b55b8edf Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 13 Sep 2022 15:13:53 -0700 Subject: [PATCH 1/5] CMake: HIP_PATH from ROCM_PATH On machines like Crusher, `ROCM_PATH` is more likely to be available then a `HIP_PATH` environment variable. This is mainly needed for our hacky ROCTX hints. --- Tools/CMake/AMReXParallelBackends.cmake | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Tools/CMake/AMReXParallelBackends.cmake b/Tools/CMake/AMReXParallelBackends.cmake index b249d28ef60..b1a73c7c659 100644 --- a/Tools/CMake/AMReXParallelBackends.cmake +++ b/Tools/CMake/AMReXParallelBackends.cmake @@ -198,10 +198,12 @@ if (AMReX_HIP) unset(_valid_hip_compilers) if(NOT DEFINED HIP_PATH) - if(NOT DEFINED ENV{HIP_PATH}) - set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed") - else() + if(DEFINED ENV{HIP_PATH}) set(HIP_PATH $ENV{HIP_PATH} CACHE PATH "Path to which HIP has been installed") + elseif(DEFINED ENV{ROCM_PATH}) + set(HIP_PATH "$ENV{ROCM_PATH}/hip" CACHE PATH "Path to which HIP has been installed") + else() + set(HIP_PATH "/opt/rocm/hip" CACHE PATH "Path to which HIP has been installed") endif() endif() From d86764796ce0ac45a8bd36776de8f695a2276030 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 13 Sep 2022 15:19:15 -0700 Subject: [PATCH 2/5] CI: Build With ROCTX --- .github/workflows/hip.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 0672287437c..47d9a89828e 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -47,6 +47,7 @@ jobs: -DAMReX_LINEAR_SOLVERS=ON \ -DAMReX_GPU_BACKEND=HIP \ -DAMReX_AMD_ARCH=gfx908 \ + -DAMReX_ROCTX=ON \ -DCMAKE_C_COMPILER=$(which clang) \ -DCMAKE_CXX_COMPILER=$(which clang++) \ -DCMAKE_Fortran_COMPILER=$(which flang) \ @@ -91,6 +92,7 @@ jobs: -DAMReX_LINEAR_SOLVERS=ON \ -DAMReX_GPU_BACKEND=HIP \ -DAMReX_AMD_ARCH=gfx908 \ + -DAMReX_ROCTX=ON \ -DCMAKE_C_COMPILER=$(which clang) \ -DCMAKE_CXX_COMPILER=$(which hipcc) \ -DCMAKE_Fortran_COMPILER=$(which gfortran) \ From f3616174cc260938c8253088c2b1912f20b2b9fa Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 13 Sep 2022 15:23:45 -0700 Subject: [PATCH 3/5] ROCTX: New Include Supposedly, there is a new include we shall use: Ref.: https://github.com/ROCm-Developer-Tools/roctracer/issues/79 --- Src/Base/AMReX_GpuDevice.cpp | 4 ++-- Src/Base/AMReX_TinyProfiler.H | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Src/Base/AMReX_GpuDevice.cpp b/Src/Base/AMReX_GpuDevice.cpp index e0ab64b76e3..fe7257ea971 100644 --- a/Src/Base/AMReX_GpuDevice.cpp +++ b/Src/Base/AMReX_GpuDevice.cpp @@ -22,9 +22,9 @@ #if defined(AMREX_USE_HIP) #include #if defined(AMREX_USE_ROCTX) -#include +#include #if defined(AMREX_PROFILING) || defined (AMREX_TINY_PROFILING) -#include +#include #endif #endif #endif diff --git a/Src/Base/AMReX_TinyProfiler.H b/Src/Base/AMReX_TinyProfiler.H index 677b4448d3b..57c9ea0479c 100644 --- a/Src/Base/AMReX_TinyProfiler.H +++ b/Src/Base/AMReX_TinyProfiler.H @@ -10,7 +10,7 @@ #endif #if defined(AMREX_USE_HIP) && defined(AMREX_USE_ROCTX) -#include +#include #endif #include From 0a536ccde4ffe244e47b39b54ec3d4a20f0cdd9a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 13 Sep 2022 16:48:45 -0700 Subject: [PATCH 4/5] ROCtracer: Include as System library Because of GNU extensions in the roctracer include files for the legacy include. But we should make this `-isystem` anyway to be robust for the future. The 5.2 deprecated include file `` throws warnings because they rely on GNU extensions: ``` In file included from /opt/rocm/hip/../roctracer/include/ext/prof_protocol.h:27: /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:70:7: warning: anonymous structs are a GNU extension [-Wgnu-anonymous-struct] struct { ^ /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:70:7: warning: anonymous types declared in an anonymous union are an extension [-Wnested-anon-types] /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:75:7: warning: anonymous types declared in an anonymous union are an extension [-Wnested-anon-types] struct { ^ /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:82:7: warning: anonymous structs are a GNU extension [-Wgnu-anonymous-struct] struct { ^ /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:86:7: warning: anonymous structs are a GNU extension [-Wgnu-anonymous-struct] struct { ^ /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:90:7: warning: anonymous structs are a GNU extension [-Wgnu-anonymous-struct] struct { ^ /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:82:7: warning: anonymous types declared in an anonymous union are an extension [-Wnested-anon-types] struct { ^ /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:86:7: warning: anonymous types declared in an anonymous union are an extension [-Wnested-anon-types] struct { ^ /opt/rocm/hip/../roctracer/include/ext/../../../include/roctracer/ext/prof_protocol.h:90:7: warning: anonymous types declared in an anonymous union are an extension [-Wnested-anon-types] struct { ^ ``` --- Tools/CMake/AMReXParallelBackends.cmake | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Tools/CMake/AMReXParallelBackends.cmake b/Tools/CMake/AMReXParallelBackends.cmake index b1a73c7c659..61b563f7c51 100644 --- a/Tools/CMake/AMReXParallelBackends.cmake +++ b/Tools/CMake/AMReXParallelBackends.cmake @@ -257,9 +257,15 @@ if (AMReX_HIP) if(AMReX_ROCTX) # To be modernized in the future, please see: # https://github.com/ROCm-Developer-Tools/roctracer/issues/56 - target_include_directories(amrex PUBLIC ${HIP_PATH}/../roctracer/include ${HIP_PATH}/../rocprofiler/include) - target_link_libraries(amrex PUBLIC "-L${HIP_PATH}/../roctracer/lib/ -lroctracer64" "-L${HIP_PATH}/../roctracer/lib -lroctx64") - endif () + target_include_directories(amrex SYSTEM PUBLIC + ${HIP_PATH}/../roctracer/include + ${HIP_PATH}/../rocprofiler/include + ) + target_link_libraries(amrex PUBLIC + "-L${HIP_PATH}/../roctracer/lib -lroctracer64" + "-L${HIP_PATH}/../roctracer/lib -lroctx64" + ) + endif() target_link_libraries(amrex PUBLIC hip::hiprand roc::rocrand roc::rocprim) # avoid forcing the rocm LLVM flags on a gfortran From c812d8d5204ee2f741446916d6a4298098433e2a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 13 Sep 2022 17:28:42 -0700 Subject: [PATCH 5/5] GNUmake: Update Includes in `hip.mak` Use public prefix. --- Tools/GNUMake/comps/hip.mak | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Tools/GNUMake/comps/hip.mak b/Tools/GNUMake/comps/hip.mak index d86f887e087..bb4c6d98a17 100644 --- a/Tools/GNUMake/comps/hip.mak +++ b/Tools/GNUMake/comps/hip.mak @@ -107,7 +107,7 @@ ifeq ($(HIP_COMPILER),clang) # Generic HIP info ROC_PATH=$(realpath $(dir $(HIP_PATH))) - SYSTEM_INCLUDE_LOCATIONS += $(HIP_PATH)/include + SYSTEM_INCLUDE_LOCATIONS += $(ROC_PATH)/include $(HIP_PATH)/include # rocRand SYSTEM_INCLUDE_LOCATIONS += $(ROC_PATH)/include/hiprand $(ROC_PATH)/include/rocrand @@ -120,13 +120,12 @@ ifeq ($(HIP_COMPILER),clang) # rocThrust - Header only # SYSTEM_INCLUDE_LOCATIONS += $(ROC_PATH)/include/rocthrust - ifeq ($(USE_ROCTX),TRUE) # rocTracer - CXXFLAGS += -DAMREX_USE_ROCTX - HIPCC_FLAGS += -DAMREX_USE_ROCTX - SYSTEM_INCLUDE_LOCATIONS += $(ROC_PATH)/include/roctracer $(ROC_PATH)/include/rocprofiler - LIBRARY_LOCATIONS += $(ROC_PATH)/lib - LIBRARIES += -lroctracer64 -lroctx64 + ifeq ($(USE_ROCTX),TRUE) + CXXFLAGS += -DAMREX_USE_ROCTX + HIPCC_FLAGS += -DAMREX_USE_ROCTX + LIBRARY_LOCATIONS += $(ROC_PATH)/lib + LIBRARIES += -Wl,--rpath=$(ROC_PATH)/lib -lroctracer64 -lroctx64 endif # hipcc passes a lot of unused arguments to clang