feat: Advance HIP support

host-configs/LLNL/amdgpu-base.cmake

@@ -0,0 +1,42 @@
+set( GEOS_ENABLE_VEM OFF CACHE BOOL "" FORCE )
+
+set( ENABLE_MATHPRESSO OFF CACHE BOOL "" )
+set( ENABLE_PAMELA OFF CACHE BOOL "" )
+set( ENABLE_PVTPackage ON CACHE BOOL "" )
+set( ENABLE_PETSC OFF CACHE BOOL "" FORCE )
+set( ENABLE_CALIPER ON CACHE BOOL "" )
+set( ENABLE_PAPI OFF CACHE BOOL "" )
+set( ENABLE_ESSL OFF CACHE BOOL "" )
+set( ENABLE_TRILINOS OFF CACHE BOOL "" )
+set( ENABLE_VTK ON CACHE BOOL "" )
+set( ENABLE_OPENMP OFF CACHE BOOL "" FORCE )
+
+set( CAMP_STANDALONE TRUE CACHE BOOL "" )
+
+# ROCM options
+set( ENABLE_ROCM ON CACHE BOOL "" FORCE )
+set( ROCM_ROOT "${HIP_ROOT}" CACHE PATH "" )
+
+set( GEOS_BUILD_OBJ_LIBS OFF CACHE BOOL "" FORCE )
+set( ENABLE_GTEST_DEATH_TESTS OFF CACHE BOOL "" )
+set( gtest_disable_pthreads ON CACHE BOOL "" )
+
+set( ENABLE_TESTS ON CACHE BOOL "" FORCE )
+set( ENABLE_EXAMPLES ON CACHE BOOL "" FORCE )
+set( ENABLE_BENCHMARKS ON CACHE BOOL "" FORCE )
+set( ENABLE_DOCS OFF CACHE BOOL "" FORCE )
+
+set( ENABLE_SCOTCH OFF CACHE BOOL "" FORCE )
+set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE )
+
+# HYPRE options
+set( ENABLE_HYPRE_DEVICE "HIP" CACHE STRING "" )
+set( ENABLE_HYPRE_MIXINT ON CACHE STRING "" )
+
+# TPLs
+include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake)
+
+# ATS
+set(USER $ENV{USER} CACHE STRING "")
+set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}"  CACHE PATH "")
+set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines"  CACHE PATH "")

host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake

@@ -0,0 +1,9 @@
+include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tioga-cce-18-rocm-6.2.1.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake)
+
+# MPI
+set(MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "")
+
+# Workaround for linking errors involving missing explicit instantiations of certain functions.
+# TODO: we should add the missing explicit instantiations
+set( CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE )

host-configs/LLNL/tioga-base.cmake → host-configs/LLNL/tuo-base.cmake

@@ -1,17 +1,15 @@
-set( ENABLE_MATHPRESSO OFF CACHE BOOL "" )
+set( GEOS_ENABLE_VEM OFF CACHE BOOL "" FORCE )
 
-set( ENABLE_PAMELA ON CACHE BOOL "" )
+set( ENABLE_MATHPRESSO OFF CACHE BOOL "" )
+set( ENABLE_PAMELA OFF CACHE BOOL "" )
 set( ENABLE_PVTPackage ON CACHE BOOL "" )
-
 set( ENABLE_PETSC OFF CACHE BOOL "" FORCE )
-
-set( ENABLE_CALIPER OFF CACHE BOOL "" )
+set( ENABLE_CALIPER ON CACHE BOOL "" )
 set( ENABLE_PAPI OFF CACHE BOOL "" )
-
 set( ENABLE_ESSL OFF CACHE BOOL "" )
-
 set( ENABLE_TRILINOS OFF CACHE BOOL "" )
-set( ENABLE_VTK OFF CACHE BOOL "" )
+set( ENABLE_VTK ON CACHE BOOL "" )
+set( ENABLE_OPENMP OFF CACHE BOOL "" FORCE )
 
 set( CAMP_STANDALONE TRUE CACHE BOOL "" )
 
@@ -30,3 +28,14 @@ set( ENABLE_DOCS OFF CACHE BOOL "" FORCE )
 
 set( ENABLE_SCOTCH OFF CACHE BOOL "" FORCE )
 set( ENABLE_SUPERLU_DIST OFF CACHE BOOL "" FORCE )
+
+# HYPRE options
+set( ENABLE_HYPRE_MIXINT ON CACHE STRING "" )
+
+# TPLs
+include(${CMAKE_CURRENT_LIST_DIR}/../tpls.cmake)
+
+# ATS
+set(USER $ENV{USER} CACHE STRING "")
+set(ATS_WORKING_DIR "/p/lustre2/${USER}/integratedTestsGEOS/${CONFIG_NAME}"  CACHE PATH "")
+set(ATS_BASELINE_DIR "/p/lustre2/${USER}/integratedTestsGEOS/baselines"  CACHE PATH "")

host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake

@@ -0,0 +1,9 @@
+include(${CMAKE_CURRENT_LIST_DIR}/../../src/coreComponents/LvArray/host-configs/LLNL/tuo-cce-18-rocm-6.2.1.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/amdgpu-base.cmake)
+
+# MPI
+set(MPI_HOME /opt/cray/pe/mpich/8.1.31/ofi/crayclang/18.0 CACHE PATH "")
+
+# Workaround for linking errors involving missing explicit instantiations of certain functions.
+# TODO: we should add the missing explicit instantiations
+set( CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--allow-shlib-undefined" CACHE STRING "" FORCE )

src/coreComponents/common/LifoStorage.hpp

@@ -64,7 +64,7 @@ class LifoStorage
     m_bufferCount( 0 )
   {
     LIFO_LOG_RANK( " LIFO : maximum size "<< m_maxNumberOfBuffers << " buffers " );
-    double bufferSize = ( ( double ) m_bufferSize ) / ( 1024.0 * 1024.0 );
+    GEOS_MAYBE_UNUSED double bufferSize = ( ( double ) m_bufferSize ) / ( 1024.0 * 1024.0 );
     LIFO_LOG_RANK( " LIFO : buffer size "<< bufferSize << "MB" );
     if( numberOfBuffersToStoreOnDevice < 0 )
     {

src/coreComponents/common/LifoStorageCommon.hpp

@@ -164,7 +164,7 @@ class LifoStorageCommon
     GEOS_ERROR( "To use LifoStorage, both _SC_AVPHYS_PAGES and _SC_PAGESIZE must be defined." );
 #endif
     int numberOfBuffersToStoreOnHost = std::max( 1, std::min( ( int )( 0.01 * percent * free / bufferSize ), maxNumberOfBuffers - numberOfBuffersToStoreOnDevice ) );
-    double freeGB = ( ( double ) free ) / ( 1024.0 * 1024.0 * 1024.0 ) / MpiWrapper::nodeCommSize();
+    GEOS_MAYBE_UNUSED double freeGB = ( ( double ) free ) / ( 1024.0 * 1024.0 * 1024.0 ) / MpiWrapper::nodeCommSize();
     LIFO_LOG_RANK( " LIFO : available memory on host " << freeGB << " GB" );
     return numberOfBuffersToStoreOnHost;
   }

src/coreComponents/common/MemoryInfos.cpp

@@ -44,6 +44,11 @@ MemoryInfos::MemoryInfos( umpire::MemoryResourceTraits::resource_type resourceTy
     case umpire::MemoryResourceTraits::resource_type::um:
       #if defined( GEOS_USE_CUDA )
       cudaMemGetInfo( &m_availableMemory, &m_totalMemory );
+      #elif defined( GEOS_USE_HIP )
+      {
+        hipError_t const err = hipMemGetInfo( &m_availableMemory, &m_totalMemory );
+        GEOS_WARNING_IF( err != hipSuccess, GEOS_FMT( "HIP error found: {} at {}:{}", hipGetErrorString( err ), __FILE__, __LINE__ ) );
+      }
       #else
       GEOS_WARNING( "Unknown device physical memory size getter for this compiler." );
       m_physicalMemoryHandled = 0;

src/coreComponents/common/TypeDispatch.hpp

@@ -376,7 +376,7 @@ bool dispatch( LIST const combinations,
   if( !success )
   {
     auto typePrinter = []( auto t ){ return LvArray::system::demangle( typeid( typename decltype(t)::type ).name() ); };
-    auto typeListPrinter = [typePrinter]( auto tlist ){ return internal::listToString( typename decltype( tlist )::type{}, "\n  ", "", typePrinter ); };
+    GEOS_MAYBE_UNUSED auto typeListPrinter = [typePrinter]( auto tlist ){ return internal::listToString( typename decltype( tlist )::type{}, "\n  ", "", typePrinter ); };
 
     GEOS_ERROR( "Types were not dispatched. The types of the input objects are:\n" <<
                 "( "<<(  ( "\n  " + LvArray::system::demangle( internal::typeIdWrapper( objects ).name() ) ) + ... )<<" \n)\n"<<

src/coreComponents/linearAlgebra/CMakeLists.txt

@@ -38,7 +38,6 @@ set( linearAlgebra_headers
      solvers/BicgstabSolver.hpp
      solvers/BlockPreconditioner.hpp
      solvers/CgSolver.hpp
-     solvers/GmresSolver.hpp
      solvers/KrylovSolver.hpp
      solvers/KrylovUtils.hpp
      solvers/PreconditionerBlockJacobi.hpp
@@ -60,6 +59,12 @@ set( linearAlgebra_headers
      utilities/NormalOperator.hpp
      utilities/ReverseCutHillMcKeeOrdering.hpp
      utilities/TransposeOperator.hpp )
+
+# Workaround for link issue with cce-18
+if( NOT ${ENABLE_HIP} )
+    list( APPEND linearAlgebra_headers solvers/GmresSolver.hpp )
+endif()
+
 #
 # Specify all sources
 #
@@ -68,26 +73,31 @@ set( linearAlgebra_sources
      solvers/BicgstabSolver.cpp
      solvers/BlockPreconditioner.cpp
      solvers/CgSolver.cpp
-     solvers/GmresSolver.cpp
      solvers/KrylovSolver.cpp
      solvers/SeparateComponentPreconditioner.cpp
      utilities/ReverseCutHillMcKeeOrdering.cpp )
 
+# Workaround for link issue with cce-18
+if( NOT ${ENABLE_HIP} )
+    list( APPEND linearAlgebra_sources solvers/GmresSolver.hpp )
+endif()
+
 set( dependencyList ${parallelDeps} mesh denseLinearAlgebra finiteVolume )
 set( tplDependencyList "" )
 
-list( APPEND linearAlgebra_headers interfaces/direct/SuiteSparse.hpp )
-list( APPEND linearAlgebra_sources interfaces/direct/SuiteSparse.cpp )
-list( APPEND tplDependencyList suitesparse )
+if( ENABLE_SUITESPARSE )
+    list( APPEND linearAlgebra_headers interfaces/direct/SuiteSparse.hpp )
+    list( APPEND linearAlgebra_sources interfaces/direct/SuiteSparse.cpp )
+    list( APPEND tplDependencyList suitesparse )
+endif( )
 
 if( ENABLE_SUPERLU_DIST )
-      list( APPEND linearAlgebra_headers interfaces/direct/SuperLUDist.hpp )
-      list( APPEND linearAlgebra_sources interfaces/direct/SuperLUDist.cpp )
-      list( APPEND tplDependencyList superlu_dist )
+    list( APPEND linearAlgebra_headers interfaces/direct/SuperLUDist.hpp )
+    list( APPEND linearAlgebra_sources interfaces/direct/SuperLUDist.cpp )
+    list( APPEND tplDependencyList superlu_dist )
 endif( )
 
 if( ENABLE_TRILINOS )
-
     list( APPEND linearAlgebra_headers
           interfaces/trilinos/EpetraExport.hpp
           interfaces/trilinos/EpetraMatrix.hpp
@@ -106,11 +116,9 @@ if( ENABLE_TRILINOS )
           interfaces/trilinos/TrilinosSolver.cpp )
 
     list( APPEND tplDependencyList trilinos )
-
 endif()
 
 if( ENABLE_HYPRE )
-
     list( APPEND linearAlgebra_headers
           interfaces/hypre/HypreExport.hpp
           interfaces/hypre/HypreInterface.hpp
@@ -151,11 +159,9 @@ if( ENABLE_HYPRE )
           interfaces/hypre/HypreVector.cpp )
 
     list( APPEND tplDependencyList hypre umpire )
-
 endif()
 
 if( ENABLE_PETSC )
-
     list( APPEND linearAlgebra_headers
           interfaces/petsc/PetscExport.hpp
           interfaces/petsc/PetscInterface.hpp
@@ -174,7 +180,6 @@ if( ENABLE_PETSC )
           interfaces/petsc/PetscVector.cpp )
 
     list( APPEND tplDependencyList petsc )
-
 endif()
 
 geos_decorate_link_dependencies( LIST decoratedDependencies
@@ -195,4 +200,3 @@ install( TARGETS linearAlgebra LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib )
 if( GEOS_ENABLE_TESTS )
   add_subdirectory( unitTests )
 endif( )
-

src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.cpp

@@ -19,6 +19,7 @@
 
 #include "SuiteSparse.hpp"
 
+#if defined(suitesparse_VERSION)
 #include "codingUtilities/Utilities.hpp"
 #include "common/Stopwatch.hpp"
 #include "linearAlgebra/common/common.hpp"
@@ -384,3 +385,5 @@ template class SuiteSparse< PetscInterface >;
 #endif
 
 }
+
+#endif /* if defined(suitesparse_VERSION) */

src/coreComponents/linearAlgebra/interfaces/direct/SuiteSparse.hpp

@@ -20,6 +20,7 @@
 #ifndef GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_
 #define GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_
 
+#if defined(suitesparse_VERSION)
 #include "common/DataTypes.hpp"
 #include "common/LinearSolverBase.hpp"
 #include "common/PreconditionerBase.hpp"
@@ -142,4 +143,5 @@ class SuiteSparse final : public LinearSolverBase< LAI >
 
 }
 
+#endif /* if defined(suitesparse_VERSION) */
 #endif /*GEOS_LINEARALGEBRA_INTERFACES_SUITESPARSE_HPP_*/

src/coreComponents/linearAlgebra/interfaces/hypre/HypreInterface.cpp

@@ -55,7 +55,13 @@ void HypreInterface::initialize()
   HYPRE_Initialize();
 #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP
   HYPRE_SetExecutionPolicy( HYPRE_EXEC_DEVICE );
+#if defined(HYPRE_USING_HIP)
+  HYPRE_SetSpGemmUseVendor( 1 );
+#else
+  // TODO: Is it better to use vendor's SpGEMM with CUDA?
   HYPRE_SetSpGemmUseVendor( 0 );
+#endif
+
   HYPRE_DeviceInitialize();
 #endif
   HYPRE_SetMemoryLocation( hypre::memoryLocation );
@@ -86,13 +92,19 @@ HypreInterface::createSolver( LinearSolverParameters params )
 #if defined(GEOS_USE_SUPERLU_DIST)
       return std::make_unique< SuperLUDist< HypreInterface > >( std::move( params ) );
 #else
-      GEOS_ERROR( "GEOSX is configured without support for SuperLU_dist." );
+      GEOS_ERROR( "GEOS is configured without support for SuperLU_dist." );
       return std::unique_ptr< LinearSolverBase< HypreInterface > >( nullptr );
 #endif
     }
     else
     {
+      /* TODO: add GEOS_USE_SUITESPARSE and use it below */
+#if defined(suitesparse_VERSION)
       return std::make_unique< SuiteSparse< HypreInterface > >( std::move( params ) );
+#else
+      GEOS_ERROR( "GEOS is configured without support for SuiteSparse." );
+      return std::unique_ptr< LinearSolverBase< HypreInterface > >( nullptr );
+#endif
     }
   }
   else

src/coreComponents/linearAlgebra/interfaces/trilinos/TrilinosInterface.cpp

@@ -38,11 +38,13 @@ TrilinosInterface::createSolver( LinearSolverParameters params )
 {
   if( params.solverType == LinearSolverParameters::SolverType::direct )
   {
+#if defined(GEOS_USE_SUPERLU_DIST)
     if( params.direct.parallel )
     {
       return std::make_unique< SuperLUDist< TrilinosInterface > >( std::move( params ) );
     }
     else
+#endif
     {
       return std::make_unique< SuiteSparse< TrilinosInterface > >( std::move( params ) );
     }