Merge pull request #62 from streeve/memory_space

Convert device_type->memory_space to match Cabana

.github/workflows/CI.yml

@@ -20,26 +20,26 @@ jobs:
         cxx: ['g++', 'clang++']
         backend: ['SERIAL', 'OPENMP']
         cmake_build_type: ['Debug', 'Release']
-        kokkos_ver: ['3.6.01']
+        kokkos_ver: ['3.7.02']
         output: ['HDF5']
         include:
           - distro: 'ubuntu:latest'
             cxx: 'g++'
             backend: 'SERIAL'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.6.01'
+            kokkos_ver: '3.7.02'
             output: 'SILO'
           - distro: 'ubuntu:latest'
             cxx: 'g++'
             backend: 'SERIAL'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.6.01'
+            kokkos_ver: '3.7.02'
             output: 'NONE'
           - distro: 'ubuntu:latest'
             cxx: 'g++'
             backend: 'SERIAL'
             cmake_build_type: 'Debug'
-            kokkos_ver: '3.6.01'
+            kokkos_ver: '3.7.02'
             output: 'BOTH'
     runs-on: ubuntu-20.04
     container: ghcr.io/ecp-copa/ci-containers/${{ matrix.distro }}
@@ -65,8 +65,8 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: ECP-CoPA/Cabana
-          # This version is post-release 0.5
-          ref: 31ba70d9e8f8e20f2e55e7e310905f950b21776f
+          # This version is post-release 0.6
+          ref: b46b5fbec4c18241bcf57ccdc84dbe857e5d489c
           path: cabana
       - name: Build Cabana
         working-directory: cabana
@@ -85,6 +85,7 @@ jobs:
             -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} \
             -DCMAKE_INSTALL_PREFIX=$HOME/Cabana \
             -DCMAKE_PREFIX_PATH="$HOME/kokkos" \
+            -DCabana_DISABLE_CAJITA_DEPRECATION_WARNINGS=ON \
             ${cabana_cmake_opts[@]}
           cmake --build build --parallel 2
           cmake --install build

examples/crack_branching.cpp

@@ -85,9 +85,8 @@ int main( int argc, char* argv[] )
 
         // Create particles from mesh.
         // Does not set displacements, velocities, etc.
-        using device_type = Kokkos::Device<exec_space, memory_space>;
         auto particles = std::make_shared<
-            CabanaPD::Particles<device_type, typename model_type::base_model>>(
+            CabanaPD::Particles<memory_space, typename model_type::base_model>>(
             exec_space(), inputs.low_corner, inputs.high_corner,
             inputs.num_cells, halo_width );
 
@@ -122,7 +121,7 @@ int main( int argc, char* argv[] )
         particles->updateParticles( exec_space{}, init_functor );
 
         // FIXME: use createSolver to switch backend at runtime.
-        auto cabana_pd = CabanaPD::createSolverFracture<device_type>(
+        auto cabana_pd = CabanaPD::createSolverFracture<memory_space>(
             inputs, particles, force_model, bc, prenotch );
         cabana_pd->init_force();
         cabana_pd->run();

examples/elastic_wave.cpp

@@ -57,10 +57,8 @@ int main( int argc, char* argv[] )
 
         // Create particles from mesh.
         // Does not set displacements, velocities, etc.
-        // FIXME: use createSolver to switch backend at runtime.
-        using device_type = Kokkos::Device<exec_space, memory_space>;
         auto particles = std::make_shared<
-            CabanaPD::Particles<device_type, typename model_type::base_model>>(
+            CabanaPD::Particles<memory_space, typename model_type::base_model>>(
             exec_space(), inputs.low_corner, inputs.high_corner,
             inputs.num_cells, halo_width );
 
@@ -92,7 +90,7 @@ int main( int argc, char* argv[] )
         };
         particles->updateParticles( exec_space{}, init_functor );
 
-        auto cabana_pd = CabanaPD::createSolverElastic<device_type>(
+        auto cabana_pd = CabanaPD::createSolverElastic<memory_space>(
             inputs, particles, force_model );
         cabana_pd->init_force();
         cabana_pd->run();

examples/kalthoff_winkler.cpp

@@ -88,10 +88,8 @@ int main( int argc, char* argv[] )
 
         // Create particles from mesh.
         // Does not set displacements, velocities, etc.
-        // FIXME: use createSolver to switch backend at runtime.
-        using device_type = Kokkos::Device<exec_space, memory_space>;
         auto particles = std::make_shared<
-            CabanaPD::Particles<device_type, typename model_type::base_model>>(
+            CabanaPD::Particles<memory_space, typename model_type::base_model>>(
             exec_space(), inputs.low_corner, inputs.high_corner,
             inputs.num_cells, halo_width );
 
@@ -122,7 +120,7 @@ int main( int argc, char* argv[] )
         };
         particles->updateParticles( exec_space{}, init_functor );
 
-        auto cabana_pd = CabanaPD::createSolverFracture<device_type>(
+        auto cabana_pd = CabanaPD::createSolverFracture<memory_space>(
             inputs, particles, force_model, bc, prenotch );
         cabana_pd->init_force();
         cabana_pd->run();

src/CabanaPD_Comm.hpp

@@ -241,10 +241,8 @@ class Comm<ParticleType, PMB>
     int mpi_rank = -1;
     int max_export;
 
-    // FIXME: this should use MemorySpace directly, but Cabana::Halo currently
-    // uses DeviceType.
-    using device_type = typename ParticleType::device_type;
-    using halo_type = Cabana::Halo<device_type>;
+    using memory_space = typename ParticleType::memory_space;
+    using halo_type = Cabana::Halo<memory_space>;
     using gather_u_type =
         Cabana::Gather<halo_type, typename ParticleType::aosoa_u_type>;
     std::shared_ptr<gather_u_type> gather_u;
@@ -311,7 +309,7 @@ class Comm<ParticleType, LPS> : public Comm<ParticleType, PMB>
 {
   public:
     using base_type = Comm<ParticleType, PMB>;
-    using device_type = typename base_type::device_type;
+    using memory_space = typename base_type::memory_space;
     using halo_type = typename base_type::halo_type;
     using base_type::gather_u;
     using base_type::halo;

src/CabanaPD_Particles.hpp

@@ -75,19 +75,15 @@
 
 namespace CabanaPD
 {
-// FIXME: this should use MemorySpace directly, but DeviceType enables the
-// friend class with Comm (which only uses DeviceType because Cabana::Halo
-// currently does).
-template <class DeviceType, class ModelType, int Dimension = 3>
+template <class MemorySpace, class ModelType, int Dimension = 3>
 class Particles;
 
-template <class DeviceType, int Dimension>
-class Particles<DeviceType, PMB, Dimension>
+template <class MemorySpace, int Dimension>
+class Particles<MemorySpace, PMB, Dimension>
 {
   public:
-    using self_type = Particles<DeviceType, PMB, Dimension>;
-    using device_type = DeviceType;
-    using memory_space = typename device_type::memory_space;
+    using self_type = Particles<MemorySpace, PMB, Dimension>;
+    using memory_space = MemorySpace;
     using execution_space = typename memory_space::execution_space;
     static constexpr int dim = Dimension;
 
@@ -191,7 +187,7 @@ class Particles<DeviceType, PMB, Dimension>
 
         // Create a local mesh.
         local_grid = Cajita::createLocalGrid( global_grid, halo_width );
-        auto local_mesh = Cajita::createLocalMesh<device_type>( *local_grid );
+        auto local_mesh = Cajita::createLocalMesh<memory_space>( *local_grid );
 
         for ( int d = 0; d < dim; d++ )
         {
@@ -207,7 +203,7 @@ class Particles<DeviceType, PMB, Dimension>
     void createParticles( const ExecSpace& exec_space )
     {
         // Create a local mesh and owned space.
-        auto local_mesh = Cajita::createLocalMesh<device_type>( *local_grid );
+        auto local_mesh = Cajita::createLocalMesh<memory_space>( *local_grid );
         auto owned_cells = local_grid->indexSpace(
             Cajita::Own(), Cajita::Cell(), Cajita::Local() );
 
@@ -454,14 +450,13 @@ class Particles<DeviceType, PMB, Dimension>
 #endif
 };
 
-template <class DeviceType, int Dimension>
-class Particles<DeviceType, LPS, Dimension>
-    : public Particles<DeviceType, PMB, Dimension>
+template <class MemorySpace, int Dimension>
+class Particles<MemorySpace, LPS, Dimension>
+    : public Particles<MemorySpace, PMB, Dimension>
 {
   public:
-    using self_type = Particles<DeviceType, LPS, Dimension>;
-    using base_type = Particles<DeviceType, PMB, Dimension>;
-    using device_type = typename base_type::device_type;
+    using self_type = Particles<MemorySpace, LPS, Dimension>;
+    using base_type = Particles<MemorySpace, PMB, Dimension>;
     using memory_space = typename base_type::memory_space;
     using base_type::dim;
 

src/CabanaPD_Solver.hpp

@@ -89,13 +89,13 @@ class SolverBase
     virtual void run() = 0;
 };
 
-template <class DeviceType, class InputType, class ParticleType,
+template <class MemorySpace, class InputType, class ParticleType,
           class ForceModel>
 class SolverElastic
 {
   public:
-    using exec_space = typename DeviceType::execution_space;
-    using memory_space = typename DeviceType::memory_space;
+    using memory_space = MemorySpace;
+    using exec_space = typename memory_space::execution_space;
 
     using particle_type = ParticleType;
     using integrator_type = Integrator<exec_space>;
@@ -332,14 +332,14 @@ class SolverElastic
     bool print;
 };
 
-template <class DeviceType, class InputType, class ParticleType,
+template <class MemorySpace, class InputType, class ParticleType,
           class ForceModel, class BoundaryCondition, class PrenotchType>
 class SolverFracture
-    : public SolverElastic<DeviceType, InputType, ParticleType, ForceModel>
+    : public SolverElastic<MemorySpace, InputType, ParticleType, ForceModel>
 {
   public:
     using base_type =
-        SolverElastic<DeviceType, InputType, ParticleType, ForceModel>;
+        SolverElastic<MemorySpace, InputType, ParticleType, ForceModel>;
     using exec_space = typename base_type::exec_space;
     using memory_space = typename base_type::memory_space;
 
@@ -495,26 +495,27 @@ class SolverFracture
     using base_type::print;
 };
 
-template <class DeviceType, class InputsType, class ParticleType,
+template <class MemorySpace, class InputsType, class ParticleType,
           class ForceModel>
 auto createSolverElastic( InputsType inputs,
                           std::shared_ptr<ParticleType> particles,
                           ForceModel model )
 {
     return std::make_shared<
-        SolverElastic<DeviceType, InputsType, ParticleType, ForceModel>>(
+        SolverElastic<MemorySpace, InputsType, ParticleType, ForceModel>>(
         inputs, particles, model );
 }
 
-template <class DeviceType, class InputsType, class ParticleType,
+template <class MemorySpace, class InputsType, class ParticleType,
           class ForceModel, class BCType, class PrenotchType>
 auto createSolverFracture( InputsType inputs,
                            std::shared_ptr<ParticleType> particles,
                            ForceModel model, BCType bc, PrenotchType prenotch )
 {
-    return std::make_shared<SolverFracture<DeviceType, InputsType, ParticleType,
-                                           ForceModel, BCType, PrenotchType>>(
-        inputs, particles, model, bc, prenotch );
+    return std::make_shared<
+        SolverFracture<MemorySpace, InputsType, ParticleType, ForceModel,
+                       BCType, PrenotchType>>( inputs, particles, model, bc,
+                                               prenotch );
 }
 
 /*

unit_test/tstComm.hpp

@@ -38,7 +38,7 @@ namespace Test
 void testHalo()
 {
     using exec_space = TEST_EXECSPACE;
-    using device_type = TEST_DEVICE;
+    using memory_space = TEST_MEMSPACE;
 
     std::array<double, 3> box_min = { -1.0, -1.0, -1.0 };
     std::array<double, 3> box_max = { 1.0, 1.0, 1.0 };
@@ -48,7 +48,7 @@ void testHalo()
     int halo_width = 2;
     // FIXME: This is for m = 1; should be calculated from m
     int expected_n = 6;
-    using particles_type = CabanaPD::Particles<device_type, CabanaPD::PMB>;
+    using particles_type = CabanaPD::Particles<memory_space, CabanaPD::PMB>;
     particles_type particles( exec_space(), box_min, box_max, num_cells,
                               halo_width );
     // Set ID equal to MPI rank.

unit_test/tstForce.hpp

@@ -373,7 +373,7 @@ computeReferenceForceX( QuadraticTag,
 // System creation.
 //---------------------------------------------------------------------------//
 template <class ModelType>
-CabanaPD::Particles<TEST_DEVICE, typename ModelType::base_model>
+CabanaPD::Particles<TEST_MEMSPACE, typename ModelType::base_model>
 createParticles( ModelType, LinearTag, const double dx, const double s0 )
 {
     std::array<double, 3> box_min = { -1.0, -1.0, -1.0 };
@@ -383,7 +383,7 @@ createParticles( ModelType, LinearTag, const double dx, const double s0 )
 
     // Create particles based on the mesh.
     using ptype =
-        CabanaPD::Particles<TEST_DEVICE, typename ModelType::base_model>;
+        CabanaPD::Particles<TEST_MEMSPACE, typename ModelType::base_model>;
     ptype particles( TEST_EXECSPACE{}, box_min, box_max, num_cells, 0 );
 
     auto x = particles.sliceReferencePosition();

unit_test/tstIntegrator.hpp

@@ -43,7 +43,7 @@ void testIntegratorReversibility( int steps )
     std::array<double, 3> box_max = { 1.0, 1.0, 1.0 };
     std::array<int, 3> num_cells = { 10, 10, 10 };
 
-    CabanaPD::Particles<TEST_DEVICE, CabanaPD::PMB> particles(
+    CabanaPD::Particles<TEST_MEMSPACE, CabanaPD::PMB> particles(
         exec_space(), box_min, box_max, num_cells, 0 );
     auto x = particles.sliceReferencePosition();
     std::size_t num_particle = x.size();