Merge branch 'dev'

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.24)
 
 project(
   NESO-Particles
-  VERSION 0.6.0
+  VERSION 0.7.0
   LANGUAGES CXX C)
 
 include(GNUInstallDirs)
@@ -17,6 +17,7 @@ option(
   ON)
 option(NESO_PARTICLES_LEGACY_DEVICE_SELECTORS
        "Option to use SYCL 1.2 selectors if 2020 ones are not supported." OFF)
+option(NESO_PARTICLES_ENABLE_PETSC "Look for PETSc." OFF)
 option(
   NESO_PARTICLES_DEVICE_AWARE_MPI
   "Enable device aware MPI by default, i.e. pass device pointers to MPI rather than copying to the host."
@@ -45,6 +46,10 @@ set(HEADER_FILES
     ${INCLUDE_DIR_NESO_PARTICLES}/cell_dat_move.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/cell_dat_move_impl.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/communication.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/communication/comm_pair.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/communication/communication_edges_counter.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/communication/communication_typedefs.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/communication/communication_utility.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/compute_target.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/blocked_binary_tree.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/cell_dat.hpp
@@ -59,32 +64,70 @@ set(HEADER_FILES
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/descendant_products.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/global_array.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/local_array.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/containers/local_memory_block.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/containers/lookup_table.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/nd_local_array.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/product_matrix.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/sym_vector.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/sym_vector_impl.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/containers/tuple.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/departing_particle_identification.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/departing_particle_identification_impl.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/device_functions.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/device_limits.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/domain.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/error_propagate.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/common/bounding_box.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/common/common.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/common/coordinate_mapping.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/common/local_claim.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/common/overlay_cartesian_mesh.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/common/dof_mapper_dg.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/common/quadrature_point_mapper.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/boundary_interaction/boundary_reflection.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/boundary_interaction/boundary_interaction.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/boundary_interaction/boundary_interaction_2d.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/boundary_interaction/boundary_interaction_common.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/dmplex_cell_serialise.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/dmplex_helper.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/dmplex_interface.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/dmplex_local_mapper.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/dmplex_project_evaluate.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/dmplex_utility.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/petsc_api_redirection.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/particle_cell_mapping/dmplex_2d_mapper.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/particle_cell_mapping/dmplex_host_mapper.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/petsc_common.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/petsc_utility.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/petsc_interface.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/project_evaluate/dmplex_project_evaluate_base.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/project_evaluate/dmplex_project_evaluate_barycentric.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/petsc/project_evaluate/dmplex_project_evaluate_dg.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/external_interfaces/vtk/vtk.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/global_mapping.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/global_mapping_impl.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/global_move.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/global_move_exchange.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/local_mapping.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/local_move.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/loop/access_descriptors.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/loop/kernel.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/loop/particle_loop.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/loop/particle_loop_base.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/loop/particle_loop_utility.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/loop/particle_loop_index.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/loop/particle_loop_iteration_set.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/loop/pli_particle_dat.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/mesh_hierarchy.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/mesh_hierarchy_data/mesh_hierarchy_container.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/mesh_hierarchy_data/mesh_hierarchy_data.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/mesh_hierarchy_data/serial_container.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/mesh_hierarchy_data/serial_interface.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/mesh_interface.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/mesh_interface_local_decomp.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/packing_unpacking.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/parallel_initialisation.hpp
+    ${INCLUDE_DIR_NESO_PARTICLES}/parameters.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/particle_dat.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/particle_group.hpp
     ${INCLUDE_DIR_NESO_PARTICLES}/particle_group_impl.hpp
@@ -168,6 +211,34 @@ if(NESO_PARTICLES_ENABLE_HDF5)
   endif()
 endif()
 
+# set the restrict keyword
+target_compile_definitions(NESO-Particles INTERFACE RESTRICT=${RESTRICT})
+
+# now look for PETSc
+if(NESO_PARTICLES_ENABLE_PETSC)
+  find_package(PkgConfig)
+  if(PKG_CONFIG_FOUND)
+    pkg_check_modules(PETSC PETSc)
+  endif()
+
+  if(PETSC_FOUND)
+    message(STATUS "PETSc found")
+    # TODO CHECK AND CLEANUP
+    target_compile_definitions(NESO-Particles INTERFACE NESO_PARTICLES_PETSC
+                                                        ${PETSC_DEFINITIONS})
+    add_definitions(-DNESO_PARTICLES_PETSC ${PETSC_DEFINITIONS})
+    target_link_libraries(NESO-Particles INTERFACE ${PETSC_LINK_LIBRARIES})
+    target_include_directories(NESO-Particles INTERFACE ${PETSC_INCLUDE_DIRS})
+    message(STATUS PETSC_LINK_LIBRARIES ${PETSC_LINK_LIBRARIES})
+    message(STATUS PETSC_LDFLAGS ${PETSC_LDFLAGS})
+    message(STATUS PETSC_INCLUDE_DIRS ${PETSC_INCLUDE_DIRS})
+    message(STATUS PETSC_LIBRARY_DIRS ${PETSC_LIBRARY_DIRS})
+    message(STATUS PETSC_CFLAGS ${PETSC_CFLAGS})
+  else()
+    message(STATUS "PETSc NOT found")
+  endif()
+endif()
+
 # Find SYCL
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/SYCL.cmake)
 if(NESO_PARTICLES_ENABLE_FIND_SYCL)

README.md

@@ -9,8 +9,8 @@ This is the particle component of NESO and is designed as a header only library.
 ## Dependencies
 
 * CMake 
-* SYCL 2020 (tested with hipsycl 0.9.4 or Intel DPCPP 2022.1.0).
-* MPI 3.0 (tested with MPICH 4.0 or IntelMPI 2021.6)
+* SYCL 2020 (tested with hipsycl 0.9.4 or Intel DPCPP 2024.1.0).
+* MPI 3.0 (tested with MPICH 4.0 or IntelMPI 2021.12.1)
 * HDF5 (parallel): (optional) If particle trajectories are required - will execute without.
 
 ## Installing

cmake/restrict-keyword.cmake

@@ -37,4 +37,3 @@ if(NOT DEFINED RESTRICT)
 endif()
 
 message(STATUS "Using restrict keyword: " ${RESTRICT})
-add_compile_definitions(RESTRICT=${RESTRICT})

docs/sphinx/source/conf.py

@@ -26,6 +26,7 @@
 html_theme = "pydata_sphinx_theme"
 html_static_path = ['_static']
 html_css_files = ["custom.css"]
+html_favicon = "_static/favicon.png"
 
 html_sidebars = {
     "**": ["globaltoc.html"]
@@ -36,7 +37,12 @@
     "navbar_align": "left",
     "primary_sidebar_end": [],
     "navigation_depth": 0,
-    "show_nav_level": 3
+    "show_nav_level": 3,
+    "logo": {
+        "text": "",
+        "image_light": "neso_particles_logo_light_small.png",
+        "image_dark": "neso_particles_logo_dark_small.png",
+    },
 }
 
 import os

docs/sphinx/source/guide-user/installation.rst

@@ -6,7 +6,7 @@ Dependencies
 ============
 
 * CMake 3.24+.
-* SYCL 2020 with USM support: Tested with hipsycl 0.9.4 or Intel DPCPP 2022.1.0.
+* SYCL 2020 with USM support: Tested with hipsycl 0.9.4 or Intel DPCPP 2024.1.0.
 * MPI 3.0: Tested with MPICH 4.0 or IntelMPI 2021.6. See known issues for Ubuntu 22.03 mpich.
 * HDF5 (parallel): (optional see CMake variable ``NESO_PARTICLES_ENABLE_HDF5``) If particle trajectories are required - will execute without.
 
@@ -49,6 +49,12 @@ These downstream implementations should pass CMake variables or compilers at con
 When a SYCL implementation is passed to NESO-Particles itself then it is used to build the tests.
 Please read the section :ref:`device-aware-mpi` for more information relating to device aware MPI.
 
+CXX Standard
+------------
+
+Although we explicitly set C++17 as the required C++ version on the NESO-Particles interface target, CMake may not pass this requirement down onto targets using NESO-Particles.
+Downstream projects may need to explicitly set the required C++ standard to C++17.
+
 Installing
 ==========
 

docs/sphinx/source/guide-user/profiling.rst

@@ -0,0 +1,68 @@
+*********
+Profiling
+*********
+
+Built-in Profiling
+==================
+
+Internally NP can record the time taken for expensive operations and internal ParticleLoops.
+The time taken for user written loops can also be recorded.
+For these recorded times to make much sense it is beneficial to given ParticleLoops a meaningful name.
+Furthermore code regions, which do not have to be ParticleLoops, can also be added to the set of recorded regions by using the ProfileRegion class.
+
+.. warning::
+    Profiling of regions is disabled unless `NESO_PARTICLES_PROFILING_REGION` is defined.
+
+
+To enable built in profiling of regions the preprocessor variable `NESO_PARTICLES_PROFILING_REGION` must be defined. 
+If this variable is not defined then the profiling functions and methods become no-ops.
+This variable can be defined by compiler flags or defined before including NP as follows.
+
+.. code-block:: cpp
+
+    #define NESO_PARTICLES_PROFILING_REGION
+    #include <neso_particles.hpp>
+
+
+The `ProfileRegion` class records the time taken for a specific piece of code. 
+These regions are recorded within a `ProfileMap` instance. 
+There is a `ProfileMap` member within the `SYCLTarget` type which records the regions for ParticleLoops which execute on that compute device.
+
+In the following code snippet we demonstrate how to profile ParticleLoops and user defined regions.
+Finally this profiling data is written to a JSON file by each rank. 
+The write operation is not collective and users may choose to only write the data from specific ranks.
+
+.. literalinclude:: ../example_sources/example_profile_regions.hpp
+   :language: cpp
+   :caption: Example of using ProfileRegion and writing events to disk.
+
+
+Plotting ProfileRegions
+=======================
+
+We provide a helper script `scripts/profile_region_plotting/profile_region_plotting.py` to aid plotting the regions written to the JSON file.
+The requirements for this script are contained in the `requirement.txt` file and can be installed into a Python virtual environment as follows:
+::
+
+    # Create and activate a virtual environment
+    $ python3 -m venv profile_region_plotting_env
+    $ source profile_region_plotting_env/bin/activate
+
+    # Install the dependencies into the virtual environment
+    (profile_region_plotting_env)$ pip install -r requirements.txt
+    # pip output omitted
+
+    # Run the script
+    $ python profile_region_plotting.py -s <start_time> -e <end_time> *.json
+
+    # Run with -h for a complete set of options.
+
+
+The script plots time on the x-axis and MPI rank on the y-axis.
+On launch all recorded events, within the specified time window, are plotted.
+To simplify the view double click on an item in the legend on the right hand side to focus only on regions with that name.
+Other regions can then be added to the view one-by-one by clicking on them in the legend.
+
+
+
+

include/neso_particles.hpp

@@ -13,20 +13,26 @@
 #include "neso_particles/containers/descendant_products.hpp"
 #include "neso_particles/containers/global_array.hpp"
 #include "neso_particles/containers/local_array.hpp"
+#include "neso_particles/containers/lookup_table.hpp"
 #include "neso_particles/containers/nd_index.hpp"
 #include "neso_particles/containers/nd_local_array.hpp"
 #include "neso_particles/containers/product_matrix.hpp"
 #include "neso_particles/containers/rng/rng.hpp"
 #include "neso_particles/containers/sym_vector.hpp"
 #include "neso_particles/containers/sym_vector_impl.hpp"
 #include "neso_particles/containers/tuple.hpp"
+#include "neso_particles/device_functions.hpp"
 #include "neso_particles/domain.hpp"
 #include "neso_particles/error_propagate.hpp"
+#include "neso_particles/external_interfaces/common/common.hpp"
+#include "neso_particles/external_interfaces/petsc/petsc_interface.hpp"
+#include "neso_particles/external_interfaces/vtk/vtk.hpp"
 #include "neso_particles/global_mapping.hpp"
 #include "neso_particles/local_mapping.hpp"
 #include "neso_particles/local_move.hpp"
 #include "neso_particles/loop/particle_loop.hpp"
 #include "neso_particles/mesh_hierarchy.hpp"
+#include "neso_particles/mesh_hierarchy_data/mesh_hierarchy_data.hpp"
 #include "neso_particles/mesh_interface.hpp"
 #include "neso_particles/mesh_interface_local_decomp.hpp"
 #include "neso_particles/parallel_initialisation.hpp"

include/neso_particles/boundary_conditions.hpp

@@ -44,7 +44,7 @@ class CartesianPeriodic {
   CartesianPeriodic(SYCLTargetSharedPtr sycl_target,
                     std::shared_ptr<CartesianHMesh> mesh,
                     ParticleDatSharedPtr<REAL> position_dat)
-      : mesh(mesh), sycl_target(sycl_target), d_extents(sycl_target, 3),
+      : d_extents(sycl_target, 3), sycl_target(sycl_target), mesh(mesh),
         position_dat(position_dat) {
 
     NESOASSERT(mesh->ndim <= 3, "bad mesh ndim");

include/neso_particles/cartesian_mesh.hpp

@@ -4,6 +4,7 @@
 #include <memory>
 #include <mpi.h>
 
+#include "cell_binning.hpp"
 #include "compute_target.hpp"
 #include "local_mapping.hpp"
 #include "loop/particle_loop.hpp"
@@ -37,13 +38,16 @@ class CartesianHMeshLocalMapperT : public LocalMapper {
   std::unique_ptr<BufferDeviceHost<int>> dh_dims;
   // Cell counts of the underlying mesh.
   std::unique_ptr<BufferDeviceHost<int>> dh_cell_counts;
+  // Cell binning for cell move
+  std::unique_ptr<CartesianCellBin> cart_cell_bin;
 
 public:
   /// Disable (implicit) copies.
   CartesianHMeshLocalMapperT(const CartesianHMeshLocalMapperT &st) = delete;
   /// Disable (implicit) copies.
   CartesianHMeshLocalMapperT &
   operator=(CartesianHMeshLocalMapperT const &a) = delete;
+  virtual ~CartesianHMeshLocalMapperT() = default;
 
   /// CartesianHMesh on which the lookup is based.
   CartesianHMeshSharedPtr mesh;
@@ -224,6 +228,8 @@ class CartesianHMeshLocalMapperT : public LocalMapper {
         }
       }
     }
+    this->cart_cell_bin =
+        std::make_unique<CartesianCellBin>(this->sycl_target, this->mesh);
   };
 
   /**
@@ -232,7 +238,8 @@ class CartesianHMeshLocalMapperT : public LocalMapper {
    *
    *  @param particle_group ParticleGroup to use.
    */
-  inline void map(ParticleGroup &particle_group, const int map_cell = -1) {
+  inline void map(ParticleGroup &particle_group,
+                  const int map_cell = -1) override {
     auto r = ProfileRegion("CartesianHMeshLocalMapper", "map");
 
     ParticleDatSharedPtr<REAL> &position_dat = particle_group.position_dat;
@@ -284,12 +291,25 @@ class CartesianHMeshLocalMapperT : public LocalMapper {
     this->sycl_target->profile_map.add_region(r);
   };
 
+  /**
+   *  Map positions to owning cells. Positions should be within the domain
+   *  prior to calling map, i.e. particles should be within the domain extents.
+   *
+   *  @param particle_group ParticleGroup to use.
+   *  @param map_cell Cell to map.
+   */
+  inline void map_cells(ParticleGroup &particle_group,
+                        const int map_cell = -1) override {
+    this->cart_cell_bin->map_cells(particle_group, map_cell);
+  }
+
   /**
    *  No-op implementation of callback.
    *
    *  @param particle_group ParticleGroup.
    */
-  inline void particle_group_callback(ParticleGroup &particle_group){};
+  inline void particle_group_callback(
+      [[maybe_unused]] ParticleGroup &particle_group) override {};
 };
 
 inline std::shared_ptr<CartesianHMeshLocalMapperT>