Start Embedded Boundary development

CMakeLists.txt

@@ -44,6 +44,7 @@ set_default_install_dirs()
 #
 option(WarpX_APP           "Build the WarpX executable application"     ON)
 option(WarpX_ASCENT        "Ascent in situ diagnostics"                 OFF)
+option(WarpX_EB            "Embedded boundary support"                  OFF)
 option(WarpX_LIB           "Build WarpX as a shared library"            OFF)
 option(WarpX_MPI           "Multi-node support (message-passing)"       ON)
 option(WarpX_OPENPMD       "openPMD I/O (HDF5, ADIOS)"                  OFF)
@@ -181,6 +182,7 @@ endif()
 
 add_subdirectory(Source/BoundaryConditions)
 add_subdirectory(Source/Diagnostics)
+add_subdirectory(Source/EmbeddedBoundary)
 add_subdirectory(Source/Evolve)
 add_subdirectory(Source/FieldSolver)
 add_subdirectory(Source/Filter)

Docs/source/building/cmake.rst

@@ -123,6 +123,7 @@ CMake Option                  Default & Values                             Descr
 ``WarpX_ASCENT``              ON/**OFF**                                   Ascent in situ visualization
 ``WarpX_COMPUTE``             NOACC/**OMP**/CUDA/SYCL/HIP                  On-node, accelerated computing backend
 ``WarpX_DIMS``                **3**/2/RZ                                   Simulation dimensionality
+``WarpX_EB``                  ON/**OFF**                                   Embedded boundary support
 ``WarpX_LIB``                 ON/**OFF**                                   Build WarpX as a shared library
 ``WarpX_MPI``                 **ON**/OFF                                   Multi-node support (message-passing)
 ``WarpX_MPI_THREAD_MULTIPLE`` **ON**/OFF                                   MPI thread-multiple support, i.e. for ``async_io``

GNUmakefile

@@ -40,5 +40,7 @@ USE_PSATD = FALSE
 USE_PSATD_PICSAR = FALSE
 USE_RZ = FALSE
 
+USE_EB = FALSE
+
 WARPX_HOME := .
 include $(WARPX_HOME)/Source/Make.WarpX

Source/EmbeddedBoundary/CMakeLists.txt

@@ -0,0 +1,4 @@
+target_sources(WarpX
+  PRIVATE
+    WarpXInitEB.cpp
+)

Source/EmbeddedBoundary/Make.package

@@ -0,0 +1,3 @@
+CEXE_sources += WarpXInitEB.cpp
+
+VPATH_LOCATIONS += $(WARPX_HOME)/Source/EmbeddedBoundary

Source/EmbeddedBoundary/WarpXInitEB.cpp

@@ -0,0 +1,22 @@
+#include "WarpX.H"
+
+#include <AMReX_Config.H>
+#ifdef AMREX_USE_EB
+#   include <AMReX_EB2.H>
+#   include <AMReX_ParmParse.H>
+#endif
+
+
+void
+WarpX::InitEB ()
+{
+#ifdef AMREX_USE_EB
+    BL_PROFILE("InitEB");
+
+    amrex::ParmParse pp("eb2");
+    if (!pp.contains("geom_type")) {
+        pp.add("geom_type", "all_regular"); // use all_regular by default
+    }
+    amrex::EB2::Build(Geom(maxLevel()), maxLevel(), maxLevel());
+#endif
+}

Source/Make.WarpX

@@ -68,6 +68,7 @@ endif
 include $(WARPX_HOME)/Source/Make.package
 include $(WARPX_HOME)/Source/BoundaryConditions/Make.package
 include $(WARPX_HOME)/Source/Diagnostics/Make.package
+include $(WARPX_HOME)/Source/EmbeddedBoundary/Make.package
 include $(WARPX_HOME)/Source/FieldSolver/Make.package
 include $(WARPX_HOME)/Source/Filter/Make.package
 include $(WARPX_HOME)/Source/Initialization/Make.package
@@ -90,6 +91,11 @@ ifeq ($(USE_SENSEI_INSITU),TRUE)
   include $(AMREX_HOME)/Src/Extern/SENSEI/Make.package
 endif
 
+ifeq ($(USE_EB),TRUE)
+  include $(AMREX_HOME)/Src/EB/Make.package
+  USERSuffix := $(USERSuffix).EB
+endif
+
 ifeq ($(QED),TRUE)
   INCLUDE_LOCATIONS += $(PICSAR_HOME)/src/multi_physics/QED/src
   CXXFLAGS += -DWARPX_QED

Source/Parallelization/WarpXRegrid.cpp

@@ -105,6 +105,14 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi
     {
         if (ParallelDescriptor::NProcs() == 1) return;
 
+#ifdef AMREX_USE_EB
+        m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm,
+                                                       {1,1,1}, // Not clear how many ghost cells we need yet
+                                                       amrex::EBSupport::full);
+#else
+        m_field_factory[lev] = std::make_unique<FArrayBoxFactory>();
+#endif
+
         // Fine patch
         for (int idim=0; idim < 3; ++idim)
         {

Source/WarpX.H

@@ -877,6 +877,24 @@ private:
     // Domain decomposition on Level 0
     amrex::IntVect numprocs{0};
 
+    //
+    // Embeded Boundary
+    //
+
+    // Factory for field data
+    amrex::Vector<std::unique_ptr<amrex::FabFactory<amrex::FArrayBox> > > m_field_factory;
+
+    amrex::FabFactory<amrex::FArrayBox> const& fieldFactory (int lev) const noexcept {
+        return *m_field_factory[lev];
+    }
+#ifdef AMREX_USE_EB
+    amrex::EBFArrayBoxFactory const& fieldEBFactory (int lev) const noexcept {
+        return static_cast<amrex::EBFArrayBoxFactory const&>(*m_field_factory[lev]);
+    }
+#endif
+
+    void InitEB ();
+
 private:
     // void EvolvePSATD (int numsteps);
     void PushPSATD (amrex::Real dt);

Source/WarpX.cpp

@@ -161,6 +161,8 @@ WarpX::WarpX ()
 
     BackwardCompatibility();
 
+    InitEB();
+
     // Geometry on all levels has been defined already.
 
     // No valid BoxArray and DistributionMapping have been defined.
@@ -236,6 +238,7 @@ WarpX::WarpX ()
     pml.resize(nlevs_max);
     costs.resize(nlevs_max);
 
+    m_field_factory.resize(nlevs_max);
 
     if (em_solver_medium == MediumForEM::Macroscopic) {
         // create object for macroscopic solver
@@ -949,6 +952,13 @@ WarpX::ClearLevel (int lev)
 void
 WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& dm)
 {
+#ifdef AMREX_USE_EB
+    m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm,
+                                             {1,1,1}, // Not clear how many ghost cells we need yet
+                                             amrex::EBSupport::full);
+#else
+    m_field_factory[lev] = std::make_unique<FArrayBoxFactory>();
+#endif
 
     bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving);
 

cmake/WarpXFunctions.cmake

@@ -173,6 +173,10 @@ function(set_warpx_binary_name)
             set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".PSATD")
         endif()
 
+        if(WarpX_EB)
+            set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".EB")
+        endif()
+
         if(WarpX_QED)
             set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".QED")
         endif()
@@ -272,6 +276,7 @@ function(warpx_print_summary)
     message("    ASCENT: ${WarpX_ASCENT}")
     message("    COMPUTE: ${WarpX_COMPUTE}")
     message("    DIMS: ${WarpX_DIMS}")
+    message("    Embedded Boundary: ${WarpX_EB}")
     message("    LIB: ${WarpX_LIB}")
     message("    MPI: ${WarpX_MPI}")
     if(MPI)

cmake/dependencies/AMReX.cmake

@@ -30,6 +30,12 @@ macro(find_amrex)
             set(AMReX_OMP          OFF    CACHE INTERNAL "")
         endif()
 
+        if(WarpX_EB)
+            set(AMReX_EB ON CACHE INTERNAL "")
+        else()
+            set(AMReX_EB OFF CACHE INTERNAL "")
+        endif()
+
         if(WarpX_MPI)
             set(AMReX_MPI ON CACHE INTERNAL "")
             if(WarpX_MPI_THREAD_MULTIPLE)
@@ -137,14 +143,19 @@ macro(find_amrex)
         else()
             set(COMPONENT_DIM ${WarpX_DIMS}D)
         endif()
+        if(WarpX_EB)
+            set(COMPONENT_EB EB)
+        else()
+            set(COMPONENT_EB)
+        endif()
         if(WarpX_LIB)
             set(COMPONENT_PIC PIC)
         else()
             set(COMPONENT_PIC)
         endif()
         set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PRECISION})
 
-        find_package(AMReX 20.11 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIM} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} TINYP LSOLVERS)
+        find_package(AMReX 21.01 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIM} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} TINYP LSOLVERS)
         message(STATUS "AMReX: Found version '${AMReX_VERSION}'")
     endif()
 endmacro()