Merge branch 'development' into SampleData

CMake/SetAmrexOptions.cmake

@@ -30,6 +30,8 @@ set(AMReX_FORTRAN OFF)
 set(AMReX_LINEAR_SOLVERS OFF)
 if(ERF_ENABLE_POISSON_SOLVE)
   set(AMReX_LINEAR_SOLVERS ON)
+  set(AMReX_LINEAR_SOLVERS_EM OFF)
+  set(AMReX_LINEAR_SOLVERS_INCFLO OFF)
 endif()
 
 set(AMReX_PARTICLES OFF)

Exec/Make.ERF

@@ -112,7 +112,9 @@ AMReXdirs             += EB
 endif
 
 ifeq ($(USE_POISSON_SOLVE),TRUE)
-AMReXdirs             += LinearSolvers/MLMG
+USE_LINEAR_SOLVERS_INCFLO = FALSE
+USE_LINEAR_SOLVERS_EM     = FALSE
+AMReXdirs                += LinearSolvers
 endif
 
 ifeq ($(USE_HDF5),TRUE)

Source/BoundaryConditions/ERF_FillPatch.cpp

@@ -208,7 +208,7 @@ ERF::FillPatch (int lev, Real time,
 
     if (m_r2d) fill_from_bndryregs(mfs_vel,time);
 
-    // We call these even if init_type == real because these will fill the vertical bcs
+    // We call these even if init_type == InitType::Real because these will fill the vertical bcs
     // Note that we call FillBoundary inside the physbcs call
     (*physbcs_cons[lev])(*mfs_vel[Vars::cons],icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
     if (!cons_only) {
@@ -418,7 +418,7 @@ ERF::FillIntermediatePatch (int lev, Real time,
 
     if (m_r2d) fill_from_bndryregs(mfs_vel,time);
 
-    // We call this even if init_type == real because this routine will fill the vertical bcs
+    // We call this even if init_type == InitType::Real because this routine will fill the vertical bcs
     (*physbcs_cons[lev])(*mfs_vel[Vars::cons],icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
     if (!cons_only) {
         (*physbcs_u[lev])(*mfs_vel[Vars::xvel],0,1,ngvect_vels,time,BCVars::xvel_bc, do_fb);

Source/ERF.H

@@ -67,6 +67,13 @@
 class MultiBlockContainer;
 #endif
 
+/**
+ * Enum of possible initialization types
+*/
+AMREX_ENUM(InitType,
+    None, Input_Sounding, Ideal, Real, Metgrid, Uniform
+);
+
 /**
  * Enum of possible coarse/fine interpolation options
 */
@@ -957,13 +964,12 @@ private:
     // Native or NetCDF
     static std::string plotfile_type;
 
-    // init_type:  "ideal", "real", "input_sounding", "metgrid" or ""
-    static std::string init_type;
+    static InitType init_type;
 
     // sponge_type:  "input_sponge"
     static std::string sponge_type;
 
-    // use_real_bcs: only true if 1) ( (init_type == real) or (init_type == metgrid) )
+    // use_real_bcs: only true if 1) ( (init_type == InitType::Real) or (init_type == InitType::Metgrid) )
     //                        AND 2) we want to use the bc's from the WRF bdy file
     static bool use_real_bcs;
 
@@ -976,7 +982,7 @@ private:
     int real_set_width{0};
 
     // Flag to trigger initialization from input_sounding like WRF's ideal.exe
-    // used with init_type == "input_sounding"
+    // used with init_type == InitType::Input_Sounding
     static bool init_sounding_ideal;
 
     // 1D CDF output (for ingestion in AMR-Wind)

Source/ERF.cpp

@@ -43,10 +43,9 @@ int  ERF::pert_interval = -1;
 // Native AMReX vs NetCDF
 std::string ERF::plotfile_type    = "amrex";
 
-// init_type:  "uniform", "ideal", "real", "input_sounding", "metgrid" or ""
-std::string ERF::init_type;
+InitType ERF::init_type;
 
-// use_real_bcs: only true if 1) ( (init_type == real) or (init_type == metgrid) )
+// use_real_bcs: only true if 1) ( (init_type == InitType::Real) or (init_type == InitGrid::Metgrid) )
 //                        AND 2) we want to use the bc's from the WRF bdy file
 bool ERF::use_real_bcs;
 
@@ -616,7 +615,7 @@ ERF::InitData_post ()
 {
     if (restart_chkfile.empty()) {
         if (solverChoice.use_terrain) {
-            if (init_type == "ideal") {
+            if (init_type == InitType::Ideal) {
                 Abort("We do not currently support init_type = ideal with terrain");
             }
         }
@@ -824,7 +823,7 @@ ERF::InitData_post ()
         solverChoice.rayleigh_damp_W ||solverChoice.rayleigh_damp_T)
     {
         initRayleigh();
-        if (init_type == "input_sounding")
+        if (init_type == InitType::Input_Sounding)
         {
             // Overwrite ubar, vbar, and thetabar with input profiles;
             // wbar is assumed to be 0. Note: the tau coefficient set by
@@ -1261,7 +1260,7 @@ ERF::init_only (int lev, Real time)
     // Map the words in the inputs file to BC types, then translate
     //     those types into what they mean for each variable
     // This must be called before initHSE (where the base state is initialized)
-    if (lev == 0 && init_type != "ideal") {
+    if (lev == 0 && init_type != InitType::Ideal) {
         init_bcs();
     }
 
@@ -1278,7 +1277,7 @@ ERF::init_only (int lev, Real time)
     lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
 
     // Initialize background flow (optional)
-    if (init_type == "input_sounding") {
+    if (init_type == InitType::Input_Sounding) {
         // The base state is initialized by integrating vertically through the
         // input sounding, if the init_sounding_ideal flag is set; otherwise
         // it is set by initHSE()
@@ -1299,23 +1298,23 @@ ERF::init_only (int lev, Real time)
         }
 
 #ifdef ERF_USE_NETCDF
-    } else if (init_type == "ideal" || init_type == "real") {
+    } else if (init_type == InitType::Ideal || init_type == InitType::Real) {
         // The base state is initialized from WRF wrfinput data, output by
         // ideal.exe or real.exe
         init_from_wrfinput(lev);
 
         // The physbc's need the terrain but are needed for initHSE
-        if (init_type == "ideal") {
+        if (init_type == InitType::Ideal) {
             make_physbcs(lev);
             initHSE(lev);
         }
 
-    } else if (init_type == "metgrid") {
+    } else if (init_type == InitType::Metgrid) {
         // The base state is initialized from data output by WPS metgrid;
         // we will rebalance after interpolation
         init_from_metgrid(lev);
 #endif
-    } else if (init_type == "uniform") {
+    } else if (init_type == InitType::Uniform) {
         // Initialize a uniform background field and base state based on the
         // problem-specified reference density and temperature
 
@@ -1430,11 +1429,12 @@ ERF::ReadParameters ()
         pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
 
         // How to initialize
-        pp.query("init_type",init_type);
+        init_type = InitType::None;
+        pp.query_enum_case_insensitive("init_type",init_type);
 
         // Should we use the bcs we've read in from wrfbdy or metgrid files?
         // We default to yes if we have them, but the user can override that option
-        use_real_bcs = ( (init_type == "real") || (init_type == "metgrid") );
+        use_real_bcs = ( (init_type == InitType::Real) || (init_type == InitType::Metgrid) );
         pp.query("use_real_bcs",use_real_bcs);
 
         // We use this to keep track of how many boxes we read in from WRF initialization
@@ -1545,7 +1545,7 @@ ERF::ReadParameters ()
 
     // No moving terrain with init real (we must do this after init_params
     //    because that is where we set terrain_type
-    if (init_type == "real" && solverChoice.terrain_type == TerrainType::Moving) {
+    if (init_type == InitType::Real && solverChoice.terrain_type == TerrainType::Moving) {
         Abort("Moving terrain is not supported with init real");
     }
 
@@ -1577,8 +1577,8 @@ ERF::ParameterSanityChecks ()
 {
     AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.);
 
-    // We don't allow use_real_bcs to be true if init_type is not either real or metgrid
-    AMREX_ALWAYS_ASSERT(!use_real_bcs || ((init_type == "real") || (init_type == "metgrid")) );
+    // We don't allow use_real_bcs to be true if init_type is not either InitType::Rreal or InitType::Metgrid
+    AMREX_ALWAYS_ASSERT(!use_real_bcs || ((init_type == InitType::Real) || (init_type == InitType::Metgrid)) );
 
     AMREX_ALWAYS_ASSERT(real_width >= 0);
     AMREX_ALWAYS_ASSERT(real_set_width >= 0);
@@ -1605,17 +1605,6 @@ ERF::ParameterSanityChecks ()
         Abort("Dont know this plotfile_type");
     }
 
-    // Enforce the init_type is one we know
-    if (!init_type.empty() &&
-        init_type != "uniform" &&
-        init_type != "ideal" &&
-        init_type != "real" &&
-        init_type != "metgrid" &&
-        init_type != "input_sounding")
-    {
-        Error("if specified, init_type must be uniform, ideal, real, metgrid or input_sounding");
-    }
-
     // If fixed_mri_dt_ratio is set, it must be even
     if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
     {

Source/ERF_Tagging.cpp

@@ -223,7 +223,7 @@ ERF::refinement_criteria_setup ()
                     boxes_at_level[lev_for_box].push_back(bx);
                     Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                 } // lev
-                if (init_type == "real" || init_type == "metgrid") {
+                if (init_type == InitType::Real || init_type == InitType::Metgrid) {
                     if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                         amrex::Error("Number of boxes doesn't match number of input files");
 
@@ -261,7 +261,7 @@ ERF::refinement_criteria_setup ()
                     boxes_at_level[lev_for_box].push_back(bx);
                     Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                 } // lev
-                if (init_type == "real" || init_type == "metgrid") {
+                if (init_type == InitType::Real || init_type == InitType::Metgrid) {
                     if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                         amrex::Error("Number of boxes doesn't match number of input files");
 

Source/ERF_make_new_arrays.cpp

@@ -445,7 +445,7 @@ ERF::init_zphys (int lev, Real time)
 {
     if (solverChoice.use_terrain)
     {
-        if (init_type != "real" && init_type != "metgrid")
+        if (init_type != InitType::Real && init_type != InitType::Metgrid)
         {
             if (lev > 0) {
                 //

Source/ERF_make_new_level.cpp

@@ -47,7 +47,7 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba_in,
     // Define dmap[lev] to be dm
     SetDistributionMap(lev, dm);
 
-    // amrex::Print() <<" BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
+    amrex::Print() <<" BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
 
     if (lev == 0) init_bcs();
 
@@ -113,22 +113,22 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba_in,
 
     // ********************************************************************************************
     // Initialize the data itself
-    // If (init_type == "real") then we are initializing terrain and the initial data in
-    //                          the same call so we must call init_only before update_terrain_arrays
-    // If (init_type != "real") then we want to initialize the terrain before the initial data
-    //                          since we may need to use the grid information before constructing
-    //                          initial idealized data
+    // If (init_type == InitType::Real) then we are initializing terrain and the initial data in
+    //                                  the same call so we must call init_only before update_terrain_arrays
+    // If (init_type != InitType::Real) then we want to initialize the terrain before the initial data
+    //                                  since we may need to use the grid information before constructing
+    //                                  initial idealized data
     // ********************************************************************************************
     if (restart_chkfile.empty()) {
-        if ((init_type == "real") || (init_type == "metgrid")) {
+        if ((init_type == InitType::Real) || (init_type == InitType::Metgrid)) {
             init_only(lev, start_time);
             init_zphys(lev, time);
             update_terrain_arrays(lev);
             make_physbcs(lev);
         } else {
             init_zphys(lev, time);
             update_terrain_arrays(lev);
-            // Note that for init_type != real or metgrid,
+            // Note that for init_type != InitType::Real or InitType::Metgrid,
             // make_physbcs is called inside init_only
             init_only(lev, start_time);
         }

Source/ERF_prob_common.H

@@ -468,7 +468,7 @@ protected:
 
     /**
      * Function to update default base parameters, currently only used for
-     * init_type=='uniform'
+     * init_type == InitType::Uniform
      */
     void init_base_parms (amrex::Real rho_0, amrex::Real T_0) {
         base_parms.rho_0 = rho_0;

Source/IO/ERF_Checkpoint.cpp

@@ -221,7 +221,7 @@ ERF::WriteCheckpointFile () const
 
 #ifdef ERF_USE_NETCDF
    // Write bdy_data files
-   if (ParallelDescriptor::IOProcessor() && ((init_type=="real") || (init_type=="metgrid"))) {
+   if (ParallelDescriptor::IOProcessor() && ((init_type==InitType::Real) || (init_type==InitType::Metgrid))) {
 
      // Vector dimensions
      int num_time = bdy_data_xlo.size();
@@ -462,7 +462,7 @@ ERF::ReadCheckpointFile ()
 
 #ifdef ERF_USE_NETCDF
     // Read bdy_data files
-    if ((init_type=="real") || (init_type=="metgrid")) {
+    if ((init_type==InitType::Real) || (init_type==InitType::Metgrid)) {
         int ioproc = ParallelDescriptor::IOProcessorNumber();  // I/O rank
         int num_time;
         int num_var;

Source/Initialization/ERF_init_from_wrfinput.cpp

@@ -277,7 +277,7 @@ ERF::init_from_wrfinput (int lev)
     IntVect ng = p_hse.nGrowVect();
     const Real l_rdOcp = solverChoice.rdOcp;
 
-    if (init_type == "real") {
+    if (init_type == InitType::Real) {
         for ( MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi )
         {
             FArrayBox&   cons_fab = lev_new[Vars::cons][mfi];
@@ -297,7 +297,7 @@ ERF::init_from_wrfinput (int lev)
         pi_hse.FillBoundary(geom[lev].periodicity());
     }
 
-    if (init_type == "real" && (lev == 0)) {
+    if (init_type == InitType::Real && (lev == 0)) {
         if (nc_bdy_file.empty()) {
             amrex::Error("NetCDF boundary file name must be provided via input");
         }

Source/TimeIntegration/ERF_TI_slow_rhs_fun.H

@@ -249,8 +249,7 @@
         // Populate RHS for relaxation zones if using real bcs
         if (use_real_bcs && (level == 0)) {
             if (real_width>0) {
-                realbdy_compute_interior_ghost_rhs(init_type,
-                                                   bdy_time_interval, start_bdy_time, new_stage_time, slow_dt,
+                realbdy_compute_interior_ghost_rhs(bdy_time_interval, start_bdy_time, new_stage_time, slow_dt,
                                                    real_width, real_set_width, fine_geom,
                                                    S_rhs, S_old, S_data,
                                                    bdy_data_xlo, bdy_data_xhi,
@@ -459,8 +458,7 @@
         // Populate RHS for relaxation zones if using real bcs
         if (use_real_bcs && (level == 0)) {
             if (real_width>0) {
-                    realbdy_compute_interior_ghost_rhs(init_type,
-                                                       bdy_time_interval, start_bdy_time, new_stage_time, slow_dt,
+                    realbdy_compute_interior_ghost_rhs(bdy_time_interval, start_bdy_time, new_stage_time, slow_dt,
                                                        real_width, real_set_width, fine_geom,
                                                        S_rhs, S_old, S_data,
                                                        bdy_data_xlo, bdy_data_xhi,

Source/Utils/ERF_InteriorGhostCells.cpp

@@ -89,7 +89,6 @@ compute_interior_ghost_bxs_xy (const Box& bx,
 /**
  * Compute the RHS in the relaxation zone
  *
- * @param[in] init_type initialization method for this simulation
  * @param[in] bdy_time_interval time interval between boundary condition time stamps
  * @param[in] time    current time
  * @param[in] delta_t timestep
@@ -105,8 +104,7 @@ compute_interior_ghost_bxs_xy (const Box& bx,
  * @param[in] start_bdy_time time of the first boundary data read in
  */
 void
-realbdy_compute_interior_ghost_rhs (const std::string& /*init_type*/,
-                                    const Real& bdy_time_interval,
+realbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval,
                                     const Real& start_bdy_time,
                                     const Real& time,
                                     const Real& delta_t,

Source/Utils/ERF_Utils.H

@@ -88,8 +88,7 @@ void compute_interior_ghost_bxs_xy (const amrex::Box& bx,
 /*
  * Compute relaxation region RHS with wrfbdy
  */
-void realbdy_compute_interior_ghost_rhs (const std::string& init_type,
-                                         const amrex::Real& bdy_time_interval,
+void realbdy_compute_interior_ghost_rhs (const amrex::Real& bdy_time_interval,
                                          const amrex::Real& start_bdy_time,
                                          const amrex::Real& time,
                                          const amrex::Real& delta_t,