From fd923adff6c884612c1fe228319907c56667cdab Mon Sep 17 00:00:00 2001 From: Ann Almgren Date: Thu, 17 Oct 2024 14:46:03 -0700 Subject: [PATCH 1/3] make init_type an InitType enum (#1894) --- Source/BoundaryConditions/ERF_FillPatch.cpp | 4 +- Source/ERF.H | 14 ++++-- Source/ERF.cpp | 43 +++++++------------ Source/ERF_Tagging.cpp | 4 +- Source/ERF_make_new_arrays.cpp | 2 +- Source/ERF_make_new_level.cpp | 16 +++---- Source/ERF_prob_common.H | 2 +- Source/IO/ERF_Checkpoint.cpp | 4 +- .../Initialization/ERF_init_from_wrfinput.cpp | 4 +- 9 files changed, 44 insertions(+), 49 deletions(-) diff --git a/Source/BoundaryConditions/ERF_FillPatch.cpp b/Source/BoundaryConditions/ERF_FillPatch.cpp index cce4a2977..8c7b369d4 100644 --- a/Source/BoundaryConditions/ERF_FillPatch.cpp +++ b/Source/BoundaryConditions/ERF_FillPatch.cpp @@ -208,7 +208,7 @@ ERF::FillPatch (int lev, Real time, if (m_r2d) fill_from_bndryregs(mfs_vel,time); - // We call these even if init_type == real because these will fill the vertical bcs + // We call these even if init_type == InitType::Real because these will fill the vertical bcs // Note that we call FillBoundary inside the physbcs call (*physbcs_cons[lev])(*mfs_vel[Vars::cons],icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb); if (!cons_only) { @@ -418,7 +418,7 @@ ERF::FillIntermediatePatch (int lev, Real time, if (m_r2d) fill_from_bndryregs(mfs_vel,time); - // We call this even if init_type == real because this routine will fill the vertical bcs + // We call this even if init_type == InitType::Real because this routine will fill the vertical bcs (*physbcs_cons[lev])(*mfs_vel[Vars::cons],icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb); if (!cons_only) { (*physbcs_u[lev])(*mfs_vel[Vars::xvel],0,1,ngvect_vels,time,BCVars::xvel_bc, do_fb); diff --git a/Source/ERF.H b/Source/ERF.H index e9ed7ad52..d4f19f592 100644 --- a/Source/ERF.H +++ b/Source/ERF.H @@ -66,6 +66,13 @@ class MultiBlockContainer; #endif +/** + * Enum of possible initialization types +*/ +AMREX_ENUM(InitType, + None, Input_Sounding, Ideal, Real, Metgrid, Uniform +); + /** * Enum of possible coarse/fine interpolation options */ @@ -956,13 +963,12 @@ private: // Native or NetCDF static std::string plotfile_type; - // init_type: "ideal", "real", "input_sounding", "metgrid" or "" - static std::string init_type; + static InitType init_type; // sponge_type: "input_sponge" static std::string sponge_type; - // use_real_bcs: only true if 1) ( (init_type == real) or (init_type == metgrid) ) + // use_real_bcs: only true if 1) ( (init_type == InitType::Real) or (init_type == InitType::Metgrid) ) // AND 2) we want to use the bc's from the WRF bdy file static bool use_real_bcs; @@ -975,7 +981,7 @@ private: int real_set_width{0}; // Flag to trigger initialization from input_sounding like WRF's ideal.exe - // used with init_type == "input_sounding" + // used with init_type == InitType::Input_Sounding static bool init_sounding_ideal; // 1D CDF output (for ingestion in AMR-Wind) diff --git a/Source/ERF.cpp b/Source/ERF.cpp index 427582722..590cb8e27 100644 --- a/Source/ERF.cpp +++ b/Source/ERF.cpp @@ -43,10 +43,9 @@ int ERF::pert_interval = -1; // Native AMReX vs NetCDF std::string ERF::plotfile_type = "amrex"; -// init_type: "uniform", "ideal", "real", "input_sounding", "metgrid" or "" -std::string ERF::init_type; +InitType ERF::init_type; -// use_real_bcs: only true if 1) ( (init_type == real) or (init_type == metgrid) ) +// use_real_bcs: only true if 1) ( (init_type == InitType::Real) or (init_type == InitGrid::Metgrid) ) // AND 2) we want to use the bc's from the WRF bdy file bool ERF::use_real_bcs; @@ -610,7 +609,7 @@ ERF::InitData_post () { if (restart_chkfile.empty()) { if (solverChoice.use_terrain) { - if (init_type == "ideal") { + if (init_type == InitType::Ideal) { Abort("We do not currently support init_type = ideal with terrain"); } } @@ -818,7 +817,7 @@ ERF::InitData_post () solverChoice.rayleigh_damp_W ||solverChoice.rayleigh_damp_T) { initRayleigh(); - if (init_type == "input_sounding") + if (init_type == InitType::Input_Sounding) { // Overwrite ubar, vbar, and thetabar with input profiles; // wbar is assumed to be 0. Note: the tau coefficient set by @@ -1250,7 +1249,7 @@ ERF::init_only (int lev, Real time) // Map the words in the inputs file to BC types, then translate // those types into what they mean for each variable // This must be called before initHSE (where the base state is initialized) - if (lev == 0 && init_type != "ideal") { + if (lev == 0 && init_type != InitType::Ideal) { init_bcs(); } @@ -1267,7 +1266,7 @@ ERF::init_only (int lev, Real time) lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0); // Initialize background flow (optional) - if (init_type == "input_sounding") { + if (init_type == InitType::Input_Sounding) { // The base state is initialized by integrating vertically through the // input sounding, if the init_sounding_ideal flag is set; otherwise // it is set by initHSE() @@ -1288,23 +1287,23 @@ ERF::init_only (int lev, Real time) } #ifdef ERF_USE_NETCDF - } else if (init_type == "ideal" || init_type == "real") { + } else if (init_type == InitType::Ideal || init_type == InitType::Real) { // The base state is initialized from WRF wrfinput data, output by // ideal.exe or real.exe init_from_wrfinput(lev); // The physbc's need the terrain but are needed for initHSE - if (init_type == "ideal") { + if (init_type == InitType::Ideal) { make_physbcs(lev); initHSE(lev); } - } else if (init_type == "metgrid") { + } else if (init_type == InitType::Metgrid) { // The base state is initialized from data output by WPS metgrid; // we will rebalance after interpolation init_from_metgrid(lev); #endif - } else if (init_type == "uniform") { + } else if (init_type == InitType::Uniform) { // Initialize a uniform background field and base state based on the // problem-specified reference density and temperature @@ -1419,11 +1418,12 @@ ERF::ReadParameters () pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio); // How to initialize - pp.query("init_type",init_type); + init_type = InitType::None; + pp.query_enum_case_insensitive("init_type",init_type); // Should we use the bcs we've read in from wrfbdy or metgrid files? // We default to yes if we have them, but the user can override that option - use_real_bcs = ( (init_type == "real") || (init_type == "metgrid") ); + use_real_bcs = ( (init_type == InitType::Real) || (init_type == InitType::Metgrid) ); pp.query("use_real_bcs",use_real_bcs); // We use this to keep track of how many boxes we read in from WRF initialization @@ -1530,7 +1530,7 @@ ERF::ReadParameters () // No moving terrain with init real (we must do this after init_params // because that is where we set terrain_type - if (init_type == "real" && solverChoice.terrain_type == TerrainType::Moving) { + if (init_type == InitType::Real && solverChoice.terrain_type == TerrainType::Moving) { Abort("Moving terrain is not supported with init real"); } @@ -1562,8 +1562,8 @@ ERF::ParameterSanityChecks () { AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.); - // We don't allow use_real_bcs to be true if init_type is not either real or metgrid - AMREX_ALWAYS_ASSERT(!use_real_bcs || ((init_type == "real") || (init_type == "metgrid")) ); + // We don't allow use_real_bcs to be true if init_type is not either InitType::Rreal or InitType::Metgrid + AMREX_ALWAYS_ASSERT(!use_real_bcs || ((init_type == InitType::Real) || (init_type == InitType::Metgrid)) ); AMREX_ALWAYS_ASSERT(real_width >= 0); AMREX_ALWAYS_ASSERT(real_set_width >= 0); @@ -1590,17 +1590,6 @@ ERF::ParameterSanityChecks () Abort("Dont know this plotfile_type"); } - // Enforce the init_type is one we know - if (!init_type.empty() && - init_type != "uniform" && - init_type != "ideal" && - init_type != "real" && - init_type != "metgrid" && - init_type != "input_sounding") - { - Error("if specified, init_type must be uniform, ideal, real, metgrid or input_sounding"); - } - // If fixed_mri_dt_ratio is set, it must be even if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) ) { diff --git a/Source/ERF_Tagging.cpp b/Source/ERF_Tagging.cpp index 5bfb5f08c..2203ac2bb 100644 --- a/Source/ERF_Tagging.cpp +++ b/Source/ERF_Tagging.cpp @@ -223,7 +223,7 @@ ERF::refinement_criteria_setup () boxes_at_level[lev_for_box].push_back(bx); Print() << "Saving in 'boxes at level' as " << bx << std::endl; } // lev - if (init_type == "real" || init_type == "metgrid") { + if (init_type == InitType::Real || init_type == InitType::Metgrid) { if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) { amrex::Error("Number of boxes doesn't match number of input files"); @@ -261,7 +261,7 @@ ERF::refinement_criteria_setup () boxes_at_level[lev_for_box].push_back(bx); Print() << "Saving in 'boxes at level' as " << bx << std::endl; } // lev - if (init_type == "real" || init_type == "metgrid") { + if (init_type == InitType::Real || init_type == InitType::Metgrid) { if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) { amrex::Error("Number of boxes doesn't match number of input files"); diff --git a/Source/ERF_make_new_arrays.cpp b/Source/ERF_make_new_arrays.cpp index a20f4517e..78a249141 100644 --- a/Source/ERF_make_new_arrays.cpp +++ b/Source/ERF_make_new_arrays.cpp @@ -445,7 +445,7 @@ ERF::init_zphys (int lev, Real time) { if (solverChoice.use_terrain) { - if (init_type != "real" && init_type != "metgrid") + if (init_type != InitType::Real && init_type != InitType::Metgrid) { if (lev > 0) { // diff --git a/Source/ERF_make_new_level.cpp b/Source/ERF_make_new_level.cpp index 7fbcae895..64e458e80 100644 --- a/Source/ERF_make_new_level.cpp +++ b/Source/ERF_make_new_level.cpp @@ -47,7 +47,7 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba_in, // Define dmap[lev] to be dm SetDistributionMap(lev, dm); - // amrex::Print() <<" BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl; + amrex::Print() <<" BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl; if (lev == 0) init_bcs(); @@ -113,14 +113,14 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba_in, // ******************************************************************************************** // Initialize the data itself - // If (init_type == "real") then we are initializing terrain and the initial data in - // the same call so we must call init_only before update_terrain_arrays - // If (init_type != "real") then we want to initialize the terrain before the initial data - // since we may need to use the grid information before constructing - // initial idealized data + // If (init_type == InitType::Real) then we are initializing terrain and the initial data in + // the same call so we must call init_only before update_terrain_arrays + // If (init_type != InitType::Real) then we want to initialize the terrain before the initial data + // since we may need to use the grid information before constructing + // initial idealized data // ******************************************************************************************** if (restart_chkfile.empty()) { - if ((init_type == "real") || (init_type == "metgrid")) { + if ((init_type == InitType::Real) || (init_type == InitType::Metgrid)) { init_only(lev, start_time); init_zphys(lev, time); update_terrain_arrays(lev); @@ -128,7 +128,7 @@ void ERF::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba_in, } else { init_zphys(lev, time); update_terrain_arrays(lev); - // Note that for init_type != real or metgrid, + // Note that for init_type != InitType::Real or InitType::Metgrid, // make_physbcs is called inside init_only init_only(lev, start_time); } diff --git a/Source/ERF_prob_common.H b/Source/ERF_prob_common.H index 070835c63..b9318abc2 100644 --- a/Source/ERF_prob_common.H +++ b/Source/ERF_prob_common.H @@ -468,7 +468,7 @@ protected: /** * Function to update default base parameters, currently only used for - * init_type=='uniform' + * init_type == InitType::Uniform */ void init_base_parms (amrex::Real rho_0, amrex::Real T_0) { base_parms.rho_0 = rho_0; diff --git a/Source/IO/ERF_Checkpoint.cpp b/Source/IO/ERF_Checkpoint.cpp index e67032d59..7a3452a51 100644 --- a/Source/IO/ERF_Checkpoint.cpp +++ b/Source/IO/ERF_Checkpoint.cpp @@ -221,7 +221,7 @@ ERF::WriteCheckpointFile () const #ifdef ERF_USE_NETCDF // Write bdy_data files - if (ParallelDescriptor::IOProcessor() && ((init_type=="real") || (init_type=="metgrid"))) { + if (ParallelDescriptor::IOProcessor() && ((init_type==InitType::Real) || (init_type==InitType::Metgrid))) { // Vector dimensions int num_time = bdy_data_xlo.size(); @@ -462,7 +462,7 @@ ERF::ReadCheckpointFile () #ifdef ERF_USE_NETCDF // Read bdy_data files - if ((init_type=="real") || (init_type=="metgrid")) { + if ((init_type==InitType::Real) || (init_type==InitType::Metgrid)) { int ioproc = ParallelDescriptor::IOProcessorNumber(); // I/O rank int num_time; int num_var; diff --git a/Source/Initialization/ERF_init_from_wrfinput.cpp b/Source/Initialization/ERF_init_from_wrfinput.cpp index 7c3a1d062..3e9cef97e 100644 --- a/Source/Initialization/ERF_init_from_wrfinput.cpp +++ b/Source/Initialization/ERF_init_from_wrfinput.cpp @@ -277,7 +277,7 @@ ERF::init_from_wrfinput (int lev) IntVect ng = p_hse.nGrowVect(); const Real l_rdOcp = solverChoice.rdOcp; - if (init_type == "real") { + if (init_type == InitType::Real) { for ( MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi ) { FArrayBox& cons_fab = lev_new[Vars::cons][mfi]; @@ -297,7 +297,7 @@ ERF::init_from_wrfinput (int lev) pi_hse.FillBoundary(geom[lev].periodicity()); } - if (init_type == "real" && (lev == 0)) { + if (init_type == InitType::Real && (lev == 0)) { if (nc_bdy_file.empty()) { amrex::Error("NetCDF boundary file name must be provided via input"); } From 1cbd7034660ff03ab689746a82201b95af044801 Mon Sep 17 00:00:00 2001 From: Ann Almgren Date: Fri, 18 Oct 2024 08:55:05 -0700 Subject: [PATCH 2/3] fix compile (#1895) --- Source/TimeIntegration/ERF_TI_slow_rhs_fun.H | 6 ++---- Source/Utils/ERF_InteriorGhostCells.cpp | 4 +--- Source/Utils/ERF_Utils.H | 3 +-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H b/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H index c9f963c68..994379325 100644 --- a/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H +++ b/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H @@ -249,8 +249,7 @@ // Populate RHS for relaxation zones if using real bcs if (use_real_bcs && (level == 0)) { if (real_width>0) { - realbdy_compute_interior_ghost_rhs(init_type, - bdy_time_interval, start_bdy_time, new_stage_time, slow_dt, + realbdy_compute_interior_ghost_rhs(bdy_time_interval, start_bdy_time, new_stage_time, slow_dt, real_width, real_set_width, fine_geom, S_rhs, S_old, S_data, bdy_data_xlo, bdy_data_xhi, @@ -459,8 +458,7 @@ // Populate RHS for relaxation zones if using real bcs if (use_real_bcs && (level == 0)) { if (real_width>0) { - realbdy_compute_interior_ghost_rhs(init_type, - bdy_time_interval, start_bdy_time, new_stage_time, slow_dt, + realbdy_compute_interior_ghost_rhs(bdy_time_interval, start_bdy_time, new_stage_time, slow_dt, real_width, real_set_width, fine_geom, S_rhs, S_old, S_data, bdy_data_xlo, bdy_data_xhi, diff --git a/Source/Utils/ERF_InteriorGhostCells.cpp b/Source/Utils/ERF_InteriorGhostCells.cpp index 1cdfafcc7..76a57beea 100644 --- a/Source/Utils/ERF_InteriorGhostCells.cpp +++ b/Source/Utils/ERF_InteriorGhostCells.cpp @@ -89,7 +89,6 @@ compute_interior_ghost_bxs_xy (const Box& bx, /** * Compute the RHS in the relaxation zone * - * @param[in] init_type initialization method for this simulation * @param[in] bdy_time_interval time interval between boundary condition time stamps * @param[in] time current time * @param[in] delta_t timestep @@ -105,8 +104,7 @@ compute_interior_ghost_bxs_xy (const Box& bx, * @param[in] start_bdy_time time of the first boundary data read in */ void -realbdy_compute_interior_ghost_rhs (const std::string& /*init_type*/, - const Real& bdy_time_interval, +realbdy_compute_interior_ghost_rhs (const Real& bdy_time_interval, const Real& start_bdy_time, const Real& time, const Real& delta_t, diff --git a/Source/Utils/ERF_Utils.H b/Source/Utils/ERF_Utils.H index 0362866b9..f713347f2 100644 --- a/Source/Utils/ERF_Utils.H +++ b/Source/Utils/ERF_Utils.H @@ -88,8 +88,7 @@ void compute_interior_ghost_bxs_xy (const amrex::Box& bx, /* * Compute relaxation region RHS with wrfbdy */ -void realbdy_compute_interior_ghost_rhs (const std::string& init_type, - const amrex::Real& bdy_time_interval, +void realbdy_compute_interior_ghost_rhs (const amrex::Real& bdy_time_interval, const amrex::Real& start_bdy_time, const amrex::Real& time, const amrex::Real& delta_t, From 6a3cdfbe82b7f2a4dca151a862a544873fe64563 Mon Sep 17 00:00:00 2001 From: Ann Almgren Date: Fri, 18 Oct 2024 10:27:46 -0700 Subject: [PATCH 3/3] include fewer files with POISSON build (#1896) --- CMake/SetAmrexOptions.cmake | 2 ++ Exec/Make.ERF | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMake/SetAmrexOptions.cmake b/CMake/SetAmrexOptions.cmake index c902bef72..9c43e753a 100644 --- a/CMake/SetAmrexOptions.cmake +++ b/CMake/SetAmrexOptions.cmake @@ -30,6 +30,8 @@ set(AMReX_FORTRAN OFF) set(AMReX_LINEAR_SOLVERS OFF) if(ERF_ENABLE_POISSON_SOLVE) set(AMReX_LINEAR_SOLVERS ON) + set(AMReX_LINEAR_SOLVERS_EM OFF) + set(AMReX_LINEAR_SOLVERS_INCFLO OFF) endif() set(AMReX_PARTICLES OFF) diff --git a/Exec/Make.ERF b/Exec/Make.ERF index 6c8a8328e..a704e7b69 100644 --- a/Exec/Make.ERF +++ b/Exec/Make.ERF @@ -112,7 +112,9 @@ AMReXdirs += EB endif ifeq ($(USE_POISSON_SOLVE),TRUE) -AMReXdirs += LinearSolvers/MLMG +USE_LINEAR_SOLVERS_INCFLO = FALSE +USE_LINEAR_SOLVERS_EM = FALSE +AMReXdirs += LinearSolvers endif ifeq ($(USE_HDF5),TRUE)