[develop] Adding Hercules as a Tier-1 platform (#911)

Modulefiles and other configuration files to adapt the SRW to Hercules system at MSU.

---------

Co-authored-by: Natalie Perlin <nperlin@hercules-login-1.hpc.msstate.edu>
Co-authored-by: Natalie Perlin <nperlin@hercules-login-4.hpc.msstate.edu>
Co-authored-by: Natalie Perlin <nperlin@hercules-login-2.hpc.msstate.edu>

.cicd/Jenkinsfile

@@ -10,11 +10,11 @@ pipeline {
     parameters {
         // Allow job runner to filter based on platform
         // Use the line below to enable all PW clusters
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
         // Use the line below to enable the PW AWS cluster
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         // Uncomment the following line to re-enable comprehensive tests
@@ -77,8 +77,8 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
-                        values 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'hercules'//, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        values 'gaea', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                     }
 
                     axis {
@@ -92,7 +92,7 @@ pipeline {
                     exclude {
                             axis {
                             name 'SRW_PLATFORM'
-                            values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                            values 'gaea', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                         }
 
                         axis {

etc/lmod-setup.csh

@@ -6,7 +6,7 @@ Usage: source etc/lmod-setup.csh PLATFORM
 
 OPTIONS:
    PLATFORM - name of machine you are building on
-      (e.g. cheyenne | hera | jet | orion | wcoss2 )
+      (e.g. cheyenne | hera | jet | orion | hercules | wcoss2 )
 EOF_USAGE
    exit 1
 else

etc/lmod-setup.sh

@@ -7,7 +7,7 @@ Usage: source etc/lmod-setup.sh PLATFORM
 
 OPTIONS:
    PLATFORM - name of machine you are building on
-      (e.g.  cheyenne | hera | jet | orion | wcoss2 )
+      (e.g. cheyenne | hera | jet | orion | hercules | wcoss2 )
 EOF_USAGE
    exit 1
 else

modulefiles/build_hercules_intel.lua

@@ -0,0 +1,30 @@
+help([[
+This module loads libraries for building the UFS SRW App on
+the MSU machine Hercules using intel-oneapi-compilers/2022.2.1
+]])
+
+whatis([===[Loads libraries needed for building the UFS SRW App on Orion ]===])
+
+load("contrib")
+load("noaatools")
+
+load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3"))
+
+prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/hpc-stack/intel-2022.2.1/modulefiles/stack")
+load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
+load(pathJoin("hpc-intel-oneapi-compilers", os.getenv("hpc_intel_ver") or "2022.2.1"))
+load(pathJoin("hpc-intel-oneapi-mpi", os.getenv("hpc_mpi_ver") or "2021.7.1"))
+
+load("srw_common")
+
+load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
+
+setenv("CFLAGS","-diag-disable=10441")
+setenv("FFLAGS","-diag-disable=10441")
+
+setenv("CMAKE_C_COMPILER","mpiicc")
+setenv("CMAKE_CXX_COMPILER","mpiicpc")
+setenv("CMAKE_Fortran_COMPILER","mpiifort")
+setenv("CMAKE_Platform","hercules.intel")
+

modulefiles/srw_common.lua

@@ -4,13 +4,13 @@ load("libpng/1.6.37")
 
 load_any("netcdf/4.9.2", "netcdf-c/4.9.2")
 load_any("netcdf/4.9.2", "netcdf-fortran/4.6.0")
-load_any("pio/2.5.10","parallelio/2.5.10")
+load_any("pio/2.5.10", "parallelio/2.5.9")
 load("esmf/8.4.2")
 load("fms/2023.01")
 
 load("bacio/2.4.1")
-load("crtm/2.4.0")
 load("g2/3.4.5")
+load("crtm/2.4.0")
 load("g2tmpl/1.10.2")
 load("ip/3.3.3")
 load("sp/2.3.3")

modulefiles/tasks/hercules/aqm_ics.local.lua

@@ -0,0 +1,2 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load("miniconda_regional_workflow_cmaq")

modulefiles/tasks/hercules/aqm_lbcs.local.lua

@@ -0,0 +1 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))

modulefiles/tasks/hercules/fire_emission.local.lua

@@ -0,0 +1,2 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load("miniconda_regional_workflow_cmaq")

modulefiles/tasks/hercules/miniconda_regional_workflow_cmaq.lua

@@ -0,0 +1,4 @@
+prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
+load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
+
+setenv("SRW_ENV", "regional_workflow_cmaq")

modulefiles/tasks/hercules/nexus_emission.local.lua

@@ -0,0 +1,2 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load("miniconda_regional_workflow_cmaq")

modulefiles/tasks/hercules/nexus_post_split.local.lua

@@ -0,0 +1,2 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load("miniconda_regional_workflow_cmaq")

modulefiles/tasks/hercules/plot_allvars.local.lua

@@ -0,0 +1,5 @@
+unload("python")
+append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
+load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
+
+setenv("SRW_ENV", "regional_workflow")

modulefiles/tasks/hercules/point_source.local.lua

@@ -0,0 +1 @@
+load("miniconda_regional_workflow_cmaq")

modulefiles/tasks/hercules/python_srw.lua

@@ -0,0 +1,5 @@
+unload("python")
+append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
+load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
+
+setenv("SRW_ENV", "workflow_tools")

modulefiles/tasks/hercules/run_vx.local.lua

@@ -0,0 +1,6 @@
+--[[
+Compiler-specific modules are used for met and metplus libraries
+--]]
+load(pathJoin("met", os.getenv("met_ver") or "10.1.2"))
+load(pathJoin("metplus", os.getenv("metplus_ver") or "4.1.3"))
+load("python_srw")

modulefiles/wflow_hercules.lua

@@ -0,0 +1,21 @@
+help([[
+This module loads python environement for running SRW on
+the MSU machine Hercules
+]])
+
+whatis([===[Loads libraries needed for running SRW on Hercules ]===])
+
+load("contrib")
+load("rocoto")
+load("set_pythonpath")
+
+unload("python")
+append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
+load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
+
+if mode() == "load" then
+   LmodMsgRaw([===[Please do the following to activate conda:
+       > conda activate workflow_tools
+]===])
+end
+

tests/WE2E/machine_suites/comprehensive.hercules

@@ -0,0 +1 @@
+comprehensive.orion

tests/build.sh

@@ -21,7 +21,7 @@ function usage() {
   exit 1
 }
 
-machines=( hera jet cheyenne derecho orion wcoss2 gaea odin singularity macos noaacloud )
+machines=( hera jet cheyenne derecho orion hercules wcoss2 gaea odin singularity macos noaacloud )
 
 [[ $# -gt 4 ]] && usage
 

ush/machine/hercules.yaml

@@ -0,0 +1,45 @@
+platform:
+  WORKFLOW_MANAGER: rocoto
+  NCORES_PER_NODE: 80
+  SCHED: slurm
+  TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc
+  TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc
+  TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc
+  DOMAIN_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen
+  PARTITION_DEFAULT: hercules
+  QUEUE_DEFAULT: batch
+  PARTITION_FCST: hercules
+  QUEUE_FCST: batch
+  PARTITION_HPSS: service
+  QUEUE_HPSS: batch
+  RUN_CMD_FCST: srun --export=ALL -n ${PE_MEMBER01}
+  RUN_CMD_POST: srun --export=ALL
+  RUN_CMD_PRDGEN: srun --export=ALL
+  RUN_CMD_SERIAL: time
+  RUN_CMD_UTILS: srun --export=ALL -n $nprocs
+  RUN_CMD_NEXUS: srun --export=ALL
+  RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS}
+  SCHED_NATIVE_CMD: --export=NONE
+  PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
+  TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data
+  TEST_AQM_INPUT_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/aqm_data
+  TEST_PREGEN_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen
+  TEST_VX_FCST_INPUT_BASEDIR: '{{ "/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/output_data/fcst_" }}{{ "ens" if (global.NUM_ENS_MEMBERS > 0) else "det" }}{{ "/{{workflow.PREDEF_GRID_NAME}}" }}{% raw %}{% endraw %}'
+  FIXaer: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_aer
+  FIXgsi: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_gsi
+  FIXgsm: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_am
+  FIXlut: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_lut
+  FIXorg: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog
+  FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo
+  FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth
+  EXTRN_MDL_DATA_STORES: aws
+data:
+  ics_lbcs:
+    FV3GFS:
+      nemsio: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh}
+      grib2: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh}
+      netcdf: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/${yyyymmdd}${hh}
+    NAM: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/NAM/${yyyymmdd}${hh}
+    HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh}
+    RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh}
+    GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh}

ush/valid_param_vals.yaml

@@ -4,7 +4,7 @@
 valid_vals_RUN_ENVIR: ["nco", "community"]
 valid_vals_VERBOSE: [True, False]
 valid_vals_DEBUG: [True, False]
-valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"]
+valid_vals_MACHINE: ["HERA", "WCOSS2", "ORION", "HERCULES", "JET", "ODIN", "CHEYENNE", "DERECHO", "STAMPEDE", "LINUX", "MACOS", "NOAACLOUD", "SINGULARITY", "GAEA"]
 valid_vals_SCHED: ["slurm", "pbspro", "lsf", "lsfcray", "none"]
 valid_vals_FCST_MODEL: ["ufs-weather-model"]
 valid_vals_WORKFLOW_MANAGER: ["rocoto", "ecflow", "none"]