Merge remote-tracking branch 'origin/develop' into catchem_Apr

commit 6ca106e (origin/develop, origin/HEAD, may30, develop) Author: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Mon May 13 22:57:38 2024 +0000 Limit gfswavepostpnt to 40 PEs/node (NOAA-EMC#2588) This fixes the slow runtime of the gfswavepostpnt job on Hercules. The job is very I/O intensive and does not scale well to large nodes, so limit the number of jobs/node to 40. Resolves NOAA-EMC#2587
zhanglikate · May 17, 2024 · 8c3093d · 8c3093d
2 parents 0af86ca + c3f8766
commit 8c3093d
Show file tree

Hide file tree

Showing 56 changed files with 552 additions and 210 deletions.
diff --git a/.gitignore b/.gitignore
@@ -48,6 +48,8 @@ parm/gdas/io
 parm/gdas/ioda
 parm/gdas/snow
 parm/gdas/soca
+parm/gdas/jcb-gdas
+parm/gdas/jcb-algorithms
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
@@ -106,6 +108,7 @@ parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
+parm/ufs/model_configure_nest.IN
 parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
@@ -194,3 +197,8 @@ versions/run.ver
 ush/python/wxflow
 workflow/wxflow
 ci/scripts/wxflow
+
+# jcb checkout and symlinks
+ush/python/jcb
+workflow/jcb
+ci/scripts/jcb
diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
@@ -149,7 +149,7 @@ pipeline {
                                                 }
                                             }
                                             sh(script: './link_workflow.sh')
-                                            sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
+                                            //sh(script: "echo ${HOMEgfs} > BUILT_semaphor")
                                         }
                                     }
                                     if (env.CHANGE_ID && system == 'gfs') {

diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml
@@ -15,3 +15,6 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - wcoss2
diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml
@@ -0,0 +1,22 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 96
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  idate: 2021122018
+  edate: 2021122118
+  nens: 0
+  gfs_cyc: 4
+  start: cold
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml
+
+skip_ci_on_hosts:
+  - hera
+  - orion
+  - hercules
diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml
@@ -0,0 +1,12 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+
+base:
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  DO_GOES: "YES"
+  DO_BUFRSND: "YES"
+  DO_GEMPAK: "YES"
+  DO_AWIPS: "NO"
+  DO_NPOESS: "YES"
+  DO_GENESIS_FSU: "NO"
+  FHMAX_GFS: 384
diff --git a/env/HERA.env b/env/HERA.env
@@ -49,6 +49,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -45,6 +45,10 @@ case ${step} in
 
     export APRUN_CALCFIMS="${launcher} -n 1"
  ;;
+ "prep_emissions")
+
+    export APRUN="${launcher} -n 1"
+ ;;
  "waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")
 
     export CFP_MP="YES"

diff --git a/env/JET.env b/env/JET.env
@@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

diff --git a/env/ORION.env b/env/ORION.env
@@ -44,6 +44,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || \
     [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]] || [[ "${step}" == "wavepostbndpntbll" ]]; then
 

diff --git a/env/S4.env b/env/S4.env
@@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -31,6 +31,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then
 
     export APRUN_CALCFIMS="${launcher} -n 1"
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${launcher} -n 1"
+
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll"  ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export USE_CFP="YES"

diff --git a/gempak/ush/gfs_meta_opc_na_ver b/gempak/ush/gfs_meta_opc_na_ver
@@ -33,8 +33,8 @@ fcsthr="f00"
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
 case ${cyc} in
-    00 | 12) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 96 12 120)) ;;
-    06 | 18) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 90 12 126)) ;;
+    00 | 12) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 96 12 120)) ;;
+    06 | 18) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 90 12 126)) ;;
     *)
         echo "FATAL ERROR: Invalid cycle ${cyc} passed to ${BASH_SOURCE[0]}"
         exit 100

diff --git a/gempak/ush/gfs_meta_opc_np_ver b/gempak/ush/gfs_meta_opc_np_ver
@@ -33,8 +33,8 @@ fcsthr="f00"
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
 case ${cyc} in
-    00 | 12) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 96 12 120)) ;;
-    06 | 18) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 90 12 126)) ;;
+    00 | 12) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 96 12 120)) ;;
+    06 | 18) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 90 12 126)) ;;
     *)
         echo "FATAL ERROR: Invalid cycle ${cyc} passed to ${BASH_SOURCE[0]}"
         exit 100

diff --git a/gempak/ush/gfs_meta_ver.sh b/gempak/ush/gfs_meta_ver.sh
@@ -32,7 +32,7 @@ MDL2="GFSHPC"
 #GENERATING THE METAFILES.
 # seq won't give us any splitting problems, ignore warnings
 # shellcheck disable=SC2207,SC2312
-IFS=$'\n' lookbacks=($(seq 6 6 180) $(seq 192 12 216))
+lookbacks=($(IFS=$'\n' seq 6 6 180) $(IFS=$'\n' seq 192 12 216))
 for lookback in "${lookbacks[@]}"; do
     init_time="$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${lookback} hours")"
     init_PDY=${init_time:0:8}

diff --git a/jobs/JGLOBAL_PREP_EMISSIONS b/jobs/JGLOBAL_PREP_EMISSIONS
@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "prep_emissions" -c "base prep_emissions"
+
+##############################################
+# Set variables used in the script
+##############################################
+# TODO: Set local variables used in this script e.g. GDATE may be needed for previous cycle
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+# Generate COM variables from templates
+# TODO: Add necessary COMIN, COMOUT variables for this job
+
+###############################################################
+# Run relevant script
+EXSCRIPT=${PREP_EMISSIONS_PY:-${SCRgfs}/exglobal_prep_emissions.py}
+${EXSCRIPT}
+status=$?
+(( status != 0 )) && ( echo "FATAL ERROR: Error executing ${EXSCRIPT}, ABORT!"; exit "${status}" )
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0
diff --git a/jobs/rocoto/prep_emissions.sh b/jobs/rocoto/prep_emissions.sh
@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status !=  0 )) && exit "${status}"
+
+export job="prep_emissions"
+export jobid="${job}.$$"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python"
+export PYTHONPATH
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_PREP_EMISSIONS"
+status=$?
+exit "${status}"
diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua
@@ -39,6 +39,9 @@ load(pathJoin("met", (os.getenv("met_ver") or "None")))
 load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
 load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 
+-- Adding perl as a module; With Rocky8, perl packages will not be from the OS
+load(pathJoin("perl", (os.getenv("perl_ver") or "None")))
+
 setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 

diff --git a/modulefiles/module_gwsetup.jet.lua b/modulefiles/module_gwsetup.jet.lua
@@ -4,7 +4,7 @@ Load environment to run GFS workflow setup scripts on Jet
 
 load(pathJoin("rocoto"))
 
-prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core")
 
 local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 local python_ver=os.getenv("python_ver") or "3.11.6"

diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
@@ -47,9 +47,9 @@ export NOSCRUB="@NOSCRUB@"
 export BASE_GIT="@BASE_GIT@"
 
 # Toggle to turn on/off GFS downstream processing.
-export DO_BUFRSND="NO"     # BUFR sounding products
-export DO_GEMPAK="NO"      # GEMPAK products
-export DO_AWIPS="NO"       # AWIPS products
+export DO_BUFRSND="@DO_BUFRSND@"     # BUFR sounding products
+export DO_GEMPAK="@DO_GEMPAK@"       # GEMPAK products
+export DO_AWIPS="@DO_AWIPS@"         # AWIPS products
 
 # NO for retrospective parallel; YES for real-time parallel
 #  arch.sh uses REALTIME for MOS.  Need to set REALTIME=YES

diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst
@@ -6,6 +6,7 @@
 echo "BEGIN: config.fcst"
 
 export USE_ESMF_THREADING="YES"  # Toggle to use ESMF-managed threading or traditional threading in UFSWM
+export COPY_FINAL_RESTARTS="NO" # Toggle to copy restarts from the end of GFS/GEFS Run (GDAS is handled seperately)
 
 # Turn off waves if not used for this CDUMP
 case ${WAVE_CDUMP} in
@@ -108,18 +109,10 @@ if (( gwd_opt == 2 )); then
 fi
 
 # Sponge layer settings
-export tau=0.
-export rf_cutoff=10.
 export d2_bg_k1=0.20
 export d2_bg_k2=0.04
 export dz_min=6
 export n_sponge=42
-if (( LEVS == 128 )) && [[ "${CDUMP}" =~ "gdas" ]]; then
-   export tau=5.0
-   export rf_cutoff=1.0e3
-   export d2_bg_k1=0.20
-   export d2_bg_k2=0.0
-fi
 
 # PBL/turbulance schemes
 export hybedmf=".false."

diff --git a/parm/config/gefs/config.prep_emissions b/parm/config/gefs/config.prep_emissions
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.prep_emissions ##########
+# aerosol emissions preprocessing specific
+
+echo "BEGIN: config.prep_emissions"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" prep_emissions
+
+echo "END: config.prep_emissions"
diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
@@ -69,6 +69,14 @@ case ${step} in
     export memory_waveinit="2GB"
     ;;
 
+  "prep_emissions")
+    export wtime_prep_emissions="00:10:00"
+    export npe_prep_emissions=1
+    export nth_prep_emissions=1
+    export npe_node_prep_emissions=$(( npe_node_max / nth_prep_emissions ))
+    export memory_prep_emissions="1GB"
+    ;;
+
   "fcst" | "efcs")
     export is_exclusive=True
 
@@ -244,31 +252,45 @@ case ${step} in
     export memory_wavepostsbs="10GB"
     ;;
 
+  # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes.
+  # Limit the number of tasks/node to 40.
   "wavepostbndpnt")
     export wtime_wavepostbndpnt="01:00:00"
     export npe_wavepostbndpnt=240
     export nth_wavepostbndpnt=1
     export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt ))
-    export NTASKS=${npe_wavepostbndpnt}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostbndpnt} -gt 40 ]]; then
+        export npe_node_wavepostbndpnt=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostbndpnt}
     ;;
 
   "wavepostbndpntbll")
     export wtime_wavepostbndpntbll="01:00:00"
     export npe_wavepostbndpntbll=448
     export nth_wavepostbndpntbll=1
     export npe_node_wavepostbndpntbll=$(( npe_node_max / nth_wavepostbndpntbll ))
-    export NTASKS=${npe_wavepostbndpntbll}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostbndpntbll} -gt 40 ]]; then
+        export npe_node_wavepostbndpntbll=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostbndpntbll}
     ;;
 
   "wavepostpnt")
     export wtime_wavepostpnt="04:00:00"
     export npe_wavepostpnt=200
     export nth_wavepostpnt=1
     export npe_node_wavepostpnt=$(( npe_node_max / nth_wavepostpnt ))
-    export NTASKS=${npe_wavepostpnt}
     export is_exclusive=True
+    if [[ ${npe_node_wavepostpnt} -gt 40 ]]; then
+        export npe_node_wavepostpnt=40
+        export is_exclusive=False
+    fi
+    export NTASKS=${npe_wavepostpnt}
     ;;
 
   *)