[develop] Enable run_post to work with varying forecast length hours

devbuild.sh

@@ -217,12 +217,11 @@ if [ "${DEFAULT_BUILD}" = true ]; then
   BUILD_UPP="on"
 fi
 
-# Choose components to build for Online-CMAQ
+# Choose components to build for air quality modeling (Online-CMAQ)
 if [ "${APPLICATION}" = "ATMAQ" ]; then
   if [ "${DEFAULT_BUILD}" = true ]; then
     BUILD_NEXUS="on"
     BUILD_AQM_UTILS="on"
-    BUILD_UPP="off"
   fi
   if [ "${PLATFORM}" = "wcoss2" ]; then
     BUILD_POST_STAT="on"

jobs/JREGIONAL_RUN_POST

@@ -132,6 +132,27 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #
 #-----------------------------------------------------------------------
 #
+# Create a flag file to make rocoto aware that the run_post task has
+# successfully completed. This flag is necessary for varying forecast
+# hours (FCST_LEN_HRS: -1)
+#
+#-----------------------------------------------------------------------
+#
+if [ "${FCST_LEN_HRS}" = "-1" ]; then
+  for i_cdate in "${!ALL_CDATES[@]}"; do
+    if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
+      break
+    fi
+  done
+  fcst_len_hrs=$( printf "%03d" "${FCST_LEN_HRS}" )
+  if [ "${fhr}" = "${fcst_len_hrs}" ]; then
+    touch "${COMIN}/${TN_RUN_POST}_${PDY}${cyc}_task_complete.txt"
+  fi
+fi
+#
+#-----------------------------------------------------------------------
+#
 # Run job postamble.
 #
 #-----------------------------------------------------------------------

modulefiles/build_hera_intel.lua

@@ -24,6 +24,8 @@ load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2"))
 
 load("srw_common")
 
+unload("netcdf")
+load(pathJoin("netcdf", os.getenv("netcdf_ver") or "4.7.4"))
 load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.8.9.0"))
 load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
 

modulefiles/tasks/wcoss2/nexus_gfs_sfc.local.lua

@@ -0,0 +1 @@
+load("python_regional_workflow")

modulefiles/tasks/wcoss2/run_post.local.lua

@@ -1,3 +1,14 @@
 load("python_regional_workflow")
 
+unload("PrgEnv-intel")
+unload("netcdf")
+load(pathJoin("envvar", os.getenv("envvar_ver")))
+load(pathJoin("intel", os.getenv("intel_ver")))
+load(pathJoin("PrgEnv-intel", os.getenv("PrgEnv_intel_ver")))
+load(pathJoin("craype", os.getenv("craype_ver")))
+load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver")))
 load(pathJoin("cray-pals", os.getenv("cray_pals_ver")))
+load(pathJoin("hdf5", os.getenv("hdf5_ver")))
+load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+load(pathJoin("libjpeg", os.getenv("libjpeg_ver")))
+load(pathJoin("prod_util", os.getenv("prod_util_ver")))

parm/FV3LAM_wflow.xml

@@ -36,6 +36,12 @@ Workflow task names.
 <!ENTITY TN_MAKE_LBCS                              "{{ tn_make_lbcs }}">
 <!ENTITY TN_RUN_FCST                               "{{ tn_run_fcst }}">
 <!ENTITY TN_RUN_POST                               "{{ tn_run_post }}">
+{%- if fcst_len_hrs == -1 %}
+<!ENTITY TN_RUN_POST_CYC1                          "{{ tn_run_post }}_c1">
+<!ENTITY TN_RUN_POST_CYC2                          "{{ tn_run_post }}_c2">
+<!ENTITY TN_RUN_POST_CYC3                          "{{ tn_run_post }}_c3">
+<!ENTITY TN_RUN_POST_CYC4                          "{{ tn_run_post }}_c4">
+{%- endif%}
 <!ENTITY TN_PLOT_ALLVARS                           "{{ tn_plot_allvars }}">
 <!ENTITY GET_OBS                                   "{{ get_obs }}">
 <!ENTITY TN_GET_OBS_CCPA                           "{{ tn_get_obs_ccpa }}">
@@ -209,6 +215,20 @@ tasks; and the "FCST" type is used for the TN_RUN_FCST task.
   <cycledef group="cycled">
     {{- cycl_next ~ " " ~ date_last_cycl ~ " " ~ cycl_freq -}}
   </cycledef>
+{%- endif %}
+{%- if fcst_len_hrs == -1 %}
+  {%- if num_fcst_len_cycl >= 1 %}
+  <cycledef group="cyc_1st"> {{- date_first_cycl ~ " " ~ date_1st_last_cycl ~ " " ~ "24:00:00" -}} </cycledef>
+  {%- endif %}
+  {%- if num_fcst_len_cycl >= 2 %}
+  <cycledef group="cyc_2nd"> {{- date_2nd_cycl ~ " " ~ date_2nd_last_cycl ~ " " ~ "24:00:00" -}} </cycledef>
+  {%- endif %}
+  {%- if num_fcst_len_cycl >= 3 %}
+  <cycledef group="cyc_3rd"> {{- date_3rd_cycl ~ " " ~ date_3rd_last_cycl ~ " " ~ "24:00:00" -}} </cycledef>
+  {%- endif %}
+  {%- if num_fcst_len_cycl >= 4 %}
+  <cycledef group="cyc_4th"> {{- date_4th_cycl ~ " " ~ date_4th_last_cycl ~ " " ~ "24:00:00" -}} </cycledef>
+  {%- endif %}
 {%- endif %}
   <log>
   {%- if run_envir == "nco"  %}
@@ -357,7 +377,7 @@ MODULES_RUN_TASK_FP script.
   {%- endif %}
     <command>&LOAD_MODULES_RUN_TASK_FP; "&TN_NEXUS_GFS_SFC;" "&JOBSdir;/JREGIONAL_NEXUS_GFS_SFC"</command>
     <nodes>{{ nnodes_nexus_gfs_sfc }}:ppn={{ ppn_nexus_gfs_sfc }}</nodes>
-    {%- if machine not in ["GAEA", "NOAACLOUD"]  %}
+  {%- if machine not in ["GAEA", "NOAACLOUD"]  %}
     {%- if not do_real_time %}   
     <memory>{{ mem_nexus_gfs_sfc }}</memory>
     {%- endif %}
@@ -867,6 +887,60 @@ MODULES_RUN_TASK_FP script.
 ************************************************************************
 -->
 {%- if run_task_run_post %}
+{%- if fcst_len_hrs == -1 %}
+  {%- for icyc in range(num_fcst_len_cycl) %}
+  <metatask name="&TN_RUN_POST;{{ uscore_ensmem_name }}">
+    <var name="fhr">{% for h in range(0, fcst_len_cycl[icyc]+1) %}{{ " %03d" % h }}{% endfor %}</var>
+      {%- if icyc == 0 %}
+      <task name="&TN_RUN_POST_CYC1;{{ uscore_ensmem_name }}_f#fhr#" cycledefs="cyc_1st" maxtries="{{ maxtries_run_post }}">
+      {%- elif icyc == 1 %}
+      <task name="&TN_RUN_POST_CYC2;{{ uscore_ensmem_name }}_f#fhr#" cycledefs="cyc_2nd" maxtries="{{ maxtries_run_post }}">
+      {%- elif icyc == 2 %}
+      <task name="&TN_RUN_POST_CYC3;{{ uscore_ensmem_name }}_f#fhr#" cycledefs="cyc_3rd" maxtries="{{ maxtries_run_post }}">
+      {%- elif icyc == 3 %}
+      <task name="&TN_RUN_POST_CYC4;{{ uscore_ensmem_name }}_f#fhr#" cycledefs="cyc_4th" maxtries="{{ maxtries_run_post }}">
+      {%- endif %}
+        &RSRV_DEFAULT;
+        <command>&LOAD_MODULES_RUN_TASK_FP; "&TN_RUN_POST;" "&JOBSdir;/JREGIONAL_RUN_POST"</command>
+        <nodes>{{ nnodes_run_post }}:ppn={{ ppn_run_post }}</nodes>
+        <walltime>{{ wtime_run_post }}</walltime>
+        <nodesize>&NCORES_PER_NODE;</nodesize>
+        <native>&SCHED_NATIVE_CMD;</native>
+      {%- if icyc == 0 %}
+        <jobname>&TN_RUN_POST_CYC1;{{ uscore_ensmem_name }}_f#fhr#</jobname>
+      {%- elif icyc == 1 %}
+        <jobname>&TN_RUN_POST_CYC2;{{ uscore_ensmem_name }}_f#fhr#</jobname>
+      {%- elif icyc == 2 %}
+        <jobname>&TN_RUN_POST_CYC3;{{ uscore_ensmem_name }}_f#fhr#</jobname>
+      {%- elif icyc == 3 %}
+        <jobname>&TN_RUN_POST_CYC4;{{ uscore_ensmem_name }}_f#fhr#</jobname>
+      {%- endif %}
+	<join>&LOGDIR;/&TN_RUN_POST;{{ uscore_ensmem_name }}_f#fhr#<cyclestr>_@Y@m@d@H</cyclestr>&LOGEXT;</join>
+        <envar><name>GLOBAL_VAR_DEFNS_FP</name><value>&GLOBAL_VAR_DEFNS_FP;</value></envar>
+        <envar><name>USHdir</name><value>&USHdir;</value></envar>
+        <envar><name>PDY</name><value><cyclestr>@Y@m@d</cyclestr></value></envar>
+        <envar><name>cyc</name><value><cyclestr>@H</cyclestr></value></envar>
+        <envar><name>subcyc</name><value><cyclestr>@M</cyclestr></value></envar>
+        <envar><name>LOGDIR</name><value>&LOGDIR;</value></envar>
+        <envar><name>fhr</name><value>#fhr#</value></envar>
+        <envar><name>SLASH_ENSMEM_SUBDIR</name><value><cyclestr>{{ slash_ensmem_subdir }}</cyclestr></value></envar>
+        <envar><name>ENSMEM_INDX</name><value><cyclestr>#{{ ensmem_indx_name }}#</cyclestr></value></envar>
+
+        <dependency>
+          <or>
+            <taskdep task="&TN_RUN_FCST;{{ uscore_ensmem_name }}"/>
+            <and>
+              <datadep age="05:00">&DYN_DIR;f#fhr#.nc</datadep>
+              <datadep age="05:00">&PHY_DIR;f#fhr#.nc</datadep>
+            </and>
+          </or>
+        </dependency>
+
+      </task>
+  </metatask>
+  {%- endfor %}
+
+{%- else %}
   {%- if sub_hourly_post %}
 <!--
 Define the post-processing task for first model output time.  The forecast 
@@ -1113,36 +1187,6 @@ the <task> tag to be identical to the ones above for other output times.
   </metatask>
   {%- endif %}
 {%- endif %}
-{%- if run_task_pre_post_stat %}
-<!--
-************************************************************************
-************************************************************************
--->
-  <task name="&TN_PRE_POST_STAT;" cycledefs="forecast" maxtries="{{ maxtries_pre_post_stat }}">
-    &RSRV_DEFAULT;
-    <command>&LOAD_MODULES_RUN_TASK_FP; "&TN_PRE_POST_STAT;" "&JOBSdir;/JREGIONAL_PRE_POST_STAT"</command>
-    <nodes>{{ nnodes_pre_post_stat }}:ppn={{ ppn_pre_post_stat }}</nodes>
-    <walltime>{{ wtime_pre_post_stat }}</walltime>
-    <nodesize>&NCORES_PER_NODE;</nodesize>
-    <jobname>&TN_PRE_POST_STAT;</jobname>
-    <join>&LOGDIR;/&TN_PRE_POST_STAT;<cyclestr>_@Y@m@d@H&LOGEXT;</cyclestr></join>
-
-    <envar><name>GLOBAL_VAR_DEFNS_FP</name><value>&GLOBAL_VAR_DEFNS_FP;</value></envar>
-    <envar><name>USHdir</name><value>&USHdir;</value></envar>    
-    <envar><name>PDY</name><value><cyclestr>@Y@m@d</cyclestr></value></envar>
-    <envar><name>cyc</name><value><cyclestr>@H</cyclestr></value></envar>
-    <envar><name>subcyc</name><value><cyclestr>@M</cyclestr></value></envar> 
-    <envar><name>LOGDIR</name><value>&LOGDIR;</value></envar>    
-    <envar><name>SLASH_ENSMEM_SUBDIR</name><value><cyclestr>{{ slash_ensmem_subdir }}</cyclestr></value></envar>
-
-    <dependency>
-{%- if run_task_run_post %}
-      <metataskdep metatask="&TN_RUN_POST;{{ uscore_ensmem_name }}"/>
-{%- else %}
-      <taskdep task="&TN_RUN_FCST;{{ uscore_ensmem_name }}"/>
-{%- endif %}
-    </dependency>
-  </task>
 {%- endif %}
 {%- if run_task_plot_allvars %}
 <!--
@@ -1189,6 +1233,41 @@ it, we'd have to include more jinja if-statements here.
 
   </task>
 {%- endif %}
+{%- if run_task_pre_post_stat %}
+<!--
+************************************************************************
+************************************************************************
+-->
+  <task name="&TN_PRE_POST_STAT;" cycledefs="forecast" maxtries="{{ maxtries_pre_post_stat }}">
+    &RSRV_DEFAULT;
+    <command>&LOAD_MODULES_RUN_TASK_FP; "&TN_PRE_POST_STAT;" "&JOBSdir;/JREGIONAL_PRE_POST_STAT"</command>
+    <nodes>{{ nnodes_pre_post_stat }}:ppn={{ ppn_pre_post_stat }}</nodes>
+    <walltime>{{ wtime_pre_post_stat }}</walltime>
+    <nodesize>&NCORES_PER_NODE;</nodesize>
+    <jobname>&TN_PRE_POST_STAT;</jobname>
+    <join>&LOGDIR;/&TN_PRE_POST_STAT;<cyclestr>_@Y@m@d@H&LOGEXT;</cyclestr></join>
+
+    <envar><name>GLOBAL_VAR_DEFNS_FP</name><value>&GLOBAL_VAR_DEFNS_FP;</value></envar>
+    <envar><name>USHdir</name><value>&USHdir;</value></envar>    
+    <envar><name>PDY</name><value><cyclestr>@Y@m@d</cyclestr></value></envar>
+    <envar><name>cyc</name><value><cyclestr>@H</cyclestr></value></envar>
+    <envar><name>subcyc</name><value><cyclestr>@M</cyclestr></value></envar> 
+    <envar><name>LOGDIR</name><value>&LOGDIR;</value></envar>    
+    <envar><name>SLASH_ENSMEM_SUBDIR</name><value><cyclestr>{{ slash_ensmem_subdir }}</cyclestr></value></envar>
+
+    <dependency>
+{%- if run_task_run_post %}
+  {%- if fcst_len_hrs == -1 %}
+      <datadep age="00:00:00:05"><cyclestr>&COMIN_DIR;</cyclestr>/&TN_RUN_POST;<cyclestr>_@Y@m@d@H</cyclestr>&CMPEXT;</datadep>
+  {%- else %}
+      <metataskdep metatask="&TN_RUN_POST;{{ uscore_ensmem_name }}"/>
+  {%- endif %}
+{%- else %}
+      <taskdep task="&TN_RUN_FCST;{{ uscore_ensmem_name }}"/>
+{%- endif %}
+    </dependency>
+  </task>
+{%- endif %}
 {%- if run_task_post_stat_o3 %}
 <!--
 ************************************************************************

scripts/exregional_aqm_lbcs.sh

@@ -97,7 +97,7 @@ mm="${PDY:4:2}"
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done

scripts/exregional_bias_correction_o3.sh

@@ -109,7 +109,7 @@ fi
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done

scripts/exregional_bias_correction_pm25.sh

@@ -109,7 +109,7 @@ fi
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done

scripts/exregional_get_extrn_mdl_files.sh

@@ -74,7 +74,7 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then
   if [ "${FCST_LEN_HRS}" = "-1" ]; then
     for i_cdate in "${!ALL_CDATES[@]}"; do
       if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-        FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+        FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
         break
       fi
     done

scripts/exregional_make_lbcs.sh

@@ -105,7 +105,7 @@ cd_vrfy $DATA
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done

scripts/exregional_nexus_emission.sh

@@ -139,7 +139,7 @@ NUM_SPLIT_NEXUS=$( printf "%02d" ${NUM_SPLIT_NEXUS} )
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi      
   done

scripts/exregional_nexus_gfs_sfc.sh

@@ -66,7 +66,7 @@ hh=${GFS_SFC_CDATE:8:2}
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done

scripts/exregional_nexus_post_split.sh

@@ -73,7 +73,7 @@ NUM_SPLIT_NEXUS=$( printf "%02d" ${NUM_SPLIT_NEXUS} )
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done

scripts/exregional_point_source.sh

@@ -56,7 +56,7 @@ eval ${PRE_TASK_CMDS}
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done

scripts/exregional_post_stat_o3.sh

@@ -117,7 +117,7 @@ POST_STEP
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done

scripts/exregional_pre_post_stat.sh

@@ -8,7 +8,7 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task "task_pre_post" ${GLOBAL_VAR_DEFNS_FP}
+source_config_for_task "task_pre_post|task_run_post" ${GLOBAL_VAR_DEFNS_FP}
 #
 #-----------------------------------------------------------------------
 #
@@ -67,10 +67,13 @@ cd_vrfy $DATA
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done
+  if [ "${RUN_TASK_RUN_POST}" = "TRUE" ]; then
+    rm_vrfy -f "${COMIN}/${TN_RUN_POST}_${PDY}${cyc}_task_complete.txt"
+  fi
 fi
 
 ist=1

scripts/exregional_run_fcst.sh

@@ -78,7 +78,7 @@ fi
 if [ "${FCST_LEN_HRS}" = "-1" ]; then
   for i_cdate in "${!ALL_CDATES[@]}"; do
     if [ "${ALL_CDATES[$i_cdate]}" = "${PDY}${cyc}" ]; then
-      FCST_LEN_HRS="${FCST_LEN_CYCL[$i_cdate]}"
+      FCST_LEN_HRS="${FCST_LEN_CYCL_ALL[$i_cdate]}"
       break
     fi
   done
@@ -422,7 +422,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then
 ===================================================================="
   else
     if [ "${CPL_AQM}" = "TRUE" ]; then
-      post_config_fp="${HOMEdir}/sorc/AQM-utils/parm/postxconfig-NT-fv3lam_cmaq.txt"
+      post_config_fp="${PARMdir}/upp/postxconfig-NT-AQM.txt"
     else
       post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam.txt"
     fi
@@ -436,7 +436,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then
   cp_vrfy ${PARMdir}/upp/params_grib2_tbl_new .
   # Set itag for inline-post:
   if [ "${CPL_AQM}" = "TRUE" ]; then
-    post_itag_add="aqfcmaq_on=.true.,"
+    post_itag_add="aqf_on=.true.,"
   else
     post_itag_add=""
   fi