Merge pull request NOAA-GSL#46 from kayeekayee/merge_rt_feb29_a41101a…

…_dev_mar19_db6a85e Update realtime branch, gsl_ufs_rt (Merge rt feb29 a41101a and dev mar19 db6a85e )
kayeekayee · Mar 20, 2024 · 01befb6 · 01befb6
2 parents a41101a + 1be9d60
commit 01befb6
Show file tree

Hide file tree

Showing 96 changed files with 2,604 additions and 1,172 deletions.
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -17,7 +17,7 @@ on:
 jobs:
 
   documentation:
-    
+
     permissions:
       pull-requests: 'write'
 
@@ -56,7 +56,8 @@ jobs:
         path: artifact/doc_warnings.log
         if-no-files-found: ignore
 
-    - name: Comment ReadDocs
+    - name: Comment ReadDocs Link in PR
+      if: github.event_name == 'pull_request'
       uses: actions/github-script@v6
       with:
         script: |
@@ -68,5 +69,5 @@ jobs:
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
-           body: message 
-          })     
+           body: message
+          })
diff --git a/.gitmodules b/.gitmodules
@@ -1,6 +1,6 @@
 [submodule "sorc/ufs_model.fd"]
 	path = sorc/ufs_model.fd
-	url = https://github.com/ufs-community/ufs-weather-model
+	url = https://github.com/NOAA-GSL/ufs-weather-model
 	ignore = dirty
 [submodule "sorc/wxflow"]
 	path = sorc/wxflow
@@ -10,7 +10,7 @@
 	url = https://github.com/NOAA-EMC/gfs-utils
 [submodule "sorc/ufs_utils.fd"]
 	path = sorc/ufs_utils.fd
-	url = https://github.com/ufs-community/UFS_UTILS.git
+	url = https://github.com/NOAA-GSL/UFS_UTILS.git
 [submodule "sorc/verif-global.fd"]
 	path = sorc/verif-global.fd
 	url = https://github.com/NOAA-EMC/EMC_verif-global.git

diff --git a/INFO b/INFO
@@ -0,0 +1,19 @@
+03-19-24
+========
+   12Jan24 global-workflow
+   UFS: 29Jan24, 625ac02
+   FV3: 29Jan24, bd38c56 (GSL: 28Feb24 , a439cc7)
+   UPP: 07Nov23, 78f369b
+   UFS_UTILS: 22Dev23, ce385ce
+
+   update pointers for ufs_model and ufs_utils
+   add GSL C3_MYNN xml suite
+
+01-08-24
+========
+  02Jan24 global-workflow
+  UFS: 21Dec23, 991d652
+  UPP:  07Nov23, 78f369b
+
+  use submodules instead of checkout.sh
+  gfsatmos_products replaces gfspost tasks
diff --git a/ci/cases/pr/C48_ATM.yaml b/ci/cases/pr/C48_ATM.yaml
@@ -5,8 +5,8 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 48
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312

diff --git a/ci/cases/pr/C48_S2SW.yaml b/ci/cases/pr/C48_S2SW.yaml
@@ -5,8 +5,9 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SW
-  resdet: 48
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 48
+  resdetocean: 5.0
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312

diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml
@@ -5,12 +5,13 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SWA
-  resdet: 48
-  resens: 48
+  resdetatmos: 48
+  resdetocean: 5.0
+  resensatmos: 48
   nens: 2
   gfs_cyc: 1
   start: cold
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312

diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml
@@ -5,9 +5,10 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 96
-  resens: 48
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 96
+  resdetocean: 5.0
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
   idate: 2021122018
@@ -16,6 +17,3 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
-
-skip_ci_on_hosts:
-  - hercules
diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml
@@ -5,16 +5,13 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 96
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 96
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: ${ICSDIR_ROOT}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
   idate: 2021122018
   edate: 2021122106
   nens: 0
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
-
-skip_ci_on_hosts:
-  - hercules
diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/ci/cases/weekly/C384C192_hybatmda.yaml
@@ -5,9 +5,10 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 384
-  resens: 192
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 384
+  resdetocean: 0.25
+  resensatmos: 192
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192
   idate: 2023040118

diff --git a/ci/cases/weekly/C384_S2SWA.yaml b/ci/cases/weekly/C384_S2SWA.yaml
@@ -5,8 +5,9 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SWA
-  resdet: 384
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 384
+  resdetocean: 0.5
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2016070100
   edate: 2016070100

diff --git a/ci/cases/weekly/C384_atm3DVar.yaml b/ci/cases/weekly/C384_atm3DVar.yaml
@@ -5,9 +5,10 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 384
-  resens: 192
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 384
+  resdetocean: 0.25
+  resensatmos: 192
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192
   idate: 2023040118

diff --git a/ci/platforms/gefs_ci_defaults.yaml b/ci/platforms/gefs_ci_defaults.yaml
@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
 base:
-  ACCOUNT: ${SLURM_ACCOUNT}
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
diff --git a/ci/platforms/gfs_defaults_ci.yaml b/ci/platforms/gfs_defaults_ci.yaml
@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  ACCOUNT: ${SLURM_ACCOUNT}
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
diff --git a/ci/scripts/check_ci.sh b/ci/scripts/check_ci.sh
@@ -22,7 +22,7 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
    ;;
@@ -149,7 +149,7 @@ for pr in ${pr_list}; do
     if [[ "${num_done}" -eq  "${num_cycles}" ]]; then
       #Remove Experment cases that completed successfully
       rm -Rf "${pslot_dir}"
-      rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}"
+      rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
       rm -f "${output_ci_single}"
       # echo "\`\`\`" > "${output_ci_single}"
       DATE=$(date +'%D %r')

diff --git a/ci/scripts/driver.sh b/ci/scripts/driver.sh
@@ -34,7 +34,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;
@@ -173,7 +173,7 @@ for pr in ${pr_list}; do
   # we need to exit this instance of the driver script
   #################################################################
   if [[ ${ci_status} -ne 0 ]]; then
-     build_PID_check=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display "{pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true
+     build_PID_check=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true
      if [[ "${build_PID_check}" -ne "$$" ]]; then
         echo "Driver build PID: ${build_PID_check} no longer running this build ... exiting"
         exit 0

diff --git a/ci/scripts/driver_weekly.sh b/ci/scripts/driver_weekly.sh
@@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;

diff --git a/ci/scripts/run-check_ci.sh b/ci/scripts/run-check_ci.sh
@@ -12,12 +12,12 @@ pslot=${2:-${pslot:-?}}        # Name of the experiment being tested by this scr
 
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# 2. RUNTESTS: A directory containing EXPDIR and COMROOT for experiments
 # # e.g. $> tree ./TEST_DIR
 # ./TEST_DIR
 # ├── HOMEgfs
 # └── RUNTESTS
-#     ├── COMROT
+#     ├── COMROOT
 #     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}

diff --git a/ci/scripts/run_ci.sh b/ci/scripts/run_ci.sh
@@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
    ;;

diff --git a/docs/doxygen/mainpage.h b/docs/doxygen/mainpage.h
@@ -12,7 +12,7 @@
 With the FV3GFS system, a Rocoto driven workflow is being used. This workflow builds on the experiences and work of Kate.Howard and Terry.McGuinness for the GSM, but strips down a lot of complications arising from the use of a cumbersome <b>\c para_config</b>. The <b>\c para_config</b> no longer exists in the workflow and instead a number of <b> \c config </b> files are added, one each for an individual task e.g. <b>\p config.anal</b> contains the analysis specific information. A base config called as <b>\c config.base</b> contains
 information related to the machine, super-structure, etc. The idea behind splitting the <b>\p para_config</b> into individual, smaller and managable configs is to provide a means to run any chosen task without the overhead of the full cycling framework. All the configs are located under <b>\c fv3gfs/config</b>
 
-Additionally, the structure of the <b>\c COMROT</b> directory is now modified to look like operations. This enables the use of the workflow much closer to the operational environment, with the exception of the workflow manager.
+Additionally, the structure of the <b>\c ROTDIR</b> directory is now modified to look like operations. This enables the use of the workflow much closer to the operational environment, with the exception of the workflow manager.
 
 This is a very much a work in progress and any issues should be reported back and are greatly appreciated.
 
@@ -23,12 +23,12 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
     $> setup_expt.py -h
     usage: setup_expt.py [-h] --pslot PSLOT
                      [--configdir CONFIGDIR] [--idate IDATE] [--icsdir ICSDIR]
-                     [--resdet RESDET] [--resens RESENS] [--comrot COMROT]
+                     [--resdetatmos RESDET] [--resensatmos RESENS] [--comroot COMROOT]
                      [--expdir EXPDIR] [--nens NENS] [--cdump CDUMP]
 
     Setup files and directories to start a GFS parallel. Create EXPDIR, copy
-    config files Create COMROT experiment directory structure, link initial
-    condition files from $ICSDIR to $COMROT
+    config files Create ROTDIR experiment directory structure, link initial
+    condition files from $ICSDIR to $ROTDIR
 
     optional arguments:
         -h, --help   show this help message and exit
@@ -40,11 +40,13 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
                         (default: 2016100100)
         --icsdir     full path to initial condition directory
                         (default: /scratch4/NCEPDEV/da/noscrub/Rahul.Mahajan/ICS)
-        --resdet     resolution of the deterministic model forecast
+        --resdetatmos atmosphere resolution of the deterministic model forecast
                         (default: 384)
-        --resens     resolution of the ensemble model forecast
+        --resdetocean ocean resolution of the deterministic model forecast
+                        (default: 0. [determined automatically based on atmosphere resolution])
+        --resensatmos     resolution of the ensemble model forecast
                         (default: 192)
-        --comrot     full path to COMROT
+        --comroot    full path to COMROOT, where ROTDIR (COMROOT+PSLOT) will be created
                         (default: None)
         --expdir     full path to EXPDIR
                         (default: None)
@@ -53,7 +55,7 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
         --cdump      CDUMP to start the experiment
                         (default: gdas)
 
-The above script creates directories <b>\c EXPDIR</b> and <b>\c COMROT</b>. It will make links for initial conditions from a location provided via the <b>\c --icsdir</b> argument for a chosen resolution for the control <b>\c --resdet</b> and the ensemble <b>\c --resens</b>. Experiment name is controlled by the input argument <b>\c --pslot</b>. The script will ask user input in case any of the directories already exist. It will copy experiment configuration files into the <b>\c EXPDIR</b> from <b>\c CONFIGDIR</b>.
+The above script creates directories <b>\c EXPDIR</b> and <b>\c ROTDIR</b>. It will make links for initial conditions from a location provided via the <b>\c --icsdir</b> argument for a chosen resolution for the control <b>\c --resdetatmos</b> and the ensemble <b>\c --resensatmos</b>. Experiment name is controlled by the input argument <b>\c --pslot</b>. The script will ask user input in case any of the directories already exist. It will copy experiment configuration files into the <b>\c EXPDIR</b> from <b>\c CONFIGDIR</b>.
 
 Sample initial conditions for a few resolutions are available at:<br>
 <b>Hera:</b> TODO: /path/here/for/initial/conditions<br>

diff --git a/docs/source/errors_faq.rst b/docs/source/errors_faq.rst
@@ -36,10 +36,10 @@ Example::
 **Solution:** set TERM to "xterm" (bash: export TERM=xterm ; csh/tcsh: setenv TERM xterm)
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Issue: Directory name change for EnKF folder in COMROT
+Issue: Directory name change for EnKF folder in ROTDIR
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-**Issue:** The EnKF COMROT folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY
+**Issue:** The EnKF ROTDIR folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY
 
 **Fix:** Older tarballs on HPSS will have the older directory name with the period between 'enkf' and 'gdas'. Make sure to rename folder to 'enkfgdas.$PDY' after obtaining. Only an issue for the initial cycle.
 
diff --git a/docs/source/init.rst b/docs/source/init.rst
@@ -391,7 +391,7 @@ The chgres jobs will have a dependency on the data-pull jobs and will wait to ru
 
 5. Check output:
 
-In the config you will have defined an output folder called ``$OUTDIR``. The converted output will be found there, including the needed abias and radstat initial condition files (if CDUMP=gdas). The files will be in the needed directory structure for the global-workflow system, therefore a user can move the contents of their ``$OUTDIR`` directly into their ``$ROTDIR/$COMROT``.
+In the config you will have defined an output folder called ``$OUTDIR``. The converted output will be found there, including the needed abias and radstat initial condition files (if CDUMP=gdas). The files will be in the needed directory structure for the global-workflow system, therefore a user can move the contents of their ``$OUTDIR`` directly into their ``$ROTDIR``.
 
 Please report bugs to George Gayno (george.gayno@noaa.gov) and Kate Friedman (kate.friedman@noaa.gov).
 
@@ -449,7 +449,7 @@ The warm starts and other output from production are at C768 deterministic and C
 What files should you pull for starting a new experiment with warm starts from production?
 ------------------------------------------------------------------------------------------
 
-That depends on what mode you want to run -- forecast-only or cycled. Whichever mode, navigate to the top of your ``COMROT`` and pull the entirety of the tarball(s) listed below for your mode. The files within the tarball are already in the ``$CDUMP.$PDY/$CYC/$ATMOS`` folder format expected by the system.
+That depends on what mode you want to run -- forecast-only or cycled. Whichever mode, navigate to the top of your ``ROTDIR`` and pull the entirety of the tarball(s) listed below for your mode. The files within the tarball are already in the ``$CDUMP.$PDY/$CYC/$ATMOS`` folder format expected by the system.
 
 For forecast-only there are two tarballs to pull
 
@@ -489,7 +489,7 @@ Tarballs per cycle:
    com_gfs_vGFSVER_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp7.tar
    com_gfs_vGFSVER_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp8.tar
 
-Go to the top of your ``COMROT/ROTDIR`` and pull the contents of all tarballs there. The tarballs already contain the needed directory structure.
+Go to the top of your ``ROTDIR`` and pull the contents of all tarballs there. The tarballs already contain the needed directory structure.
 
 .. _warmstarts-preprod-parallels:
 
@@ -514,7 +514,7 @@ Recent pre-implementation parallel series was for GFS v16 (implemented March 202
     + ../$GDATE/gdas_restartb.tar
     + ../$GDATE/enkfgdas_restartb_grp##.tar (where ## is 01 through 08) (note, older tarballs may include a period between enkf and gdas: "enkf.gdas")
 
-* **Where do I put the warm-start initial conditions?** Extraction should occur right inside your COMROT. You may need to rename the enkf folder (enkf.gdas.$PDY -> enkfgdas.$PDY).
+* **Where do I put the warm-start initial conditions?** Extraction should occur right inside your ROTDIR. You may need to rename the enkf folder (enkf.gdas.$PDY -> enkfgdas.$PDY).
 
 Due to a recent change in the dycore, you may also need an additional offline step to fix the checksum of the NetCDF files for warm start. See the :ref:`Fix netcdf checksum section <gfsv17-checksum>`.
 
@@ -602,5 +602,5 @@ And then on all platforms:
 
 ::
 
-   cd $COMROT
+   cd $ROTDIR
    for f in $(find ./ -name *tile*.nc); do echo $f; ncatted -a checksum,,d,, $f; done