Update UFS to Sept 9 version (#1073)

Updates the UFS version. This captures the conversion of UFS module from TCL to lua. A couple of the CICE namelist variables are no longer valid in this version, so they are removed.

Due to memory limitations on Hera and the increased memory requirements of GOCART, the number of threads is increased there when running the forecast with aerosols.

Also added a temporary block to delete any existing gocart output files. The ability to clobber files was deactivated a while ago and I got fed up with forecast jobs failing on retry.

Externals.cfg

@@ -1,7 +1,7 @@
 # External sub-modules of global-workflow
 
 [UFS]
-tag = Prototype-P8
+tag = 3c3548d
 local_path = sorc/ufs_model.fd
 repo_url = https://github.com/ufs-community/ufs-weather-model.git
 protocol = git

modulefiles/module_base.hera.lua

@@ -8,6 +8,8 @@ load(pathJoin("hpc", "1.1.0"))
 load(pathJoin("hpc-intel", "18.0.5.274"))
 load(pathJoin("hpc-impi", "2018.0.4"))
 
+load(pathJoin("cmake", "3.20.1"))
+
 load(pathJoin("hpss", "hpss"))
 load(pathJoin("nco", "4.9.1"))
 load(pathJoin("gempak", "7.4.2"))
@@ -25,8 +27,8 @@ load(pathJoin("png", "1.6.35"))
 
 load(pathJoin("hdf5", "1.10.6"))
 load(pathJoin("netcdf", "4.7.4"))
-load(pathJoin("pio", "2.5.2"))
-load(pathJoin("esmf", "8.2.1b04"))
+load(pathJoin("pio", "2.5.7"))
+load(pathJoin("esmf", "8.3.0b09"))
 load(pathJoin("fms", "2021.03"))
 
 load(pathJoin("bacio", "2.4.1"))

modulefiles/module_base.orion.lua

@@ -8,6 +8,8 @@ load(pathJoin("hpc", "1.1.0"))
 load(pathJoin("hpc-intel", "2018.4"))
 load(pathJoin("hpc-impi", "2018.4"))
 
+load(pathJoin("cmake", "3.22.1"))
+
 load(pathJoin("nco", "4.8.1"))
 load(pathJoin("gempak", "7.5.1"))
 load(pathJoin("ncl", "6.6.2"))
@@ -24,8 +26,8 @@ load(pathJoin("png", "1.6.35"))
 
 load(pathJoin("hdf5", "1.10.6"))
 load(pathJoin("netcdf", "4.7.4"))
-load(pathJoin("pio", "2.5.2"))
-load(pathJoin("esmf", "8.2.1b04"))
+load(pathJoin("pio", "2.5.7"))
+load(pathJoin("esmf", "8.3.0b09"))
 load(pathJoin("fms", "2021.03"))
 
 load(pathJoin("bacio", "2.4.1"))

modulefiles/modulefile.ww3.hera.lua

@@ -20,4 +20,4 @@ load(pathJoin("g2", "3.4.5"))
 load(pathJoin("hdf5", "1.10.6"))
 load(pathJoin("netcdf", "4.7.4"))
 
-load(pathJoin("w3nco", "2.4.1"))
+load(pathJoin("w3emc", "2.9.2"))

modulefiles/modulefile.ww3.orion.lua

@@ -22,4 +22,4 @@ load(pathJoin("g2", "3.4.5"))
 load(pathJoin("hdf5", "1.10.6"))
 load(pathJoin("netcdf", "4.7.4"))
 
-load(pathJoin("w3nco", "2.4.1"))
+load(pathJoin("w3emc", "2.9.2"))

modulefiles/modulefile.ww3.s4.lua

@@ -18,4 +18,4 @@ load(pathJoin("g2", "3.4.1"))
 load(pathJoin("hdf5", "1.10.6"))
 load(pathJoin("netcdf", "4.7.4"))
 
-load(pathJoin("w3nco", "2.4.1"))
+load(pathJoin("w3emc", "2.9.2"))

modulefiles/modulefile.ww3.wcoss2.lua

@@ -19,4 +19,4 @@ load(pathJoin("g2", "3.4.5"))
 load(pathJoin("hdf5", "1.10.6"))
 load(pathJoin("netcdf", "4.7.4"))
 
-load(pathJoin("w3nco", "2.4.1"))
+load(pathJoin("w3emc", "2.9.2"))

parm/config/config.resources

@@ -311,6 +311,11 @@ elif [ ${step} = "fcst" ]; then
             # Aerosol model only uses the atm model PETS
             export CHMPETS=${ATMPETS}
             # Aerosol model runs on same PETs as ATM, so don't add to $NTASKS_TOT
+
+            if [[ ${machine} == "HERA" ]]; then
+                # Need more memory on Hera for aerosols, so increase threads to spread it out
+                nth_fv3_gfs=${nth_fv3_gfs:-4}
+            fi
         fi
 
         # If using in-line post, add the write tasks to the ATMPETS

sorc/build_ufs.sh

@@ -33,6 +33,6 @@ CLEAN_AFTER=NO
 
 ./tests/compile.sh $MACHINE_ID "$MAKE_OPT" $COMPILE_NR $CLEAN_BEFORE $CLEAN_AFTER
 mv ./tests/fv3_${COMPILE_NR}.exe ./tests/ufs_model.x
-mv ./tests/modules.fv3_${COMPILE_NR} ./tests/modules.ufs_model
+mv ./tests/modules.fv3_${COMPILE_NR}.lua ./tests/modules.ufs_model.lua
 
 exit 0

sorc/checkout.sh

@@ -107,7 +107,6 @@ checkout_gsi="NO"
 checkout_gdas="NO"
 checkout_gtg="NO"
 checkout_wafs="NO"
-ufs_model_hash="Prototype-P8"
 
 # Parse command line arguments
 while getopts ":chgum:o" option; do
@@ -153,9 +152,9 @@ mkdir -p "${logdir}"
 # The checkout version should always be a speciifc commit (hash or tag), not a branch
 errs=0
 checkout "gfs_utils.fd"    "https://github.com/NOAA-EMC/gfs-utils"              "7bf599f"          ; errs=$((errs + $?))
-checkout "ufs_model.fd"    "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash}"; errs=$((errs + $?))
-checkout "ufs_utils.fd"    "https://github.com/ufs-community/UFS_UTILS.git"     "8b990c0"          ; errs=$((errs + $?))
-checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git"   "c267780"          ; errs=$((errs + $?))
+checkout "ufs_model.fd"    "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-3c3548d}" ; errs=$((errs + $?))
+checkout "ufs_utils.fd"    "https://github.com/ufs-community/UFS_UTILS.git"     "8b990c0"                    ; errs=$((errs + $?))
+checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git"   "c267780"                    ; errs=$((errs + $?))
 
 if [[ ${checkout_gsi} == "YES" ]]; then
   checkout "gsi_enkf.fd"     "https://github.com/NOAA-EMC/GSI.git"         "67f5ab4"; errs=$((errs + $?))

ush/forecast_postdet.sh

@@ -1014,6 +1014,13 @@ GOCART_postdet() {
     HH=$(echo $VDATE | cut -c9-10)
     SS=$((10#$HH*3600))
 
+    #
+    # Temporarily delete existing files due to noclobber in GOCART
+    #
+    if [[ -e "${COMOUTaero}/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4" ]]; then
+      rm "${COMOUTaero}/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4"
+    fi
+
     $NLN $COMOUTaero/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4 $DATA/gocart.inst_aod.${YYYY}${MM}${DD}_${HH}00z.nc4
   done
 }

ush/parsing_namelists_CICE.sh

@@ -80,8 +80,6 @@ cat > ice_in <<eof
    restart_FY   = .false.
    tr_lvl       = .true.
    restart_lvl  = .false.
-   tr_pond_cesm = .false.
-   restart_pond_cesm = .false.
    tr_pond_topo = .false.
    restart_pond_topo = .false.
    tr_pond_lvl  = $tr_pond_lvl