Remove gfs-utils code and update for new fix

Removes all of the code and scripts that were moved to the new gfs-utils
repo and adjusts workflow scripts to build and use them from the new
location.

Some of the build scripts had unnecessary calls to machine-setup that are
removed because the lower-level script already has the same functionality.

This PR also includes updates to use the new fix organization. This includes
the addition of a fix versions file, updates to the link script, and some
changes in the fix directories used by scripts to account for files that
have been relocated. The versions file sets the version number for each
component of fix, so that fix can be more easily maintained and documented.
The initial versions are all the same, and correspond to the old fix_NEW
directory (other than some directories have been renamed or reorganizated).

Finally, the link script has been updated to match the syntax of the rest
of the build system (`checkout.sh` and `build_all.sh`). `link_workflow.sh`
now detects the machine automatically instead of requiring an argument, and
dev mode, which used to be set using `emc` as an argument, is now assumed.
To run in ops mode (copy instead of link), the `-o` option is used. The
full syntax is now simply:
```
./link_workflow.sh [-o]

-o: Run in operations (NCO) mode (copy instead of link)
```

This commit uses a temporary clone of the UFS_UTILS repo pending the merger
of ufs-community/UFS_UTILS#688.

Fixes NOAA-EMC#356

.gitignore

@@ -90,62 +90,8 @@ parm/wafs
 #--------------------------------------------
 sorc/*log
 sorc/logs
-sorc/ufs_model.fd
-sorc/gfs_post.fd
-sorc/gfs_wafs.fd
-sorc/gldas.fd
-sorc/gsi_enkf.fd
-sorc/gsi.fd
-sorc/enkf.fd
-sorc/gdas.cd
-sorc/gsi_utils.fd
-sorc/gsi_monitor.fd
-sorc/ufs_utils.fd
-sorc/verif-global.fd
-
-# Ignore sorc symlinks
-#---------------------
-sorc/calc_analysis.fd
-sorc/calc_increment_ens.fd
-sorc/calc_increment_ens_ncio.fd
-sorc/emcsfc_ice_blend.fd
-sorc/emcsfc_snow2mdl.fd
-sorc/fregrid.fd
-sorc/gdas2gldas.fd
-sorc/getsfcensmeanp.fd
-sorc/getsigensmeanp_smooth.fd
-sorc/getsigensstatp.fd
-sorc/gfs_ncep_post.fd
-sorc/gldas2gdas.fd
-sorc/gldas_forcing.fd
-sorc/gldas_model.fd
-sorc/gldas_post.fd
-sorc/gldas_rst.fd
-sorc/global_chgres.fd
-sorc/global_cycle.fd
-sorc/global_enkf.fd
-sorc/global_gsi.fd
-sorc/interp_inc.fd
-sorc/make_hgrid.fd
-sorc/make_solo_mosaic.fd
-sorc/ncdiag_cat.fd
-sorc/nst_tf_chg.fd
-sorc/oznmon_horiz.fd
-sorc/oznmon_time.fd
-sorc/radmon_angle.fd
-sorc/radmon_bcoef.fd
-sorc/radmon_bcor.fd
-sorc/radmon_time.fd
-sorc/recentersigp.fd
-sorc/upp.fd
-sorc/wafs_awc_wafavn.fd
-sorc/wafs_blending.fd
-sorc/wafs_blending_0p25.fd
-sorc/wafs_cnvgrib2.fd
-sorc/wafs_gcip.fd
-sorc/wafs_grib2_0p25.fd
-sorc/wafs_makewafs.fd
-sorc/wafs_setmissing.fd
+sorc/*.cd
+sorc/*.fd
 
 # Ignore scripts from externals
 #------------------------------

scripts/exglobal_atmos_analysis_calc.sh

@@ -72,7 +72,7 @@ CALCANLPY=${CALCANLPY:-$HOMEgfs/ush/calcanl_gfs.py}
 
 DOGAUSFCANL=${DOGAUSFCANL-"NO"}
 GAUSFCANLSH=${GAUSFCANLSH:-$HOMEgfs/ush/gaussian_sfcanl.sh}
-export GAUSFCANLEXE=${GAUSFCANLEXE:-$HOMEgfs/exec/gaussian_sfcanl.exe}
+export GAUSFCANLEXE=${GAUSFCANLEXE:-$HOMEgfs/exec/gaussian_sfcanl.x}
 NTHREADS_GAUSFCANL=${NTHREADS_GAUSFCANL:-1}
 APRUN_GAUSFCANL=${APRUN_GAUSFCANL:-${APRUN:-""}}
 

sorc/build_all.sh

@@ -29,6 +29,9 @@ EOF
   exit 1
 }
 
+script_dir=$(cd $(dirname "${BASH_SOURCE[0]}") &> /dev/null && pwd)
+cd ${script_dir}
+
 _build_ufs_opt=""
 _ops_opt=""
 _verbose_opt=""
@@ -55,8 +58,7 @@ done
 
 shift $((OPTIND-1))
 
-build_dir=$(pwd)
-logs_dir=$build_dir/logs
+logs_dir=${script_dir}/logs
 if [ ! -d $logs_dir  ]; then
   echo "Creating logs folder"
   mkdir $logs_dir
@@ -72,7 +74,11 @@ fi
 # GET MACHINE
 #------------------------------------
 target=""
-source ./machine-setup.sh > /dev/null 2>&1
+source gfs_utils.fd/ush/machine-setup.sh > /dev/null 2>&1
+if [[ -z "${target}" ]]; then
+  echo "FATAL: Unable to determine target machine"
+  exit 1
+fi
 
 #------------------------------------
 # INCLUDE PARTIAL BUILD
@@ -254,29 +260,15 @@ if [ -d gfs_wafs.fd ]; then
 fi
 
 #------------------------------------
-# build workflow_utils
-#------------------------------------
-$Build_workflow_utils && {
-  echo " .... Building workflow_utils .... "
-  target=$target ./build_workflow_utils.sh $_verbose_opt > $logs_dir/build_workflow_utils.log 2>&1
-  rc=$?
-  if [[ $rc -ne 0 ]] ; then
-    echo "Fatal error in building workflow_utils."
-    echo "The log file is in $logs_dir/build_workflow_utils.log"
-  fi
-  ((err+=$rc))
-}
-
-#------------------------------------
-# build gfs_util
+# build gfs_utils
 #------------------------------------
-$Build_gfs_util && {
-  echo " .... Building gfs_util .... "
-  ./build_gfs_util.sh $_verbose_opt > $logs_dir/build_gfs_util.log 2>&1
+$Build_gfs_utils && {
+  echo " .... Building gfs_utils .... "
+  target=$target ./build_gfs_utils.sh $_verbose_opt > $logs_dir/build_gfs_utils.log 2>&1
   rc=$?
   if [[ $rc -ne 0 ]] ; then
-      echo "Fatal error in building gfs_util."
-      echo "The log file is in $logs_dir/build_gfs_util.log"
+    echo "Fatal error in building gfs_utils."
+    echo "The log file is in $logs_dir/build_gfs_utils.log"
   fi
   ((err+=$rc))
 }

sorc/build_gdas.sh

@@ -1,8 +1,10 @@
 #! /usr/bin/env bash
 set -eux
 
-source ./machine-setup.sh > /dev/null 2>&1
-cwd=$(pwd)
+script_dir=$(dirname ${BASH_SOURCE[0]})
+cd ${script_dir}
+
+source gfs_utils.fd/ush/machine-setup.sh > /dev/null 2>&1
 
 export BUILD_TARGET=$target
 

sorc/build_gfs_utils.sh

@@ -0,0 +1,25 @@
+#! /usr/bin/env bash
+set -eux
+
+cwd=$(pwd)
+
+OPTIND=1
+while getopts ":dov" option; do
+  case "${option}" in
+    d) export BUILD_TYPE="DEBUG";;
+    v) export BUILD_VERBOSE="YES";;
+    \?)
+      echo "[$BASH_SOURCE]: Unrecognized option: ${option}"
+      usage
+      ;;
+    :)
+      echo "[$BASH_SOURCE]: ${option} requires an argument"
+      usage
+      ;;
+  esac
+done
+shift $((OPTIND-1))
+
+${cwd}/gfs_utils.fd/sorc/build_gfs_utils.sh
+
+exit

sorc/build_gfs_wafs.sh

@@ -1,8 +1,8 @@
 #! /usr/bin/env bash
 set -eux
 
-source ./machine-setup.sh > /dev/null 2>&1
-cwd=$(pwd)
+script_dir=$(dirname ${BASH_SOURCE[0]})
+cd ${script_dir}
 
 # Check final exec folder exists
 if [ ! -d "../exec" ]; then

sorc/build_gldas.sh

@@ -1,8 +1,8 @@
 #! /usr/bin/env bash
 set -eux
 
-source ./machine-setup.sh > /dev/null 2>&1
-cwd=$(pwd)
+script_dir=$(dirname ${BASH_SOURCE[0]})
+cd ${script_dir}
 
 # Check final exec folder exists
 if [ ! -d "../exec" ]; then

sorc/build_ufs_utils.sh

@@ -1,10 +1,8 @@
 #! /usr/bin/env bash
 set -eux
 
-source ./machine-setup.sh > /dev/null 2>&1
-cwd=$(pwd)
-
-cd ufs_utils.fd
+script_dir=$(dirname ${BASH_SOURCE[0]})
+cd ${script_dir}/ufs_utils.fd
 
 ./build_all.sh
 

sorc/build_upp.sh

@@ -1,8 +1,8 @@
 #! /usr/bin/env bash
 set -eux
 
-source ./machine-setup.sh > /dev/null 2>&1
-cwd=$(pwd)
+script_dir=$(dirname ${BASH_SOURCE[0]})
+cd ${script_dir}
 
 OPTIND=1
 _opts=""

sorc/build_ww3prepost.sh

@@ -1,6 +1,11 @@
 #!/bin/sh
 set -x
 
+script_dir=$(dirname ${BASH_SOURCE[0]})
+cd ${script_dir}
+
+source gfs_utils.fd/ush/machine-setup.sh > /dev/null 2>&1
+
 # Default settings
 APP="S2SWA"
 
@@ -33,10 +38,8 @@ finalexecdir=$( pwd -P )/../exec
 
 #Determine machine and load modules
 set +x
-source ./machine-setup.sh > /dev/null 2>&1
-
 module use ../modulefiles
-module load modulefile.ww3.$target
+module load modulefile.ww3.${target}
 set -x
 
 #Set WW3 directory, switch, prep and post exes 

sorc/checkout.sh

@@ -102,8 +102,8 @@ function checkout() {
 
 # Set defaults for variables toggled by options
 export CLEAN="NO"
-CHECKOUT_GSI="NO"
-CHECKOUT_GDAS="NO"
+checkout_gsi="NO"
+checkout_gdas="NO"
 checkout_gtg="NO"
 checkout_wafs="NO"
 ufs_model_hash="Prototype-P8"
@@ -112,17 +112,17 @@ ufs_model_hash="Prototype-P8"
 while getopts ":chgum:o" option; do
   case $option in
     c)
-      echo "Recieved -c flag, will delete any existing directories and start clean"
+      echo "Received -c flag, will delete any existing directories and start clean"
       export CLEAN="YES"
       ;;
     g)
-      echo "Receieved -g flag for optional checkout of GSI-based DA"
-      CHECKOUT_GSI="YES"
+      echo "Received -g flag for optional checkout of GSI-based DA"
+      checkout_gsi="YES"
       ;;
     h)  usage;;
     u)
       echo "Received -u flag for optional checkout of UFS-based DA"
-      CHECKOUT_GDAS="YES"
+      checkout_gdas="YES"
       ;;
     o)
       echo "Received -o flag for optional checkout of operational-only codes"
@@ -145,30 +145,29 @@ while getopts ":chgum:o" option; do
 done
 shift $((OPTIND-1))
 
-export topdir=$(pwd)
+export topdir=$(cd $(dirname "${BASH_SOURCE[0]}") &> /dev/null && pwd)
 export logdir="${topdir}/logs"
 mkdir -p ${logdir}
 
 # The checkout version should always be a speciifc commit (hash or tag), not a branch
 errs=0
+checkout "gfs_utils.fd"    "https://github.com/NOAA-EMC/gfs-utils"              "08d440c"          ; errs=$((errs + $?))
 checkout "ufs_model.fd"    "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash}"; errs=$((errs + $?))
-checkout "ufs_utils.fd"    "https://github.com/ufs-community/UFS_UTILS.git"     "ufs_utils_1_8_0"  ; errs=$((errs + $?))
+# checkout "ufs_utils.fd"    "https://github.com/ufs-community/UFS_UTILS.git"     "ufs_utils_1_8_0"  ; errs=$((errs + $?))
+checkout "ufs_utils.fd"    "https://github.com/WalterKolczynski-NOAA/UFS_UTILS.git"     "b1c7a1c"  ; errs=$((errs + $?))
 checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git"   "c267780"          ; errs=$((errs + $?))
 
-if [[ $CHECKOUT_GSI == "YES" ]]; then
-  checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "67f5ab4"; errs=$((errs + $?))
-fi
-
-if [[ $CHECKOUT_GDAS == "YES" ]]; then
-  checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "5952c9d"; errs=$((errs + $?))
-fi
-
-if [[ $CHECKOUT_GSI == "YES" || $CHECKOUT_GDAS == "YES" ]]; then
+if [[ $checkout_gsi == "YES" ]]; then
+  checkout "gsi_enkf.fd"     "https://github.com/NOAA-EMC/GSI.git"         "67f5ab4"; errs=$((errs + $?))
   checkout "gsi_utils.fd"    "https://github.com/NOAA-EMC/GSI-Utils.git"   "322cc7b"; errs=$((errs + $?))
   checkout "gsi_monitor.fd"  "https://github.com/NOAA-EMC/GSI-Monitor.git" "c64cc47"; errs=$((errs + $?))
   checkout "gldas.fd"        "https://github.com/NOAA-EMC/GLDAS.git"       "fd8ba62"; errs=$((errs + $?))
 fi
 
+if [[ $checkout_gdas == "YES" ]]; then
+  checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "5952c9d"; errs=$((errs + $?))
+fi
+
 if [[ $checkout_wafs == "YES" ]]; then
   checkout "gfs_wafs.fd" "https://github.com/NOAA-EMC/EMC_gfs_wafs.git" "014a0b8"; errs=$((errs + $?))
 fi