Implement fix reorg and remove gfs-utils code (#1009)

Removes all of the code and scripts that were moved to the new [gfs-utils repo](https://github.com/NOAA-EMC/gfs-utils) and adjusts workflow scripts to build and use them from the new location. Some of the build scripts had unnecessary calls to machine-setup that are removed because the lower-level script already has the same functionality. This PR also includes updates to use the new fix organization. This includes the addition of a fix versions file, updates to the link script, and some changes in the fix directories used by scripts to account for files that have been relocated. The versions file sets the version number for each component of fix, so that fix can be more easily maintained and documented. The initial versions are all the same, and correspond to the old fix_NEW directory (other than some directories have been renamed or reorganized). Except gdas, which has already had a new set of fix files added. The fix update also required an update to UFS-Utils. Finally, the link script has been updated to match the syntax of the rest of the build system ([checkout.sh](sorc/checkout.sh) and [build_all.sh](sorc/build_all.sh)). [link_workflow.sh](sorc/link_workflow.sh) now detects the machine automatically instead of requiring an argument, and dev mode, which used to be set using `emc` as an argument, is now assumed. To run in ops mode (copy instead of link), the `-o` option is used. The full syntax is now simply: ``` ./link_workflow.sh [-o] Options: -o: Run in operations (NCO) mode (copy instead of link) ``` Fixes #356 Fixes #966
NOAA-EMC · Oct 4, 2022 · e7f72e8 · e7f72e8
1 parent 948f941
commit e7f72e8
Show file tree

Hide file tree

Showing 163 changed files with 593 additions and 46,864 deletions.
diff --git a/.gitignore b/.gitignore
@@ -17,8 +17,23 @@ install*/
 # Ignore fix directory symlinks
 #------------------------------
 fix/0readme
-fix/fix_*
-fix/gdas/
+fix/aer
+fix/am
+fix/chem
+fix/cice
+fix/cpl
+fix/datm
+fix/gdas
+fix/gldas
+fix/gsi
+fix/lut
+fix/mom6
+fix/orog
+fix/reg2grb2
+fix/sfc_climo
+fix/ugwd
+fix/verif
+fix/wave
 fix/wafs
 
 # Ignore parm file symlinks
@@ -75,62 +90,8 @@ parm/wafs
 #--------------------------------------------
 sorc/*log
 sorc/logs
-sorc/ufs_model.fd
-sorc/gfs_post.fd
-sorc/gfs_wafs.fd
-sorc/gldas.fd
-sorc/gsi_enkf.fd
-sorc/gsi.fd
-sorc/enkf.fd
-sorc/gdas.cd
-sorc/gsi_utils.fd
-sorc/gsi_monitor.fd
-sorc/ufs_utils.fd
-sorc/verif-global.fd
-
-# Ignore sorc symlinks
-#---------------------
-sorc/calc_analysis.fd
-sorc/calc_increment_ens.fd
-sorc/calc_increment_ens_ncio.fd
-sorc/emcsfc_ice_blend.fd
-sorc/emcsfc_snow2mdl.fd
-sorc/fregrid.fd
-sorc/gdas2gldas.fd
-sorc/getsfcensmeanp.fd
-sorc/getsigensmeanp_smooth.fd
-sorc/getsigensstatp.fd
-sorc/gfs_ncep_post.fd
-sorc/gldas2gdas.fd
-sorc/gldas_forcing.fd
-sorc/gldas_model.fd
-sorc/gldas_post.fd
-sorc/gldas_rst.fd
-sorc/global_chgres.fd
-sorc/global_cycle.fd
-sorc/global_enkf.fd
-sorc/global_gsi.fd
-sorc/interp_inc.fd
-sorc/make_hgrid.fd
-sorc/make_solo_mosaic.fd
-sorc/ncdiag_cat.fd
-sorc/nst_tf_chg.fd
-sorc/oznmon_horiz.fd
-sorc/oznmon_time.fd
-sorc/radmon_angle.fd
-sorc/radmon_bcoef.fd
-sorc/radmon_bcor.fd
-sorc/radmon_time.fd
-sorc/recentersigp.fd
-sorc/upp.fd
-sorc/wafs_awc_wafavn.fd
-sorc/wafs_blending.fd
-sorc/wafs_blending_0p25.fd
-sorc/wafs_cnvgrib2.fd
-sorc/wafs_gcip.fd
-sorc/wafs_grib2_0p25.fd
-sorc/wafs_makewafs.fd
-sorc/wafs_setmissing.fd
+sorc/*.cd
+sorc/*.fd
 
 # Ignore scripts from externals
 #------------------------------

diff --git a/Externals.cfg b/Externals.cfg
@@ -7,8 +7,15 @@ repo_url = https://github.com/ufs-community/ufs-weather-model.git
 protocol = git
 required = True
 
+[gfs-utils]
+hash = 93898e1
+local_path = sorc/gfs_utils.fd
+repo_url = https://github.com/NOAA-EMC/gfs-utils
+protocol = git
+required = True
+
 [UFS-Utils]
-hash = ufs_utils_1_8_0
+hash = 8b990c0
 local_path = sorc/ufs_utils.fd
 repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git
 protocol = git

diff --git a/jobs/JGDAS_ATMOS_GLDAS b/jobs/JGDAS_ATMOS_GLDAS
@@ -83,7 +83,7 @@ export COMPONENT=${COMPONENT:-atmos}
 ##############################################
 export gldas_ver=${gldas_ver:-v2.3.0}
 export HOMEgldas=${HOMEgldas:-${HOMEgfs}}
-export FIXgldas=${FIXgldas:-${HOMEgldas}/fix/fix_gldas}
+export FIXgldas=${FIXgldas:-${HOMEgldas}/fix/gldas}
 export PARMgldas=${PARMgldas:-${HOMEgldas}/parm/gldas}
 export EXECgldas=${EXECgldas:-${HOMEgldas}/exec}
 export USHgldas=${USHgldas:-${HOMEgldas}/ush}

diff --git a/parm/config/config.aeroanl b/parm/config/config.aeroanl
@@ -9,7 +9,7 @@ export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/aero/obs/config/
 export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/aero/obs/lists/aero_prototype.yaml
 export AEROVARYAML=$HOMEgfs/sorc/gdas.cd/parm/aero/variational/3dvar_dripcg.yaml
 export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/aero/berror/static_bump.yaml
-export FV3JEDI_FIX=$HOMEgfs/fix/fix_jedi
+export FV3JEDI_FIX=$HOMEgfs/fix/gdas
 
 export io_layout_x=1
 export io_layout_y=1

diff --git a/parm/config/config.atmanal b/parm/config/config.atmanal
@@ -9,7 +9,7 @@ export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/
 export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/gdas_prototype.yaml
 export ATMVARYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/variational/3dvar_dripcg.yaml
 export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml
-export FV3JEDI_FIX=$HOMEgfs/fix/fix_jedi
+export FV3JEDI_FIX=$HOMEgfs/fix/gdas
 export R2D2_OBS_DB='ufsda_test'
 export R2D2_OBS_DUMP='oper_gdas'
 export R2D2_OBS_SRC='ncdiag'

diff --git a/parm/config/config.atmensanal b/parm/config/config.atmensanal
@@ -9,7 +9,7 @@ export OBS_YAML_DIR=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/config/
 export OBS_LIST=$HOMEgfs/sorc/gdas.cd/parm/atm/obs/lists/lgetkf_prototype.yaml
 export BERROR_YAML=$HOMEgfs/sorc/gdas.cd/parm/atm/berror/hybvar_bump.yaml
 export ATMENSYAML=$HOMEgfs/sorc/gdas.cd/parm/atm/lgetkf/lgetkf.yaml
-export FV3JEDI_FIX=$HOMEgfs/fix/fix_jedi
+export FV3JEDI_FIX=$HOMEgfs/fix/gdas
 export R2D2_OBS_DB='ufsda_test'
 export R2D2_OBS_DUMP='oper_gdas'
 export R2D2_OBS_SRC='ncdiag'

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
@@ -29,9 +29,9 @@ export UTILgfs=$HOMEgfs/util
 export EXECgfs=$HOMEgfs/exec
 export SCRgfs=$HOMEgfs/scripts
 
-export FIXcice=$HOMEgfs/fix/fix_cice
-export FIXmom=$HOMEgfs/fix/fix_mom6
-export FIXreg2grb2=$HOMEgfs/fix/fix_reg2grb2
+export FIXcice=$HOMEgfs/fix/cice
+export FIXmom=$HOMEgfs/fix/mom6
+export FIXreg2grb2=$HOMEgfs/fix/reg2grb2
 
 ########################################################################
 
@@ -74,7 +74,7 @@ export MODE="@MODE@" # cycled/forecast-only
 # CLEAR
 ####################################################
 # Build paths relative to $HOMEgfs
-export FIXgsi="$HOMEgfs/fix/fix_gsi"
+export FIXgsi="$HOMEgfs/fix/gsi"
 export HOMEfv3gfs="$HOMEgfs/sorc/fv3gfs.fd"
 export HOMEpost="$HOMEgfs"
 export HOMEobsproc_prep="$BASE_GIT/obsproc/obsproc_prep.v5.5.0_hpc-stack"

diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static
@@ -62,7 +62,7 @@ export REALTIME="YES"
 # CLEAR
 ####################################################
 # Build paths relative to $HOMEgfs
-export FIXgsi="$HOMEgfs/fix/fix_gsi"
+export FIXgsi="$HOMEgfs/fix/gsi"
 export HOMEfv3gfs="$HOMEgfs/sorc/fv3gfs.fd"
 export HOMEpost="$HOMEgfs"
 export HOMEobsproc_prep="$NWPROD/obsproc_prep.v5.5.0"

diff --git a/parm/config/config.prep b/parm/config/config.prep
@@ -21,7 +21,7 @@ export COMINsyn=${COMINsyn:-$(compath.py ${envir}/com/gfs/${gfs_ver})/syndat}
 
 export HOMERELO=$HOMEgfs
 export EXECRELO=${HOMERELO}/exec
-export FIXRELO=${HOMERELO}/fix/fix_am
+export FIXRELO=${HOMERELO}/fix/am
 export USHRELO=${HOMERELO}/ush
 
 # Adjust observation error for GFS v16 parallels

diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy
@@ -30,8 +30,8 @@ if [ $VRFYFITS = "YES" ]; then
 
     export fit_ver="newm.1.3"
     export fitdir="$BASE_GIT/verif/global/Fit2Obs/${fit_ver}/batrun"
-    export PRVT=$HOMEgfs/fix/fix_gsi/prepobs_errtable.global
-    export HYBLEVS=$HOMEgfs/fix/fix_am/global_hyblev.l${LEVS}.txt
+    export PRVT=$HOMEgfs/fix/gsi/prepobs_errtable.global
+    export HYBLEVS=$HOMEgfs/fix/am/global_hyblev.l${LEVS}.txt
     export CUE2RUN=$QUEUE
 
     export VBACKUP_FITS=24

diff --git a/parm/config/config.wave b/parm/config/config.wave
@@ -11,7 +11,7 @@ echo "BEGIN: config.wave"
 export wave_sys_ver=v1.0.0
 
 export EXECwave="$HOMEgfs/exec"
-export FIXwave="$HOMEgfs/fix/fix_wave"
+export FIXwave="$HOMEgfs/fix/wave"
 export PARMwave="$HOMEgfs/parm/wave"
 export USHwave="$HOMEgfs/ush"
 

diff --git a/scripts/exgdas_atmos_chgres_forenkf.sh b/scripts/exgdas_atmos_chgres_forenkf.sh
@@ -21,7 +21,7 @@ source "$HOMEgfs/ush/preamble.sh"
 
 #  Directories.
 pwd=$(pwd)
-export FIXgsm=${FIXgsm:-$HOMEgfs/fix/fix_am}
+export FIXgsm=${FIXgsm:-$HOMEgfs/fix/am}
 
 # Base variables
 CDATE=${CDATE:-"2001010100"}

diff --git a/scripts/exgdas_atmos_gldas.sh b/scripts/exgdas_atmos_gldas.sh
@@ -60,8 +60,8 @@ nlat=$((2*res))
 nlon=$((4*res))
 
 export USHgldas=${USHgldas:?}
-export FIXgldas=${FIXgldas:-${HOMEgfs}/fix/fix_gldas}
-export topodir=${topodir:-${HOMEgfs}/fix/fix_fv3_gmted2010/${CASE}/}
+export FIXgldas=${FIXgldas:-${HOMEgfs}/fix/gldas}
+export topodir=${topodir:-${HOMEgfs}/fix/orog/${CASE}}
 
 DATA=${DATA:-${pwd}/gldastmp$$}
 mkdata=NO

diff --git a/scripts/exgdas_enkf_ecen.sh b/scripts/exgdas_enkf_ecen.sh
@@ -75,8 +75,8 @@ CYCLESH=${CYCLESH:-$HOMEgfs/ush/global_cycle.sh}
 export CYCLEXEC=${CYCLEXEC:-$HOMEgfs/exec/global_cycle}
 APRUN_CYCLE=${APRUN_CYCLE:-${APRUN:-""}}
 NTHREADS_CYCLE=${NTHREADS_CYCLE:-${NTHREADS:-1}}
-export FIXfv3=${FIXfv3:-$HOMEgfs/fix/fix_fv3_gmted2010}
-export FIXgsm=${FIXgsm:-$HOMEgfs/fix/fix_am}
+export FIXfv3=${FIXfv3:-$HOMEgfs/fix/orog}
+export FIXgsm=${FIXgsm:-$HOMEgfs/fix/am}
 export CYCLVARS=${CYCLVARS:-"FSNOL=-2.,FSNOS=99999.,"}
 export FHOUR=${FHOUR:-0}
 export DELTSFC=${DELTSFC:-6}

diff --git a/scripts/exgdas_enkf_fcst.sh b/scripts/exgdas_enkf_fcst.sh
@@ -23,7 +23,7 @@ source "$HOMEgfs/ush/preamble.sh"
 # Directories.
 pwd=$(pwd)
 export FIX_DIR=${FIX_DIR:-$HOMEgfs/fix}
-export FIX_AM=${FIX_AM:-$FIX_DIR/fix_am}
+export FIX_AM=${FIX_AM:-$FIX_DIR/am}
 
 # Utilities
 export NCP=${NCP:-"/bin/cp -p"}

diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh
@@ -57,8 +57,8 @@ CYCLESH=${CYCLESH:-$HOMEgfs/ush/global_cycle.sh}
 export CYCLEXEC=${CYCLEXEC:-$HOMEgfs/exec/global_cycle}
 APRUN_CYCLE=${APRUN_CYCLE:-${APRUN:-""}}
 NTHREADS_CYCLE=${NTHREADS_CYCLE:-${NTHREADS:-1}}
-export FIXfv3=${FIXfv3:-$HOMEgfs/fix/fix_fv3_gmted2010}
-export FIXgsm=${FIXgsm:-$HOMEgfs/fix/fix_am}
+export FIXfv3=${FIXfv3:-$HOMEgfs/fix/orog}
+export FIXgsm=${FIXgsm:-$HOMEgfs/fix/am}
 export CYCLVARS=${CYCLVARS:-"FSNOL=-2.,FSNOS=99999.,"}
 export FHOUR=${FHOUR:-0}
 export DELTSFC=${DELTSFC:-6}

diff --git a/scripts/exglobal_atmos_analysis_calc.sh b/scripts/exglobal_atmos_analysis_calc.sh
@@ -23,7 +23,7 @@ source "$HOMEgfs/ush/preamble.sh"
 
 #  Directories.
 pwd=$(pwd)
-export FIXgsm=${FIXgsm:-$HOMEgfs/fix/fix_am}
+export FIXgsm=${FIXgsm:-$HOMEgfs/fix/am}
 
 # Base variables
 CDATE=${CDATE:-"2001010100"}
@@ -72,7 +72,7 @@ CALCANLPY=${CALCANLPY:-$HOMEgfs/ush/calcanl_gfs.py}
 
 DOGAUSFCANL=${DOGAUSFCANL-"NO"}
 GAUSFCANLSH=${GAUSFCANLSH:-$HOMEgfs/ush/gaussian_sfcanl.sh}
-export GAUSFCANLEXE=${GAUSFCANLEXE:-$HOMEgfs/exec/gaussian_sfcanl.exe}
+export GAUSFCANLEXE=${GAUSFCANLEXE:-$HOMEgfs/exec/gaussian_sfcanl.x}
 NTHREADS_GAUSFCANL=${NTHREADS_GAUSFCANL:-1}
 APRUN_GAUSFCANL=${APRUN_GAUSFCANL:-${APRUN:-""}}
 

diff --git a/scripts/exglobal_atmos_sfcanl.sh b/scripts/exglobal_atmos_sfcanl.sh
@@ -61,8 +61,8 @@ export SNOW_NUDGE_COEFF=${SNOW_NUDGE_COEFF:-'-2.'}
 export CYCLVARS=${CYCLVARS:-""}
 export FHOUR=${FHOUR:-0}
 export DELTSFC=${DELTSFC:-6}
-export FIXgsm=${FIXgsm:-$HOMEgfs/fix/fix_am}
-export FIXfv3=${FIXfv3:-$HOMEgfs/fix/fix_fv3_gmted2010}
+export FIXgsm=${FIXgsm:-$HOMEgfs/fix/am}
+export FIXfv3=${FIXfv3:-$HOMEgfs/fix/orog}
 
 # FV3 specific info (required for global_cycle)
 export CASE=${CASE:-"C384"}

diff --git a/scripts/run_reg2grb2.sh b/scripts/run_reg2grb2.sh
@@ -5,7 +5,7 @@ source "$HOMEgfs/ush/preamble.sh"
 #requires grib_util module 
 
 MOM6REGRID=${MOM6REGRID:-$HOMEgfs}
-export mask_file=$MOM6REGRID/fix/fix_reg2grb2/mask.0p25x0p25.grb2
+export mask_file=$MOM6REGRID/fix/reg2grb2/mask.0p25x0p25.grb2
 
 # offline testing:
 #export DATA=