Skip to content

Commit

Permalink
Optimize DA clone, build, and link (#931)
Browse files Browse the repository at this point in the history 
The PR contains changes to optimize the DA clone, build, and link.   Changes are made to `checkout.sh`, `build_all.sh`, and `link_workflow.sh` in the g-w `sorc/` directory.  These changes are in g-w branch `feature/clone`

Two arguments are added to `checkout.sh` to allow the user to specify which DA package to build the global workflow with.  These options are 
- `-g`: clone from the [GSI](https://github.com/NOAA-EMC/GSI) repo and build the g-w for GSI-based DA
- `-u`:  clone from the [GDASApp](https://github.com/NOAA-EMC/GDASApp) repo and build the g-w for UFS-based DA

If no option is specified, `checkout.sh` does not clone any DA and DA related repos.  This is the default behavior of `checkout.sh`.   (_DA related_ repos include [GLDAS](https://github.com/NOAA-EMC/GLDAS), [GSI-utils](https://github.com/NOAA-EMC/GSI-utils), and [GSI-Monitor](https://github.com/NOAA-EMC/GSI-Monitor).)

`build_all.sh` is modified to detect which repos and have been cloned and to build accordingly.   `link_workflow.sh` is modified to detect which directories are present and link/copy accordingly.

Closes #930
  • Loading branch information
@RussTreadon-NOAA
RussTreadon-NOAA authored Jul 26, 2022
1 parent 2cad536 commit 4eb296f
Show file tree
Hide file tree
Showing 3 changed files with 184 additions and 89 deletions.
52 changes: 36 additions & 16 deletions sorc/build_all.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,10 @@ $Build_ufs_model && {
}

#------------------------------------
# build GSI and EnKF
# build GSI and EnKF - optional checkout
#------------------------------------
$Build_gsi_enkf && {
if [ -d gsi_enkf.fd ]; then
$Build_gsi_enkf && {
echo " .... Building gsi and enkf .... "
./build_gsi_enkf.sh $_ops_opt $_verbose_opt > $logs_dir/build_gsi_enkf.log 2>&1
rc=$?
Expand All @@ -132,11 +133,15 @@ $Build_gsi_enkf && {
fi
((err+=$rc))
}
else
echo " .... Skip building gsi and enkf .... "
fi

#------------------------------------
# build gsi utilities
#------------------------------------
$Build_gsi_utils && {
if [ -d gsi_utils.fd ]; then
$Build_gsi_utils && {
echo " .... Building gsi utilities .... "
./build_gsi_utils.sh $_ops_opt $_verbose_opt > $logs_dir/build_gsi_utils.log 2>&1
rc=$?
Expand All @@ -146,25 +151,33 @@ $Build_gsi_utils && {
fi
((err+=$rc))
}
else
echo " .... Skip building gsi utilities .... "
fi

#------------------------------------
# build gdas
#------------------------------------
$Build_gdas && {
echo " .... Building GDASApp .... "
./build_gdas.sh > $logs_dir/build_gdas.log 2>&1
rc=$?
if [[ $rc -ne 0 ]] ; then
echo "Fatal error in building GDAS."
echo "The log file is in $logs_dir/build_gdas.log"
fi
((err+=$rc))
# build gdas - optional checkout
#------------------------------------
if [ -d gdas.cd ]; then
$Build_gdas && {
echo " .... Building GDASApp .... "
./build_gdas.sh $_verbose_opt > $logs_dir/build_gdas.log 2>&1
rc=$?
if [[ $rc -ne 0 ]] ; then
echo "Fatal error in building GDASApp."
echo "The log file is in $logs_dir/build_gdas.log"
fi
((err+=$rc))
}
else
echo " .... Skip building GDASApp .... "
fi

#------------------------------------
# build gsi monitor
#------------------------------------
$Build_gsi_monitor && {
if [ -d gsi_monitor.fd ]; then
$Build_gsi_monitor && {
echo " .... Building gsi monitor .... "
./build_gsi_monitor.sh $_ops_opt $_verbose_opt > $logs_dir/build_gsi_monitor.log 2>&1
rc=$?
Expand All @@ -174,6 +187,9 @@ $Build_gsi_monitor && {
fi
((err+=$rc))
}
else
echo " .... Skip building gsi monitor .... "
fi

#------------------------------------
# build UPP
Expand Down Expand Up @@ -206,7 +222,8 @@ $Build_ufs_utils && {
#------------------------------------
# build gldas
#------------------------------------
$Build_gldas && {
if [ -d gldas.fd ]; then
$Build_gldas && {
echo " .... Building gldas .... "
./build_gldas.sh $_verbose_opt > $logs_dir/build_gldas.log 2>&1
rc=$?
Expand All @@ -216,6 +233,9 @@ $Build_gldas && {
fi
((err+=$rc))
}
else
echo " .... Skip building gldas .... "
fi

#------------------------------------
# build gfs_wafs - optional checkout
Expand Down
56 changes: 43 additions & 13 deletions sorc/checkout.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ Usage: $BASH_SOURCE [-c][-h][-m ufs_hash][-o]
Check out this UFS hash instead of the default
-o:
Check out operational-only code (GTG and WAFS)
-g:
Check out GSI for GSI-based DA
-u:
Check out GDASApp for UFS-based DA
EOF
exit 1
}
Expand Down Expand Up @@ -56,7 +60,7 @@ function checkout() {
fi

cd "${topdir}"
if [[ -d "${dir}" && "${CLEAN:-NO}" == "YES" ]]; then
if [[ -d "${dir}" && $CLEAN == "YES" ]]; then
echo "|-- Removing existing clone in ${dir}"
rm -Rf "$dir"
fi
Expand Down Expand Up @@ -96,13 +100,30 @@ function checkout() {
return 0
}

while getopts ":chm:o" option; do
# Set defaults for variables toggled by options
export CLEAN="NO"
CHECKOUT_GSI="NO"
CHECKOUT_GDAS="NO"
checkout_gtg="NO"
checkout_wafs="NO"
ufs_model_hash="b97375c"

# Parse command line arguments
while getopts ":chgum:o" option; do
case $option in
c)
echo "Recieved -c flag, will delete any existing directories and start clean"
export CLEAN="YES"
;;
g)
echo "Receieved -g flag for optional checkout of GSI-based DA"
CHECKOUT_GSI="YES"
;;
h) usage;;
u)
echo "Received -u flag for optional checkout of UFS-based DA"
CHECKOUT_GDAS="YES"
;;
o)
echo "Received -o flag for optional checkout of operational-only codes"
checkout_gtg="YES"
Expand Down Expand Up @@ -130,20 +151,29 @@ mkdir -p ${logdir}

# The checkout version should always be a speciifc commit (hash or tag), not a branch
errs=0
checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-b97375c}" ; errs=$((errs + $?))
checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "67f5ab4" ; errs=$((errs + $?))
checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "322cc7b" ; errs=$((errs + $?))
checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "acf8870" ; errs=$((errs + $?))
checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "5952c9d" ; errs=$((errs + $?))
checkout "gldas.fd" "https://github.com/NOAA-EMC/GLDAS.git" "fd8ba62" ; errs=$((errs + $?))
checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "a2b0817" ; errs=$((errs + $?))
checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?))

if [[ "${checkout_wafs:-NO}" == "YES" ]]; then
checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash}"; errs=$((errs + $?))
checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "a2b0817" ; errs=$((errs + $?))
checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?))

if [[ $CHECKOUT_GSI == "YES" ]]; then
checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "67f5ab4"; errs=$((errs + $?))
fi

if [[ $CHECKOUT_GDAS == "YES" ]]; then
checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "5952c9d"; errs=$((errs + $?))
fi

if [[ $CHECKOUT_GSI == "YES" || $CHECKOUT_GDAS == "YES" ]]; then
checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "322cc7b"; errs=$((errs + $?))
checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "acf8870"; errs=$((errs + $?))
checkout "gldas.fd" "https://github.com/NOAA-EMC/GLDAS.git" "fd8ba62"; errs=$((errs + $?))
fi

if [[ $checkout_wafs == "YES" ]]; then
checkout "gfs_wafs.fd" "https://github.com/NOAA-EMC/EMC_gfs_wafs.git" "014a0b8"; errs=$((errs + $?))
fi

if [[ "${checkout_gtg:-NO}" == "YES" ]]; then
if [[ $checkout_gtg == "YES" ]]; then
################################################################################
# checkout_gtg
## yes: The gtg code at NCAR private repository is available for ops. GFS only.
Expand Down
Loading

0 comments on commit 4eb296f

Please sign in to comment.