From da1fb440d02a5f8b5b69cd22a17e5a6fc55dde6c Mon Sep 17 00:00:00 2001 From: chunhua zhou Date: Fri, 29 Oct 2021 13:36:17 -0600 Subject: [PATCH] Changes for running EnKF updates (#244) * Changes for running EnKF updates * Adding anavinfo for enkf * Use variables for EnKF cores and walltime * Cosmetic changes to exregional_run_enkf.sh, add some descriptions --- fix/gsi/anavinfo.enkf_fv3lam_hrrr | 67 +++++ jobs/JREGIONAL_RUN_ENKF | 110 +++++++ scripts/exregional_run_enkf.sh | 467 ++++++++++++++++++++++++++++++ ush/config_defaults.sh | 4 + ush/generate_FV3LAM_wflow.sh | 2 + ush/load_modules_run_task.sh | 2 +- ush/templates/FV3LAM_wflow.xml | 38 ++- 7 files changed, 688 insertions(+), 2 deletions(-) create mode 100644 fix/gsi/anavinfo.enkf_fv3lam_hrrr create mode 100755 jobs/JREGIONAL_RUN_ENKF create mode 100755 scripts/exregional_run_enkf.sh diff --git a/fix/gsi/anavinfo.enkf_fv3lam_hrrr b/fix/gsi/anavinfo.enkf_fv3lam_hrrr new file mode 100644 index 000000000..c93b9c3a1 --- /dev/null +++ b/fix/gsi/anavinfo.enkf_fv3lam_hrrr @@ -0,0 +1,67 @@ +met_guess:: +!var level crtm_use desc orig_name + ps 1 -1 surface_pressure ps + z 1 -1 geopotential_height phis + u 65 2 zonal_wind u + v 65 2 meridional_wind v + tv 65 2 virtual_temperature tv + q 65 2 specific_humidity sphu + oz 65 2 ozone ozone + delp 65 -1 fv3 del pressure delp +:: + +state_derivatives:: +!var level src + ps 1 met_guess + u 65 met_guess + v 65 met_guess + tv 65 met_guess + q 65 met_guess + oz 65 met_guess + cw 65 met_guess + prse 66 met_guess +:: + +state_tendencies:: +!var levels source + u 65 met_guess + v 65 met_guess + tv 65 met_guess + q 65 met_guess + oz 65 met_guess + prse 66 met_guess +:: + +state_vector:: +!var level itracer source funcof + u 65 0 met_guess u + v 65 0 met_guess v + tv 65 0 met_guess tv + tsen 65 0 met_guess tv,q + q 65 1 met_guess q + oz 65 1 met_guess oz + prse 66 0 met_guess prse + ps 1 0 met_guess prse + sst 1 0 met_guess sst +:: +control_vector_enkf:: +!var level itracer as/tsfc_sdv an_amp0 source funcof + u 65 0 1.00 -1.0 state u,v + v 65 0 1.00 -1.0 state u,v + ps 1 0 0.50 -1.0 state prse + t 65 0 0.70 -1.0 state tv + oz 65 0 0.70 -1.0 state oz + q 65 1 0.70 -1.0 state q +:: + +control_vector:: +!var level itracer as/tsfc_sdv an_amp0 source funcof + sf 65 0 1.00 -1.0 state u,v + vp 65 0 1.00 -1.0 state u,v + ps 1 0 0.50 -1.0 state prse + t 65 0 0.70 -1.0 state tv + q 65 1 0.70 -1.0 state q + sst 1 0 1.00 -1.0 state sst + stl 1 0 1.00 -1.0 motley sst + sti 1 0 1.00 -1.0 motley sst +:: diff --git a/jobs/JREGIONAL_RUN_ENKF b/jobs/JREGIONAL_RUN_ENKF new file mode 100755 index 000000000..a329d496b --- /dev/null +++ b/jobs/JREGIONAL_RUN_ENKF @@ -0,0 +1,110 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This J-JOB script runs the EnKF updates +# for the FV3-LAM model +# +#----------------------------------------------------------------------- + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs a analysis with FV3 for +the specified cycle. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Create the working directory under the cycle directory. +# +#----------------------------------------------------------------------- +# CYCLE_TYPE is to indicate which type of cycle this job for: +# spinup: this job is for spin-up cycles +# others(prod): this job is for product cycle +# +CYCLE_TYPE=${CYCLE_TYPE:-prod} +if [ ${CYCLE_TYPE} == "spinup" ]; then + enkfworkdir="${CYCLE_DIR}/enkfupdt_spinup" +else + enkfworkdir="${CYCLE_DIR}/enkfupdt" +fi +rm -fr ${enkfworkdir} +mkdir_vrfy -p ${enkfworkdir} +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_enkf.sh \ + cycle_dir="${CYCLE_DIR}" \ + cycle_type="${CYCLE_TYPE}" \ + enkfworkdir="${enkfworkdir}" \ + NWGES_DIR="${NWGES_DIR}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/scripts/exregional_run_enkf.sh b/scripts/exregional_run_enkf.sh new file mode 100755 index 000000000..f07cb6037 --- /dev/null +++ b/scripts/exregional_run_enkf.sh @@ -0,0 +1,467 @@ +#!/bin/bash +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh + +function ncvarlst_noaxis_time { ncks --trd -m ${1} | grep -E ': type' | cut -f 1 -d ' ' | sed 's/://' | sort |grep -v -i -E "axis|time" ; } +function ncvarlst_noaxis_time_new { ncks -m ${1} | grep -E 'name.*=' | cut -f 2 -d '=' | grep -o '"*.*"' | sed 's/"//g' | sort |grep -v -i -E "axis|time" ; } +export HDF5_USE_FILE_LOCKING=FALSE #clt to avoild recenter's error "NetCDF: HDF error" +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs EnKF analysis with FV3 for the +specified cycle. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" "cycle_type" "enkfworkdir" "NWGES_DIR" ) +process_args valid_args "$@" + +cycle_type=${cycle_type:-prod} + +case $MACHINE in +# +"WCOSS_C" | "WCOSS") +# + module load NCO/4.7.0 + module list + ulimit -s unlimited + ulimit -a + APRUN="mpirun -l -np ${PE_MEMBER01}" + ;; +# +"WCOSS_DELL_P3") +# + module load NCO/4.7.0 + module list + ulimit -s unlimited + ulimit -a + APRUN="mpirun -l -np ${PE_MEMBER01}" + ;; +# +"THEIA") +# + ulimit -s unlimited + ulimit -a + np=${SLURM_NTASKS} + APRUN="mpirun -np ${np}" + ;; +# +"HERA") + module load nco/4.9.3 + ulimit -s unlimited + ulimit -v unlimited + ulimit -a + export OMP_NUM_THREADS=1 + export OMP_STACKSIZE=300M + APRUN="srun" + ;; +# +"ORION") + ulimit -s unlimited + ulimit -a + export OMP_NUM_THREADS=1 + export OMP_STACKSIZE=1024M + APRUN="srun" + ;; +# +"JET") + ulimit -s unlimited + ulimit -a + APRUN="srun" + ;; +# +"ODIN") +# + module list + + ulimit -s unlimited + ulimit -a + APRUN="srun -n ${PE_MEMBER01}" + ;; +# +esac +# +#----------------------------------------------------------------------- +# +# Extract from CDATE the starting year, month, day, and hour of the +# forecast. These are needed below for various operations. +# +#----------------------------------------------------------------------- +# +START_DATE=$(echo "${CDATE}" | sed 's/\([[:digit:]]\{2\}\)$/ \1/') + +YYYYMMDDHH=$(date +%Y%m%d%H -d "${START_DATE}") + +vlddate=$CDATE + +# +#----------------------------------------------------------------------- +# +# Go to working directory. +# Define fix path +# +#----------------------------------------------------------------------- +# + +cd_vrfy $enkfworkdir +fixgriddir=$FIX_GSI/${PREDEF_GRID_NAME} + +# +#----------------------------------------------------------------------- +# +# Loop through the members, copy over the background and +# observer output (diag*ges*) files to the running directory +# +#----------------------------------------------------------------------- +# + cp_vrfy ${fixgriddir}/fv3_coupler.res coupler.res + cp_vrfy ${fixgriddir}/fv3_akbk fv3sar_tile1_akbk.nc + cp_vrfy ${fixgriddir}/fv3_grid_spec fv3sar_tile1_grid_spec.nc + + for imem in $(seq 1 $nens) ensmean; do + + if [ ${imem} == "ensmean" ]; then + memchar="ensmean" + memcharv0="ensmean" + else + memchar="mem"$(printf %04i $imem) + memcharv0="mem"$(printf %03i $imem) + fi + slash_ensmem_subdir=$memchar + if [ ${cycle_type} == "spinup" ]; then + bkpath=${cycle_dir}/${slash_ensmem_subdir}/fcst_fv3lam_spinup/INPUT + observer_nwges_dir="${NWGES_DIR}/${slash_ensmem_subdir}/observer_gsi_spinup" + else + bkpath=${cycle_dir}/${slash_ensmem_subdir}/fcst_fv3lam/INPUT + observer_nwges_dir="${NWGES_DIR}/${slash_ensmem_subdir}/observer_gsi" + fi + + cp_vrfy ${bkpath}/fv_core.res.tile1.nc fv3_${memcharv0}_dynvars + cp_vrfy ${bkpath}/fv_tracer.res.tile1.nc fv3_${memcharv0}_tracer + cp_vrfy ${bkpath}/sfc_data.nc fv3_${memcharv0}_sfcdata + +# +#----------------------------------------------------------------------- +# +# get the variable list from the tracer and dynvar files +# +#----------------------------------------------------------------------- +# + if [ $imem == 1 ];then + ncvarlst_noaxis_time_new fv3_${memcharv0}_tracer > nck_tracer_list.txt + ncvarlst_noaxis_time_new fv3_${memcharv0}_dynvars > nck_dynvar_list.txt + fi + user_nck_dynvar_list=`cat nck_dynvar_list.txt|paste -sd "," - | tr -d '[:space:]'` + user_nck_tracer_list=`cat nck_tracer_list.txt |paste -sd "," - | tr -d '[:space:]'` +# This file contains horizontal grid information + ncrename -d yaxis_1,yaxis_2 -v yaxis_1,yaxis_2 fv3_${memcharv0}_tracer +# +#----------------------------------------------------------------------- +# +# Copy tracer variables from tracer file to dynvars file, and +# get a combined dynvartracer background +# +#----------------------------------------------------------------------- +# + ncks -A -v $user_nck_tracer_list fv3_${memcharv0}_tracer fv3_${memcharv0}_dynvars + mv fv3_${memcharv0}_dynvars fv3sar_tile1_${memcharv0}_dynvartracer +# +#----------------------------------------------------------------------- +# +# Copy observer outputs (diag*ges*) to the working directory +# +#----------------------------------------------------------------------- +# + for diagfile0 in `ls ${observer_nwges_dir}/diag*ges*`; do + diagfile=$(basename $diagfile0) + cp_vrfy $diagfile0 ${diagfile}_$memcharv0 + done + + done + +# +#---------------------------------------------------------------------- +# +# Set GSI fix files +# +#---------------------------------------------------------------------- +# +ANAVINFO=${FIX_GSI}/anavinfo.enkf_fv3lam_hrrr +CONVINFO=${FIX_GSI}/${CONVINFO_FN} +SATINFO=${FIX_GSI}/global_satinfo.txt +OZINFO=${FIX_GSI}/global_ozinfo.txt + +cp_vrfy ${ANAVINFO} anavinfo +cp_vrfy $SATINFO satinfo +cp_vrfy $CONVINFO convinfo +cp_vrfy $OZINFO ozinfo + +# +#---------------------------------------------------------------------- +# +# Set namelist parameters for EnKF +# +#---------------------------------------------------------------------- +# + +EnKFTracerVars=${EnKFTracerVar:-"sphum,o3mr"} +ldo_enscalc_option=${ldo_enscalc_option:-0} + +# We expect 81 total files to be present (80 enkf + 1 mean) +nens=${nens:-81} +USEGFSO3=.false. +# Not using FGAT or 4DEnVar, so hardwire nhr_assimilation to 3 +nhr_assimilation=3. +vs=1. +fstat=.false. +i_gsdcldanal_type=0 +use_gfs_nemsio=.true., + +# +#---------------------------------------------------------------------- +# +# Make enkf namelist +# +#---------------------------------------------------------------------- +# + cat > enkf.nml << EOFnml + &nam_enkf + datestring="$vlddate",datapath="$enkfworkdir/", + analpertwtnh=0.85,analpertwtsh=0.85,analpertwttr=0.85, + covinflatemax=1.e2,covinflatemin=1,pseudo_rh=.true.,iassim_order=0, + corrlengthnh=400,corrlengthsh=400,corrlengthtr=400, + lnsigcutoffnh=0.5,lnsigcutoffsh=0.5,lnsigcutofftr=0.5, + lnsigcutoffpsnh=0.5,lnsigcutoffpssh=0.5,lnsigcutoffpstr=0.5, + lnsigcutoffsatnh=0.5,lnsigcutoffsatsh=0.5,lnsigcutoffsattr=0.5, + obtimelnh=1.e30,obtimelsh=1.e30,obtimeltr=1.e30, + saterrfact=1.0,numiter=1, + sprd_tol=1.e30,paoverpb_thresh=0.98, + nlons=${NX_RES:-396},nlats= ${NY_RES:-232}, nlevs= ${nlevs:-65},nanals=$nens, + deterministic=.true.,sortinc=.true.,lupd_satbiasc=.false., + reducedgrid=.true.,readin_localization=.false., + use_gfs_nemsio=.true.,imp_physics=99,lupp=.false., + univaroz=.false.,adp_anglebc=.true.,angord=4,use_edges=.false.,emiss_bc=.true., + lobsdiag_forenkf=.false., + write_spread_diag=.false., + netcdf_diag=${netcdf_diag:-.false.}, + fv3_native=.true., + / + &satobs_enkf + sattypes_rad(1) = 'amsua_n15', dsis(1) = 'amsua_n15', + sattypes_rad(2) = 'amsua_n18', dsis(2) = 'amsua_n18', + sattypes_rad(3) = 'amsua_n19', dsis(3) = 'amsua_n19', + sattypes_rad(4) = 'amsub_n16', dsis(4) = 'amsub_n16', + sattypes_rad(5) = 'amsub_n17', dsis(5) = 'amsub_n17', + sattypes_rad(6) = 'amsua_aqua', dsis(6) = 'amsua_aqua', + sattypes_rad(7) = 'amsua_metop-a', dsis(7) = 'amsua_metop-a', + sattypes_rad(8) = 'airs_aqua', dsis(8) = 'airs_aqua', + sattypes_rad(9) = 'hirs3_n17', dsis(9) = 'hirs3_n17', + sattypes_rad(10)= 'hirs4_n19', dsis(10)= 'hirs4_n19', + sattypes_rad(11)= 'hirs4_metop-a', dsis(11)= 'hirs4_metop-a', + sattypes_rad(12)= 'mhs_n18', dsis(12)= 'mhs_n18', + sattypes_rad(13)= 'mhs_n19', dsis(13)= 'mhs_n19', + sattypes_rad(14)= 'mhs_metop-a', dsis(14)= 'mhs_metop-a', + sattypes_rad(15)= 'goes_img_g11', dsis(15)= 'imgr_g11', + sattypes_rad(16)= 'goes_img_g12', dsis(16)= 'imgr_g12', + sattypes_rad(17)= 'goes_img_g13', dsis(17)= 'imgr_g13', + sattypes_rad(18)= 'goes_img_g14', dsis(18)= 'imgr_g14', + sattypes_rad(19)= 'goes_img_g15', dsis(19)= 'imgr_g15', + sattypes_rad(20)= 'avhrr_n18', dsis(20)= 'avhrr3_n18', + sattypes_rad(21)= 'avhrr_metop-a', dsis(21)= 'avhrr3_metop-a', + sattypes_rad(22)= 'avhrr_n19', dsis(22)= 'avhrr3_n19', + sattypes_rad(23)= 'amsre_aqua', dsis(23)= 'amsre_aqua', + sattypes_rad(24)= 'ssmis_f16', dsis(24)= 'ssmis_f16', + sattypes_rad(25)= 'ssmis_f17', dsis(25)= 'ssmis_f17', + sattypes_rad(26)= 'ssmis_f18', dsis(26)= 'ssmis_f18', + sattypes_rad(27)= 'ssmis_f19', dsis(27)= 'ssmis_f19', + sattypes_rad(28)= 'ssmis_f20', dsis(28)= 'ssmis_f20', + sattypes_rad(29)= 'sndrd1_g11', dsis(29)= 'sndrD1_g11', + sattypes_rad(30)= 'sndrd2_g11', dsis(30)= 'sndrD2_g11', + sattypes_rad(31)= 'sndrd3_g11', dsis(31)= 'sndrD3_g11', + sattypes_rad(32)= 'sndrd4_g11', dsis(32)= 'sndrD4_g11', + sattypes_rad(33)= 'sndrd1_g12', dsis(33)= 'sndrD1_g12', + sattypes_rad(34)= 'sndrd2_g12', dsis(34)= 'sndrD2_g12', + sattypes_rad(35)= 'sndrd3_g12', dsis(35)= 'sndrD3_g12', + sattypes_rad(36)= 'sndrd4_g12', dsis(36)= 'sndrD4_g12', + sattypes_rad(37)= 'sndrd1_g13', dsis(37)= 'sndrD1_g13', + sattypes_rad(38)= 'sndrd2_g13', dsis(38)= 'sndrD2_g13', + sattypes_rad(39)= 'sndrd3_g13', dsis(39)= 'sndrD3_g13', + sattypes_rad(40)= 'sndrd4_g13', dsis(40)= 'sndrD4_g13', + sattypes_rad(41)= 'sndrd1_g14', dsis(41)= 'sndrD1_g14', + sattypes_rad(42)= 'sndrd2_g14', dsis(42)= 'sndrD2_g14', + sattypes_rad(43)= 'sndrd3_g14', dsis(43)= 'sndrD3_g14', + sattypes_rad(44)= 'sndrd4_g14', dsis(44)= 'sndrD4_g14', + sattypes_rad(45)= 'sndrd1_g15', dsis(45)= 'sndrD1_g15', + sattypes_rad(46)= 'sndrd2_g15', dsis(46)= 'sndrD2_g15', + sattypes_rad(47)= 'sndrd3_g15', dsis(47)= 'sndrD3_g15', + sattypes_rad(48)= 'sndrd4_g15', dsis(48)= 'sndrD4_g15', + sattypes_rad(49)= 'iasi_metop-a', dsis(49)= 'iasi_metop-a', + sattypes_rad(50)= 'seviri_m08', dsis(50)= 'seviri_m08', + sattypes_rad(51)= 'seviri_m09', dsis(51)= 'seviri_m09', + sattypes_rad(52)= 'seviri_m10', dsis(52)= 'seviri_m10', + sattypes_rad(53)= 'amsua_metop-b', dsis(53)= 'amsua_metop-b', + sattypes_rad(54)= 'hirs4_metop-b', dsis(54)= 'hirs4_metop-b', + sattypes_rad(55)= 'mhs_metop-b', dsis(55)= 'mhs_metop-b', + sattypes_rad(56)= 'iasi_metop-b', dsis(56)= 'iasi_metop-b', + sattypes_rad(57)= 'avhrr_metop-b', dsis(57)= 'avhrr3_metop-b', + sattypes_rad(58)= 'atms_npp', dsis(58)= 'atms_npp', + sattypes_rad(59)= 'atms_n20', dsis(59)= 'atms_n20', + sattypes_rad(60)= 'cris_npp', dsis(60)= 'cris_npp', + sattypes_rad(61)= 'cris-fsr_npp', dsis(61)= 'cris-fsr_npp', + sattypes_rad(62)= 'cris-fsr_n20', dsis(62)= 'cris-fsr_n20', + sattypes_rad(63)= 'gmi_gpm', dsis(63)= 'gmi_gpm', + sattypes_rad(64)= 'saphir_meghat', dsis(64)= 'saphir_meghat', + / + &ozobs_enkf + sattypes_oz(1) = 'sbuv2_n16', + sattypes_oz(2) = 'sbuv2_n17', + sattypes_oz(3) = 'sbuv2_n18', + sattypes_oz(4) = 'sbuv2_n19', + sattypes_oz(5) = 'omi_aura', + sattypes_oz(6) = 'gome_metop-a', + sattypes_oz(7) = 'gome_metop-b', + sattypes_oz(8) = 'mls30_aura', + / + &nam_fv3 + fv3fixpath="XXX",nx_res=${NX_RES:-396},ny_res=${NY_RES-232},ntiles=1, + / +EOFnml + +# +#----------------------------------------------------------------------- +# +# Copy the EnKF executable to the run directory. +# +#----------------------------------------------------------------------- +# + +echo pwd is `pwd` +ENKFEXEC=${EXECDIR}/enkf_fv3reg.x + +if [ -f ${ENKFEXEC} ]; then + print_info_msg "$VERBOSE" " +Copying the EnKF executable to the run directory..." + cp_vrfy ${ENKFEXEC} ${enkfworkdir}/enkf.x +else + print_err_msg_exit "\ +The EnKF executable specified in ENKFEXEC does not exist: + EnKF_EXEC = \"${ENKFEXEC}\" +Build EnKF and rerun." +fi + +# +#----------------------------------------------------------------------- +# +# Run the EnKF +# +#----------------------------------------------------------------------- +# +${APRUN} $enkfworkdir/enkf.x < enkf.nml 1>stdout 2>stderr || print_err_msg_exit "\ +Call to executable to run EnKF returned with nonzero exit code." + +# +#----------------------------------------------------------------------- +# +# Loop through the members, restore the EnKF analysis back to +# separate tracer and dynvar files, copy them +# to nwges staging locations +# +#----------------------------------------------------------------------- +# + +for imem in $(seq 1 $nens); do + + memchar="mem"$(printf %04i $imem) + memcharv0="mem"$(printf %03i $imem) + memstr=$memchar + slash_ensmem_subdir=$memchar + + if [ ${cycle_type} == "spinup" ]; then + enkfanal_nwges_dir=${NWGES_DIR}/${slash_ensmem_subdir}/anal_enkf_spinup + bkpath=${cycle_dir}/${slash_ensmem_subdir}/fcst_fv3lam_spinup/INPUT + else + enkfanal_nwges_dir=${NWGES_DIR}/${slash_ensmem_subdir}/anal_enkf + bkpath=${cycle_dir}/${slash_ensmem_subdir}/fcst_fv3lam/INPUT + fi + mkdir_vrfy -p ${enkfanal_nwges_dir} + + FileUpdated=fv3sar_tile1_${memcharv0}_dynvartracer + + cp_vrfy $bkpath/fv_tracer.res.tile1.nc ./${memcharv0}_fv3_tracer + cp_vrfy $bkpath/fv_tracer.res.tile1.nc ./${memcharv0}_fv3_dynvars + if [ $imem == 1 ];then + ncvarlst_noaxis_time_new ${memcharv0}_fv3_tracer > nck_tracer_list.txt + ncvarlst_noaxis_time_new ${memcharv0}_fv3_dynvars > nck_dynvar_list.txt + fi + user_nck_dynvar_list=`cat nck_dynvar_list.txt|paste -sd "," - | tr -d '[:space:]'` + user_nck_tracer_list=`cat nck_tracer_list.txt |paste -sd "," - | tr -d '[:space:]'` + + ncks -A -v $user_nck_dynvar_list $FileUpdated ${memcharv0}_fv3_dynvars + cp_vrfy ${memcharv0}_fv3_dynvars ${enkfanal_nwges_dir}/fv_core.res.tile1.nc + ncks -A -v $user_nck_tracer_list $FileUpdated ${memcharv0}_fv3_tracer + ncks --no_abc -O -x -v yaxis_2 ${memcharv0}_fv3_tracer tmp_${memcharv0}_tracer + cp_vrfy tmp_${memcharv0}_tracer ${enkfanal_nwges_dir}/fv_tracer.res.tile1.nc + +done + +print_info_msg " +======================================================================== +EnKF PROCESS completed successfully!!! + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 57d90d486..874030848 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1598,6 +1598,7 @@ MAXTRIES_MAKE_LBCS="1" MAXTRIES_RUN_PREPSTART="1" MAXTRIES_RUN_FCST="1" MAXTRIES_ANAL_GSI="1" +MAXTRIES_ANAL_ENKF="1" MAXTRIES_RUN_POST="1" MAXTRIES_RUN_WGRIB2="1" MAXTRIES_RUN_ANAL="1" @@ -1723,12 +1724,15 @@ TILE_SETS="full" # DO_GSIOBSERVER: # Decide whether or not to run GSI observer # +# DO_ENKFUPDATE: +# Decide whether or not to run EnKF update for the ensemble members #----------------------------------------------------------------------- # DO_ENSEMBLE="FALSE" NUM_ENS_MEMBERS="1" DO_ENSCONTROL="FALSE" DO_GSIOBSERVER="FALSE" +DO_ENKFUPDATE="FALSE" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 7d60a3fa5..9e2045266 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -307,6 +307,7 @@ settings="\ 'maxtries_run_prepstart': ${MAXTRIES_RUN_PREPSTART} 'maxtries_run_fcst': ${MAXTRIES_RUN_FCST} 'maxtries_anal_gsi': ${MAXTRIES_ANAL_GSI} + 'maxtries_anal_enkf': ${MAXTRIES_ANAL_ENKF} 'maxtries_run_post': ${MAXTRIES_RUN_POST} 'maxtries_run_wgrib2': ${MAXTRIES_RUN_WGRIB2} 'maxtries_process_radarref': ${MAXTRIES_PROCESS_RADARREF} @@ -389,6 +390,7 @@ settings="\ 'slash_ensmem_subdir': ${slash_ensmem_subdir} 'do_enscontrol': ${DO_ENSCONTROL} 'do_gsiobserver': ${DO_GSIOBSERVER} + 'do_enkfupdate': ${DO_ENKFUPDATE} # # data assimilation related parameters. # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 0e30ac4b9..c777bd196 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -161,7 +161,7 @@ workflow task specified by task_name failed: task_name = \"${task_name}\" env_fp = \"${env_fp}\"" -if [ ${task_name} = "run_anal_gsi" ] || [ ${task_name} = "run_prepstart" ]; then +if [ ${task_name} = "run_anal_gsi" ] || [ ${task_name} = "run_enkfupdt" || [ ${task_name} = "run_prepstart" ]; then source "${env_fp}_DA" || print_err_msg_exit "\ Sourcing platform- and compiler-specific environment file (env_fp) for the workflow task specified by task_name failed: diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index b4e2668c0..1495dee80 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -54,6 +54,7 @@ Workflow task names. + @@ -1341,7 +1342,7 @@ MODULES_RUN_TASK_FP script. ************************************************************************ --> {%- if do_ensemble %} - + {%- for m in range(1, num_ens_members+1) -%} {%- set fmtstr=" %0"~ndigits_ensmem_names~"d" -%} @@ -1390,4 +1391,39 @@ MODULES_RUN_TASK_FP script. {%- endif %} +{%- if do_enkfupdate %} + + + &RSRV_ANALYSIS; + &WALL_LIMIT_ANAL; + &LOAD_MODULES_RUN_TASK_FP; "&RUN_ENKFUPDT_TN;" &JOBSDIR;/JREGIONAL_RUN_ENKF + + {% if machine in ["JET", "HERA"] -%} + {{ ncores_run_anal }} + {{ native_run_anal }} &RRFS_RESERVE; + {% else -%} + {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }} + &NCORES_PER_NODE; + {% endif -%} + {{ wtime_run_anal }} + &TAG;_&RUN_ENKFUPDT_TN; + &LOGDIR;/&RUN_ENKFUPDT_TN;_@Y@m@d@H.log + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + + nens{{ num_ens_members }} + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CYCLE_ROOT&CYCLE_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + + + + + + +{%- endif %} +