diff --git a/.gitignore b/.gitignore index 39d140fd65..04193fca0a 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,8 @@ parm/gdas/io parm/gdas/ioda parm/gdas/snow parm/gdas/soca +parm/gdas/jcb-gdas +parm/gdas/jcb-algorithms parm/monitor parm/post/AEROSOL_LUTS.dat parm/post/nam_micro_lookup.dat @@ -106,6 +108,7 @@ parm/post/ice.csv parm/post/ocnicepost.nml.jinja2 parm/ufs/noahmptable.tbl parm/ufs/model_configure.IN +parm/ufs/model_configure_nest.IN parm/ufs/MOM_input_*.IN parm/ufs/MOM6_data_table.IN parm/ufs/ice_in.IN @@ -194,3 +197,8 @@ versions/run.ver ush/python/wxflow workflow/wxflow ci/scripts/wxflow + +# jcb checkout and symlinks +ush/python/jcb +workflow/jcb +ci/scripts/jcb diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 5f64ba4d8d..8c70e31cc4 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -149,7 +149,7 @@ pipeline { } } sh(script: './link_workflow.sh') - sh(script: "echo ${HOMEgfs} > BUILT_semaphor") + //sh(script: "echo ${HOMEgfs} > BUILT_semaphor") } } if (env.CHANGE_ID && system == 'gfs') { diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml index d992938f7f..8a89ff25ec 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/ci/cases/pr/C96_atm3DVar.yaml @@ -15,3 +15,6 @@ arguments: gfs_cyc: 1 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + +skip_ci_on_hosts: + - wcoss2 diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml new file mode 100644 index 0000000000..994d3ef3a0 --- /dev/null +++ b/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -0,0 +1,22 @@ +experiment: + system: gfs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: ATM + resdetatmos: 96 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + idate: 2021122018 + edate: 2021122118 + nens: 0 + gfs_cyc: 4 + start: cold + yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml + +skip_ci_on_hosts: + - hera + - orion + - hercules diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/ci/cases/yamls/gfs_extended_ci.yaml new file mode 100644 index 0000000000..4d4f79e0e8 --- /dev/null +++ b/ci/cases/yamls/gfs_extended_ci.yaml @@ -0,0 +1,12 @@ +defaults: + !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml + +base: + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} + DO_GOES: "YES" + DO_BUFRSND: "YES" + DO_GEMPAK: "YES" + DO_AWIPS: "NO" + DO_NPOESS: "YES" + DO_GENESIS_FSU: "NO" + FHMAX_GFS: 384 diff --git a/env/HERA.env b/env/HERA.env index e047039c39..e7d7420b8f 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -49,6 +49,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then export APRUN_CALCFIMS="${launcher} -n 1" +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${launcher} -n 1" + elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then export CFP_MP="YES" diff --git a/env/HERCULES.env b/env/HERCULES.env index da5ad972f2..0b62120536 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -45,6 +45,10 @@ case ${step} in export APRUN_CALCFIMS="${launcher} -n 1" ;; + "prep_emissions") + + export APRUN="${launcher} -n 1" + ;; "waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll") export CFP_MP="YES" diff --git a/env/JET.env b/env/JET.env index 3b4c2c2c53..976e42a025 100755 --- a/env/JET.env +++ b/env/JET.env @@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then export APRUN_CALCFIMS="${launcher} -n 1" +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${launcher} -n 1" + elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then export CFP_MP="YES" diff --git a/env/ORION.env b/env/ORION.env index 6aac84a169..795346f0c6 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -44,6 +44,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then export APRUN_CALCFIMS="${launcher} -n 1" +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${launcher} -n 1" + elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || \ [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]] || [[ "${step}" == "wavepostbndpntbll" ]]; then diff --git a/env/S4.env b/env/S4.env index 9cbf8b7bdb..ce68fddb89 100755 --- a/env/S4.env +++ b/env/S4.env @@ -37,6 +37,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then export APRUN_CALCFIMS="${launcher} -n 1" +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${launcher} -n 1" + elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then export CFP_MP="YES" diff --git a/env/WCOSS2.env b/env/WCOSS2.env index ba55495655..ff0121e034 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -31,6 +31,10 @@ elif [[ "${step}" = "prepsnowobs" ]]; then export APRUN_CALCFIMS="${launcher} -n 1" +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${launcher} -n 1" + elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then export USE_CFP="YES" diff --git a/gempak/ush/gfs_meta_opc_na_ver b/gempak/ush/gfs_meta_opc_na_ver index d38ddacee0..3aaf93db68 100755 --- a/gempak/ush/gfs_meta_opc_na_ver +++ b/gempak/ush/gfs_meta_opc_na_ver @@ -33,8 +33,8 @@ fcsthr="f00" # seq won't give us any splitting problems, ignore warnings # shellcheck disable=SC2207,SC2312 case ${cyc} in - 00 | 12) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 96 12 120)) ;; - 06 | 18) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 90 12 126)) ;; + 00 | 12) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 96 12 120)) ;; + 06 | 18) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 90 12 126)) ;; *) echo "FATAL ERROR: Invalid cycle ${cyc} passed to ${BASH_SOURCE[0]}" exit 100 diff --git a/gempak/ush/gfs_meta_opc_np_ver b/gempak/ush/gfs_meta_opc_np_ver index 9446417403..0968b55747 100755 --- a/gempak/ush/gfs_meta_opc_np_ver +++ b/gempak/ush/gfs_meta_opc_np_ver @@ -33,8 +33,8 @@ fcsthr="f00" # seq won't give us any splitting problems, ignore warnings # shellcheck disable=SC2207,SC2312 case ${cyc} in - 00 | 12) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 96 12 120)) ;; - 06 | 18) IFS=$'\n' lookbacks=($(seq 6 6 84) $(seq 90 12 126)) ;; + 00 | 12) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 96 12 120)) ;; + 06 | 18) lookbacks=($(IFS=$'\n' seq 6 6 84) $(IFS=$'\n' seq 90 12 126)) ;; *) echo "FATAL ERROR: Invalid cycle ${cyc} passed to ${BASH_SOURCE[0]}" exit 100 diff --git a/gempak/ush/gfs_meta_ver.sh b/gempak/ush/gfs_meta_ver.sh index 1e00cd3094..eb8b5b15c6 100755 --- a/gempak/ush/gfs_meta_ver.sh +++ b/gempak/ush/gfs_meta_ver.sh @@ -32,7 +32,7 @@ MDL2="GFSHPC" #GENERATING THE METAFILES. # seq won't give us any splitting problems, ignore warnings # shellcheck disable=SC2207,SC2312 -IFS=$'\n' lookbacks=($(seq 6 6 180) $(seq 192 12 216)) +lookbacks=($(IFS=$'\n' seq 6 6 180) $(IFS=$'\n' seq 192 12 216)) for lookback in "${lookbacks[@]}"; do init_time="$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${lookback} hours")" init_PDY=${init_time:0:8} diff --git a/jobs/JGLOBAL_PREP_EMISSIONS b/jobs/JGLOBAL_PREP_EMISSIONS new file mode 100755 index 0000000000..84edac8e50 --- /dev/null +++ b/jobs/JGLOBAL_PREP_EMISSIONS @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "prep_emissions" -c "base prep_emissions" + +############################################## +# Set variables used in the script +############################################## +# TODO: Set local variables used in this script e.g. GDATE may be needed for previous cycle + +############################################## +# Begin JOB SPECIFIC work +############################################## +# Generate COM variables from templates +# TODO: Add necessary COMIN, COMOUT variables for this job + +############################################################### +# Run relevant script +EXSCRIPT=${PREP_EMISSIONS_PY:-${SCRgfs}/exglobal_prep_emissions.py} +${EXSCRIPT} +status=$? +(( status != 0 )) && ( echo "FATAL ERROR: Error executing ${EXSCRIPT}, ABORT!"; exit "${status}" ) + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/prep_emissions.sh b/jobs/rocoto/prep_emissions.sh new file mode 100755 index 0000000000..0677073947 --- /dev/null +++ b/jobs/rocoto/prep_emissions.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +status=$? +(( status != 0 )) && exit "${status}" + +export job="prep_emissions" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_PREP_EMISSIONS" +status=$? +exit "${status}" diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index afd2701503..31f8aa676d 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -39,6 +39,9 @@ load(pathJoin("met", (os.getenv("met_ver") or "None"))) load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) +-- Adding perl as a module; With Rocky8, perl packages will not be from the OS +load(pathJoin("perl", (os.getenv("perl_ver") or "None"))) + setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) diff --git a/modulefiles/module_gwsetup.jet.lua b/modulefiles/module_gwsetup.jet.lua index 72c40469e4..bc14b19a79 100644 --- a/modulefiles/module_gwsetup.jet.lua +++ b/modulefiles/module_gwsetup.jet.lua @@ -4,7 +4,7 @@ Load environment to run GFS workflow setup scripts on Jet load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local python_ver=os.getenv("python_ver") or "3.11.6" diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index bc37c5abbc..90a75e3639 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -47,9 +47,9 @@ export NOSCRUB="@NOSCRUB@" export BASE_GIT="@BASE_GIT@" # Toggle to turn on/off GFS downstream processing. -export DO_BUFRSND="NO" # BUFR sounding products -export DO_GEMPAK="NO" # GEMPAK products -export DO_AWIPS="NO" # AWIPS products +export DO_BUFRSND="@DO_BUFRSND@" # BUFR sounding products +export DO_GEMPAK="@DO_GEMPAK@" # GEMPAK products +export DO_AWIPS="@DO_AWIPS@" # AWIPS products # NO for retrospective parallel; YES for real-time parallel # arch.sh uses REALTIME for MOS. Need to set REALTIME=YES diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst index 5c592556c8..f91316c7d2 100644 --- a/parm/config/gefs/config.fcst +++ b/parm/config/gefs/config.fcst @@ -6,6 +6,7 @@ echo "BEGIN: config.fcst" export USE_ESMF_THREADING="YES" # Toggle to use ESMF-managed threading or traditional threading in UFSWM +export COPY_FINAL_RESTARTS="NO" # Toggle to copy restarts from the end of GFS/GEFS Run (GDAS is handled seperately) # Turn off waves if not used for this CDUMP case ${WAVE_CDUMP} in @@ -108,18 +109,10 @@ if (( gwd_opt == 2 )); then fi # Sponge layer settings -export tau=0. -export rf_cutoff=10. export d2_bg_k1=0.20 export d2_bg_k2=0.04 export dz_min=6 export n_sponge=42 -if (( LEVS == 128 )) && [[ "${CDUMP}" =~ "gdas" ]]; then - export tau=5.0 - export rf_cutoff=1.0e3 - export d2_bg_k1=0.20 - export d2_bg_k2=0.0 -fi # PBL/turbulance schemes export hybedmf=".false." diff --git a/parm/config/gefs/config.prep_emissions b/parm/config/gefs/config.prep_emissions new file mode 100644 index 0000000000..fa411c27ad --- /dev/null +++ b/parm/config/gefs/config.prep_emissions @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.prep_emissions ########## +# aerosol emissions preprocessing specific + +echo "BEGIN: config.prep_emissions" + +# Get task specific resources +source "${EXPDIR}/config.resources" prep_emissions + +echo "END: config.prep_emissions" diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 9bf62cf514..04d55ae082 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -69,6 +69,14 @@ case ${step} in export memory_waveinit="2GB" ;; + "prep_emissions") + export wtime_prep_emissions="00:10:00" + export npe_prep_emissions=1 + export nth_prep_emissions=1 + export npe_node_prep_emissions=$(( npe_node_max / nth_prep_emissions )) + export memory_prep_emissions="1GB" + ;; + "fcst" | "efcs") export is_exclusive=True @@ -244,13 +252,19 @@ case ${step} in export memory_wavepostsbs="10GB" ;; + # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. + # Limit the number of tasks/node to 40. "wavepostbndpnt") export wtime_wavepostbndpnt="01:00:00" export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt )) - export NTASKS=${npe_wavepostbndpnt} export is_exclusive=True + if [[ ${npe_node_wavepostbndpnt} -gt 40 ]]; then + export npe_node_wavepostbndpnt=40 + export is_exclusive=False + fi + export NTASKS=${npe_wavepostbndpnt} ;; "wavepostbndpntbll") @@ -258,8 +272,12 @@ case ${step} in export npe_wavepostbndpntbll=448 export nth_wavepostbndpntbll=1 export npe_node_wavepostbndpntbll=$(( npe_node_max / nth_wavepostbndpntbll )) - export NTASKS=${npe_wavepostbndpntbll} export is_exclusive=True + if [[ ${npe_node_wavepostbndpntbll} -gt 40 ]]; then + export npe_node_wavepostbndpntbll=40 + export is_exclusive=False + fi + export NTASKS=${npe_wavepostbndpntbll} ;; "wavepostpnt") @@ -267,8 +285,12 @@ case ${step} in export npe_wavepostpnt=200 export nth_wavepostpnt=1 export npe_node_wavepostpnt=$(( npe_node_max / nth_wavepostpnt )) - export NTASKS=${npe_wavepostpnt} export is_exclusive=True + if [[ ${npe_node_wavepostpnt} -gt 40 ]]; then + export npe_node_wavepostpnt=40 + export is_exclusive=False + fi + export NTASKS=${npe_wavepostpnt} ;; *) diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index b8695b6dbb..9b42e4aa82 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -80,8 +80,14 @@ case "${fv3_res}" in export nthreads_fv3_gfs=1 export nthreads_ufs=1 export nthreads_ufs_gfs=1 - export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction + export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite + export k_split=1 + export n_split=4 + export tau=10.0 + export rf_cutoff=100.0 + export fv_sg_adj=3600 export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD export WRITE_GROUP=1 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1 @@ -98,16 +104,22 @@ case "${fv3_res}" in export nthreads_fv3_gfs=1 export nthreads_ufs=1 export nthreads_ufs_gfs=1 - export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export xr_cnvcld=".false." # Do not pass conv. clouds to Xu-Randall cloud fraction + export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD + export k_split=1 + export n_split=4 + export tau=8.0 + export rf_cutoff=100.0 + export fv_sg_adj=1800 export WRITE_GROUP=1 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1 export WRITE_GROUP_GFS=1 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1 ;; "C192") - export DELTIM=450 + export DELTIM=600 export layout_x=4 export layout_y=6 export layout_x_gfs=4 @@ -119,6 +131,11 @@ case "${fv3_res}" in export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=6.0 + export rf_cutoff=100.0 + export fv_sg_adj=1800 export WRITE_GROUP=1 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 export WRITE_GROUP_GFS=2 @@ -137,6 +154,11 @@ case "${fv3_res}" in export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=4.0 + export rf_cutoff=100.0 + export fv_sg_adj=900 export WRITE_GROUP=2 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=8 export WRITE_GROUP_GFS=2 @@ -155,13 +177,18 @@ case "${fv3_res}" in export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=3.0 + export rf_cutoff=100.0 + export fv_sg_adj=450 export WRITE_GROUP=2 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 export WRITE_GROUP_GFS=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 #Note this should be 10 for WCOSS2 ;; "C1152") - export DELTIM=120 + export DELTIM=150 export layout_x=8 export layout_y=16 export layout_x_gfs=8 @@ -173,6 +200,11 @@ case "${fv3_res}" in export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=6 + export tau=2.5 + export rf_cutoff=100.0 + export fv_sg_adj=450 export WRITE_GROUP=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 # TODO: refine these numbers when a case is available export WRITE_GROUP_GFS=4 @@ -191,6 +223,11 @@ case "${fv3_res}" in export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD + export k_split=4 + export n_split=5 + export tau=0.5 + export rf_cutoff=100.0 + export fv_sg_adj=300 export WRITE_GROUP=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 # TODO: refine these numbers when a case is available export WRITE_GROUP_GFS=4 diff --git a/parm/config/gefs/yaml/defaults.yaml b/parm/config/gefs/yaml/defaults.yaml index d252e0d1b2..5c763ad29e 100644 --- a/parm/config/gefs/yaml/defaults.yaml +++ b/parm/config/gefs/yaml/defaults.yaml @@ -4,6 +4,9 @@ base: DO_JEDIOCNVAR: "NO" DO_JEDISNOWDA: "NO" DO_MERGENSST: "NO" + DO_BUFRSND: "NO" + DO_GEMPAK: "NO" + DO_AWIPS: "NO" KEEPDATA: "NO" FHMAX_GFS: 120 USE_OCN_PERTURB_FILES: "false" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 972f393feb..24a5e92644 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -18,7 +18,7 @@ export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE="${EXECgfs}/fv3jedi_var.x" +export JEDIEXE="${EXECgfs}/gdas.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 7cfd0cb47f..dd8ca80b11 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -5,18 +5,20 @@ echo "BEGIN: config.atmanl" -export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/gdas_prototype_3d.yaml.j2" -export JEDIYAML="${PARMgfs}/gdas/atm/variational/3dvar_drpcg.yaml.j2" +export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" +export JCB_ALGO_YAML="${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" + export STATICB_TYPE="gsibec" +export LOCALIZATION_TYPE="bump" export INTERP_METHOD='barycentric' if [[ ${DOHYBVAR} = "YES" ]]; then # shellcheck disable=SC2153 export CASE_ANL=${CASE_ENS} - export BERROR_YAML="${PARMgfs}/gdas/atm/berror/hybvar_${STATICB_TYPE}.yaml.j2" + export BERROR_YAML="background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}" else export CASE_ANL=${CASE} - export BERROR_YAML="${PARMgfs}/gdas/atm/berror/staticb_${STATICB_TYPE}.yaml.j2" + export BERROR_YAML="background_error_static_${STATICB_TYPE}" fi export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" @@ -28,6 +30,6 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/fv3jedi_var.x +export JEDIEXE=${EXECgfs}/gdas.x echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmanlfv3inc b/parm/config/gfs/config.atmanlfv3inc index 14c11d3dd3..ab7efa3a60 100644 --- a/parm/config/gfs/config.atmanlfv3inc +++ b/parm/config/gfs/config.atmanlfv3inc @@ -8,7 +8,7 @@ echo "BEGIN: config.atmanlfv3inc" # Get task specific resources . "${EXPDIR}/config.resources" atmanlfv3inc -export JEDIYAML=${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc_variational.yaml.j2 +export JCB_ALGO=fv3jedi_fv3inc_variational export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x echo "END: config.atmanlfv3inc" diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index 8e824b22f6..3484cb670d 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -5,8 +5,9 @@ echo "BEGIN: config.atmensanl" -export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/lgetkf_prototype.yaml.j2" -export JEDIYAML="${PARMgfs}/gdas/atm/lgetkf/lgetkf.yaml.j2" +export JCB_BASE_YAML="${PARMgfs}/gdas/atm/jcb-base.yaml.j2" +export JCB_ALGO_YAML="${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" + export INTERP_METHOD='barycentric' export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" @@ -18,6 +19,6 @@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/fv3jedi_letkf.x +export JEDIEXE=${EXECgfs}/gdas.x echo "END: config.atmensanl" diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 58f335c801..f42af41fbf 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -65,18 +65,18 @@ export NOSCRUB="@NOSCRUB@" export BASE_GIT="@BASE_GIT@" # Toggle to turn on/off GFS downstream processing. -export DO_GOES="@DO_GOES@" # GOES products -export DO_BUFRSND="NO" # BUFR sounding products -export DO_GEMPAK="NO" # GEMPAK products -export DO_AWIPS="NO" # AWIPS products -export DO_NPOESS="NO" # NPOESS products -export DO_TRACKER="YES" # Hurricane track verification -export DO_GENESIS="YES" # Cyclone genesis verification -export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU) -export DO_VERFOZN="YES" # Ozone data assimilation monitoring -export DO_VERFRAD="YES" # Radiance data assimilation monitoring -export DO_VMINMON="YES" # GSI minimization monitoring -export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 +export DO_GOES="@DO_GOES@" # GOES products +export DO_BUFRSND="@DO_BUFRSND@" # BUFR sounding products +export DO_GEMPAK="@DO_GEMPAK@" # GEMPAK products +export DO_AWIPS="@DO_AWIPS@" # AWIPS products +export DO_NPOESS="@DO_NPOESS@" # NPOESS products +export DO_TRACKER="@DO_TRACKER@" # Hurricane track verification +export DO_GENESIS="@DO_GENESIS@" # Cyclone genesis verification +export DO_GENESIS_FSU="@DO_GENESIS_FSU@" # Cyclone genesis verification (FSU) +export DO_VERFOZN="YES" # Ozone data assimilation monitoring +export DO_VERFRAD="YES" # Radiance data assimilation monitoring +export DO_VMINMON="YES" # GSI minimization monitoring +export DO_MOS="NO" # GFS Model Output Statistics - Only supported on WCOSS2 # NO for retrospective parallel; YES for real-time parallel # arch.sh uses REALTIME for MOS. Need to set REALTIME=YES diff --git a/parm/config/gfs/config.fcst b/parm/config/gfs/config.fcst index acd99ea007..1f4ceed92c 100644 --- a/parm/config/gfs/config.fcst +++ b/parm/config/gfs/config.fcst @@ -122,18 +122,10 @@ if (( gwd_opt == 2 )); then fi # Sponge layer settings -export tau=0. -export rf_cutoff=10. export d2_bg_k1=0.20 export d2_bg_k2=0.04 export dz_min=6 export n_sponge=42 -if (( LEVS == 128 )) && [[ "${CDUMP}" =~ "gdas" ]]; then - export tau=5.0 - export rf_cutoff=1.0e3 - export d2_bg_k1=0.20 - export d2_bg_k2=0.0 -fi # PBL/turbulance schemes export hybedmf=".false." diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 9defb0c560..918ee7ece1 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -148,13 +148,19 @@ case ${step} in export memory_wavepostsbs_gfs="10GB" ;; + # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. + # Limit the number of tasks/node to 40. "wavepostbndpnt") export wtime_wavepostbndpnt="01:00:00" export npe_wavepostbndpnt=240 export nth_wavepostbndpnt=1 export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt )) - export NTASKS=${npe_wavepostbndpnt} export is_exclusive=True + if [[ ${npe_node_wavepostbndpnt} -gt 40 ]]; then + export npe_node_wavepostbndpnt=40 + export is_exclusive=False + fi + export NTASKS=${npe_wavepostbndpnt} ;; "wavepostbndpntbll") @@ -162,8 +168,12 @@ case ${step} in export npe_wavepostbndpntbll=448 export nth_wavepostbndpntbll=1 export npe_node_wavepostbndpntbll=$(( npe_node_max / nth_wavepostbndpntbll )) - export NTASKS=${npe_wavepostbndpntbll} export is_exclusive=True + if [[ ${npe_node_wavepostbndpntbll} -gt 40 ]]; then + export npe_node_wavepostbndpntbll=40 + export is_exclusive=False + fi + export NTASKS=${npe_wavepostbndpntbll} ;; "wavepostpnt") @@ -171,8 +181,12 @@ case ${step} in export npe_wavepostpnt=200 export nth_wavepostpnt=1 export npe_node_wavepostpnt=$(( npe_node_max / nth_wavepostpnt )) - export NTASKS=${npe_wavepostpnt} export is_exclusive=True + if [[ ${npe_node_wavepostpnt} -gt 40 ]]; then + export npe_node_wavepostpnt=40 + export is_exclusive=False + fi + export NTASKS=${npe_wavepostpnt} ;; "wavegempak") @@ -717,7 +731,7 @@ case ${step} in case "${CASE}" in "C48" | "C96" | "C192") - declare -x "wtime_${step}"="00:15:00" + declare -x "wtime_${step}"="00:20:00" declare -x "wtime_${step}_gfs"="03:00:00" ;; "C384") @@ -765,6 +779,7 @@ case ${step} in exit 4 ;; esac + if [[ ${machine} == "JET" ]]; then unset memory_upp ; fi export npe_node_upp=${npe_upp} export nth_upp=1 @@ -1050,7 +1065,7 @@ case ${step} in export npe_node_esfc=$(( npe_node_max / nth_esfc )) export nth_cycle=${nth_esfc} export npe_node_cycle=$(( npe_node_max / nth_cycle )) - export memory_esfc="80GB" + if [[ ${machine} != "JET" ]]; then export memory_esfc="80G" ; fi ;; "epos") diff --git a/parm/config/gfs/config.snowanl b/parm/config/gfs/config.snowanl index 7b3ffa47f3..a2984f190b 100644 --- a/parm/config/gfs/config.snowanl +++ b/parm/config/gfs/config.snowanl @@ -11,7 +11,7 @@ source "${EXPDIR}/config.resources" snowanl export OBS_LIST="${PARMgfs}/gdas/snow/obs/lists/gdas_snow.yaml.j2" # Name of the JEDI executable and its yaml template -export JEDIEXE="${EXECgfs}/fv3jedi_letkf.x" +export JEDIEXE="${EXECgfs}/gdas.x" export JEDIYAML="${PARMgfs}/gdas/snow/letkfoi/letkfoi.yaml.j2" # Ensemble member properties diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 2330997633..944cba8625 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -102,9 +102,15 @@ case "${fv3_res}" in export nthreads_fv3_gfs=1 export nthreads_ufs=1 export nthreads_ufs_gfs=1 - export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export xr_cnvcld=".false." # Do not pass conv. clouds to Xu-Randall cloud fraction + export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="40.0,1.77,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=6.0e-3 # setting for UGWPv1 non-stationary GWD + export k_split=1 + export n_split=4 + export tau=10.0 + export rf_cutoff=100.0 + export fv_sg_adj=3600 export WRITE_GROUP=1 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1 export WRITE_GROUP_GFS=1 @@ -138,9 +144,15 @@ case "${fv3_res}" in export nthreads_fv3_gfs=1 export nthreads_ufs=1 export nthreads_ufs_gfs=1 - export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling + export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction + export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="20.0,2.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=3.0e-3 # setting for UGWPv1 non-stationary GWD + export k_split=1 + export n_split=4 + export tau=8.0 + export rf_cutoff=100.0 + export fv_sg_adj=1800 export WRITE_GROUP=1 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1 export WRITE_GROUP_GFS=1 @@ -166,7 +178,7 @@ case "${fv3_res}" in export WRITE_GROUP_GFS=2 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=15 else - export DELTIM=450 + export DELTIM=600 export layout_x=4 export layout_y=6 export layout_x_gfs=4 @@ -178,6 +190,11 @@ case "${fv3_res}" in export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="10.0,3.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=1.5e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=6.0 + export rf_cutoff=100.0 + export fv_sg_adj=1800 export WRITE_GROUP=1 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 export WRITE_GROUP_GFS=2 @@ -223,6 +240,11 @@ case "${fv3_res}" in export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="5.0,5.0,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.8e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=4.0 + export rf_cutoff=100.0 + export fv_sg_adj=900 export WRITE_GROUP=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 #S2S: export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=20 @@ -266,6 +288,11 @@ case "${fv3_res}" in export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="2.5,7.5,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.5e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=4 + export tau=3.0 + export rf_cutoff=100.0 + export fv_sg_adj=450 export WRITE_GROUP=2 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 export WRITE_GROUP_GFS=4 @@ -273,7 +300,7 @@ case "${fv3_res}" in fi ;; "C1152") - export DELTIM=120 + export DELTIM=150 export layout_x=8 export layout_y=16 export layout_x_gfs=8 @@ -285,6 +312,11 @@ case "${fv3_res}" in export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="1.67,8.8,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.35e-3 # setting for UGWPv1 non-stationary GWD + export k_split=2 + export n_split=6 + export tau=2.5 + export rf_cutoff=100.0 + export fv_sg_adj=450 export WRITE_GROUP=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 # TODO: refine these numbers when a case is available export WRITE_GROUP_GFS=4 @@ -303,6 +335,11 @@ case "${fv3_res}" in export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export cdmbgwd_gsl="0.625,14.1,1.0,1.0" # settings for GSL drag suite export knob_ugwp_tauamp=0.13e-3 # setting for UGWPv1 non-stationary GWD + export k_split=4 + export n_split=5 + export tau=0.5 + export rf_cutoff=100.0 + export fv_sg_adj=300 export WRITE_GROUP=4 export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10 # TODO: refine these numbers when a case is available export WRITE_GROUP_GFS=4 diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 445fee144e..bdb5f47f04 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -6,6 +6,13 @@ base: DO_JEDISNOWDA: "NO" DO_MERGENSST: "NO" DO_GOES: "NO" + DO_BUFRSND: "NO" + DO_GEMPAK: "NO" + DO_AWIPS: "NO" + DO_NPOESS: "NO" + DO_TRACKER: "YES" + DO_GENESIS: "YES" + DO_GENESIS_FSU: "NO" FHMAX_GFS: 120 DO_VRFY_OCEANDA: "NO" GSI_SOILANAL: "NO" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 70f1319139..2b2d417a96 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 70f13191391d0909e92da47dc7d17ddf1dc4c6c6 +Subproject commit 2b2d417a96528527d7d3e7eedaccf150dc075d92 diff --git a/sorc/gsi_monitor.fd b/sorc/gsi_monitor.fd index 8efe38eade..f9d6f5f744 160000 --- a/sorc/gsi_monitor.fd +++ b/sorc/gsi_monitor.fd @@ -1 +1 @@ -Subproject commit 8efe38eadebbd5d50284aee44f6d8b6799a7f6e6 +Subproject commit f9d6f5f744462a449e70abed8c5860b1c4564ad8 diff --git a/sorc/jcb b/sorc/jcb new file mode 160000 index 0000000000..de75655d81 --- /dev/null +++ b/sorc/jcb @@ -0,0 +1 @@ +Subproject commit de75655d81ec2ee668d8d47bf4a43625c81dde7c diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index f5553e36d8..7d49d7f9e2 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -88,11 +88,17 @@ if [[ "${LINK_NEST:-OFF}" == "ON" ]] ; then source "${HOMEgfs}/versions/fix.nest.ver" fi -# Link wxflow in ush/python, workflow and ci/scripts +# Link python pacakges in ush/python +# TODO: This will be unnecessary when these are part of the virtualenv +packages=("wxflow" "jcb") +for package in "${packages[@]}"; do + cd "${HOMEgfs}/ush/python" || exit 1 + [[ -s "${package}" ]] && rm -f "${package}" + ${LINK} "${HOMEgfs}/sorc/${package}/src/${package}" . +done + +# Link wxflow in workflow and ci/scripts # TODO: This will be unnecessary when wxflow is part of the virtualenv -cd "${HOMEgfs}/ush/python" || exit 1 -[[ -s "wxflow" ]] && rm -f wxflow -${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" . cd "${HOMEgfs}/workflow" || exit 1 [[ -s "wxflow" ]] && rm -f wxflow ${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" . @@ -100,7 +106,6 @@ cd "${HOMEgfs}/ci/scripts" || exit 1 [[ -s "wxflow" ]] && rm -f wxflow ${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" . - # Link fix directories if [[ -n "${FIX_DIR}" ]]; then if [[ ! -d "${HOMEgfs}/fix" ]]; then mkdir "${HOMEgfs}/fix" || exit 1; fi @@ -241,7 +246,7 @@ fi #------------------------------ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/parm/gdas" || exit 1 - declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca") + declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca" "jcb-gdas" "jcb-algorithms") for comp in "${gdasapp_comps[@]}"; do [[ -d "${comp}" ]] && rm -rf "${comp}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/parm/${comp}" . @@ -346,33 +351,16 @@ fi # GDASApp if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then - declare -a JEDI_EXE=("fv3jedi_addincrement.x" \ - "fv3jedi_diffstates.x" \ - "fv3jedi_ensvariance.x" \ - "fv3jedi_hofx.x" \ - "fv3jedi_var.x" \ - "fv3jedi_convertincrement.x" \ - "fv3jedi_dirac.x" \ - "fv3jedi_error_covariance_training.x" \ - "fv3jedi_letkf.x" \ - "fv3jedi_convertstate.x" \ - "fv3jedi_eda.x" \ - "fv3jedi_forecast.x" \ + declare -a JEDI_EXE=("gdas.x" \ + "gdas_soca_gridgen.x" \ + "gdas_soca_error_covariance_toolbox.x" \ + "gdas_soca_setcorscales.x" \ "fv3jedi_plot_field.x" \ - "fv3jedi_data_checker.py" \ - "fv3jedi_enshofx.x" \ - "fv3jedi_hofx_nomodel.x" \ - "fv3jedi_testdata_downloader.py" \ "fv3jedi_fv3inc.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ "gdas_obsprovider2ioda.x" \ "gdas_socahybridweights.x" \ - "soca_convertincrement.x" \ - "soca_error_covariance_training.x" \ - "soca_setcorscales.x" \ - "soca_gridgen.x" \ - "soca_var.x" \ "bufr2ioda.x" \ "calcfIMS.exe" \ "apply_incr.exe" ) diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index 2ca3ca352a..a11c61a1d2 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -360,6 +360,7 @@ cat >> input.nml < None: chdir(self.task_config.DATA) exec_cmd = Executable(self.task_config.APRUN_AEROANL) - exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('variational') exec_cmd.add_default_arg(self.task_config.jedi_yaml) try: diff --git a/ush/python/pygfs/task/aero_emissions.py b/ush/python/pygfs/task/aero_emissions.py new file mode 100644 index 0000000000..17d2f528e4 --- /dev/null +++ b/ush/python/pygfs/task/aero_emissions.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import Dict, Any, Union +from pprint import pformat + +from wxflow import (AttrDict, + parse_j2yaml, + FileHandler, + Jinja, + logit, + Task, + add_to_datetime, to_timedelta, + WorkflowException, + Executable, which) + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolEmissions(Task): + """Aerosol Emissions pre-processing Task + """ + + @logit(logger, name="AerosolEmissions") + def __init__(self, config: Dict[str, Any]) -> None: + """Constructor for the Aerosol Emissions task + + Parameters + ---------- + config : Dict[str, Any] + Incoming configuration for the task from the environment + + Returns + ------- + None + """ + super().__init__(config) + + local_variable = "something" + + localdict = AttrDict( + {'variable_used_repeatedly': local_variable} + ) + self.task_config = AttrDict(**self.config, **self.runtime_config, **localdict) + + @staticmethod + @logit(logger) + def initialize() -> None: + """Initialize the work directory + """ + + @staticmethod + @logit(logger) + def configure() -> None: + """Configure the artifacts in the work directory. + Copy run specific data to run directory + """ + + @staticmethod + @logit(logger) + def execute(workdir: Union[str, os.PathLike], aprun_cmd: str) -> None: + """Run the executable (if any) + + Parameters + ---------- + workdir : str | os.PathLike + work directory with the staged data, parm files, namelists, etc. + aprun_cmd : str + launcher command for executable.x + + Returns + ------- + None + """ + + @staticmethod + @logit(logger) + def finalize() -> None: + """Perform closing actions of the task. + Copy data back from the DATA/ directory to COM/ + """ diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 02011423b7..5464c25370 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -6,8 +6,9 @@ from logging import getLogger from pprint import pformat from netCDF4 import Dataset -from typing import List, Dict, Any, Union +from typing import List, Dict, Any, Union, Optional +from jcb import render from wxflow import (parse_j2yaml, FileHandler, rm_p, logit, Task, Executable, WorkflowException, to_fv3time, to_YMD, Template, TemplateConstants) @@ -46,11 +47,14 @@ def initialize(self) -> None: self.link_jediexe() @logit(logger) - def get_jedi_config(self) -> Dict[str, Any]: + def get_jedi_config(self, algorithm: Optional[str] = None) -> Dict[str, Any]: """Compile a dictionary of JEDI configuration from JEDIYAML template file Parameters ---------- + algorithm (optional) : str + Name of the algorithm to use in the JEDI configuration. Will override the algorithm + set in the self.config.JCB_<>_YAML file Returns ---------- @@ -60,7 +64,31 @@ def get_jedi_config(self) -> Dict[str, Any]: # generate JEDI YAML file logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}") - jedi_config = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + + if 'JCB_BASE_YAML' in self.task_config.keys(): + # Step 1: fill templates of the jcb base YAML file + jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) + + # Step 2: (optional) fill templates of algorithm override YAML and merge + if 'JCB_ALGO_YAML' in self.task_config.keys(): + jcb_algo_config = parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config) + jcb_config = {**jcb_config, **jcb_algo_config} + + # If algorithm is present override the algorithm in the JEDI config + if algorithm: + jcb_config['algorithm'] = algorithm + + # Step 3: generate the JEDI Yaml using JCB driving YAML + jedi_config = render(jcb_config) + elif 'JEDIYAML' in self.task_config.keys(): + # Generate JEDI YAML file (without using JCB) + logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}") + jedi_config = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + logger.debug(f"JEDI config:\n{pformat(jedi_config)}") + else: + raise KeyError(f"Task config must contain JCB_BASE_YAML or JEDIYAML") + logger.debug(f"JEDI config:\n{pformat(jedi_config)}") return jedi_config @@ -82,7 +110,7 @@ def get_obs_dict(self) -> Dict[str, Any]: a dictionary containing the list of observation files to copy for FileHandler """ - logger.info(f"Extracting a list of observation files from {self.task_config.JEDIYAML}") + logger.info(f"Extracting a list of observation files from Jedi config file") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -116,7 +144,7 @@ def get_bias_dict(self) -> Dict[str, Any]: a dictionary containing the list of observation bias files to copy for FileHandler """ - logger.info(f"Extracting a list of bias correction files from {self.task_config.JEDIYAML}") + logger.info(f"Extracting a list of bias correction files from Jedi config file") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -292,46 +320,6 @@ def get_fv3ens_dict(config: Dict[str, Any]) -> Dict[str, Any]: } return ens_dict - @staticmethod - @logit(logger) - def execute_jediexe(workdir: Union[str, os.PathLike], aprun_cmd: str, jedi_exec: str, jedi_yaml: str) -> None: - """ - Run a JEDI executable - - Parameters - ---------- - workdir : str | os.PathLike - Working directory where to run containing the necessary files and executable - aprun_cmd : str - Launcher command e.g. mpirun -np or srun, etc. - jedi_exec : str - Name of the JEDI executable e.g. fv3jedi_var.x - jedi_yaml : str | os.PathLike - Name of the yaml file to feed the JEDI executable e.g. fv3jedi_var.yaml - - Raises - ------ - OSError - Failure due to OS issues - WorkflowException - All other exceptions - """ - - os.chdir(workdir) - - exec_cmd = Executable(aprun_cmd) - exec_cmd.add_default_arg([os.path.join(workdir, jedi_exec), jedi_yaml]) - - logger.info(f"Executing {exec_cmd}") - try: - exec_cmd() - except OSError: - logger.exception(f"FATAL ERROR: Failed to execute {exec_cmd}") - raise OSError(f"{exec_cmd}") - except Exception: - logger.exception(f"FATAL ERROR: Error occured during execution of {exec_cmd}") - raise WorkflowException(f"{exec_cmd}") - @staticmethod @logit(logger) def tgz_diags(statfile: str, diagdir: str) -> None: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index ebeb0c7ba6..95545c57a4 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -49,6 +49,9 @@ def __init__(self, config): 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, + 'atm_obsdatain_path': f"{self.runtime_config.DATA}/obs/", + 'atm_obsdataout_path': f"{self.runtime_config.DATA}/diags/", + 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) @@ -118,8 +121,10 @@ def variational(self: Analysis) -> None: chdir(self.task_config.DATA) exec_cmd = Executable(self.task_config.APRUN_ATMANLVAR) - exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('variational') exec_cmd.add_default_arg(self.task_config.jedi_yaml) try: @@ -135,8 +140,9 @@ def variational(self: Analysis) -> None: @logit(logger) def init_fv3_increment(self: Analysis) -> None: # Setup JEDI YAML file - self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, os.path.basename(self.task_config.JEDIYAML)) - save_as_yaml(self.get_jedi_config(), self.task_config.jedi_yaml) + self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, + f"{self.task_config.JCB_ALGO}.yaml") + save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml) # Link JEDI executable to run directory self.task_config.jedi_exe = self.link_jediexe() @@ -144,10 +150,17 @@ def init_fv3_increment(self: Analysis) -> None: @logit(logger) def fv3_increment(self: Analysis) -> None: # Run executable - self.execute_jediexe(self.runtime_config.DATA, - self.task_config.APRUN_ATMANLFV3INC, - self.task_config.jedi_exe, - self.task_config.jedi_yaml) + exec_cmd = Executable(self.task_config.APRUN_ATMANLFV3INC) + exec_cmd.add_default_arg(self.task_config.jedi_exe) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") @logit(logger) def finalize(self: Analysis) -> None: diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1037b557c2..5aaacc42e8 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -46,6 +46,9 @@ def __init__(self, config): 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, + 'atm_obsdatain_path': f"./obs/", + 'atm_obsdataout_path': f"./diags/", + 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) @@ -148,8 +151,10 @@ def execute(self: Analysis) -> None: chdir(self.task_config.DATA) exec_cmd = Executable(self.task_config.APRUN_ATMENSANL) - exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_letkf.x') + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('localensembleda') exec_cmd.add_default_arg(self.task_config.jedi_yaml) try: diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index c149f140b6..fe21a67536 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -310,10 +310,20 @@ def execute(self) -> None: AttrDict({key: localconf[key] for key in ['DATA', 'ntiles', 'current_cycle']})) logger.info("Running JEDI LETKF") - self.execute_jediexe(localconf.DATA, - localconf.APRUN_SNOWANL, - os.path.basename(localconf.JEDIEXE), - localconf.jedi_yaml) + exec_cmd = Executable(localconf.APRUN_SNOWANL) + exec_name = os.path.join(localconf.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('localensembleda') + exec_cmd.add_default_arg(localconf.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") logger.info("Creating analysis from backgrounds and increments") self.add_increments(localconf) diff --git a/versions/run.hera.ver b/versions/run.hera.ver index d612619acc..6280e8e115 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -4,7 +4,7 @@ export spack_env=gsi-addon-dev-rocky8 export hpss_ver=hpss export ncl_ver=6.6.2 -export R_ver=3.5.0 +export R_ver=3.6.1 export gempak_ver=7.17.0 export perl_ver=5.38.0 diff --git a/versions/run.jet.ver b/versions/run.jet.ver index d5b98bf514..3aa586ee42 100644 --- a/versions/run.jet.ver +++ b/versions/run.jet.ver @@ -1,11 +1,14 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev +export spack_env=gsi-addon-dev-rocky8 export hpss_ver= export ncl_ver=6.6.2 export R_ver=4.0.2 export gempak_ver=7.4.2 +# Adding perl as a module; With Rocky8, perl packages will not be from the OS +export perl_ver=5.38.0 + source "${HOMEgfs:-}/versions/run.spack.ver" export spack_mod_path="/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.spack.ver b/versions/run.spack.ver index 6685d748cb..5640f7f5f8 100644 --- a/versions/run.spack.ver +++ b/versions/run.spack.ver @@ -32,4 +32,4 @@ export obsproc_run_ver=1.1.2 export prepobs_run_ver=1.0.1 export ens_tracker_ver=feature-GFSv17_com_reorg -export fit2obs_ver=1.0.0 +export fit2obs_ver=1.1.1 diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index b14c1a9003..c165f9d1ca 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -27,6 +27,9 @@ def _get_app_configs(self): if self.do_ocean or self.do_ice: configs += ['oceanice_products'] + if self.do_aero: + configs += ['prep_emissions'] + return configs @staticmethod @@ -45,6 +48,9 @@ def get_task_names(self): if self.do_wave: tasks += ['waveinit'] + if self.do_aero: + tasks += ['prep_emissions'] + tasks += ['fcst'] if self.nens > 0: diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index c2a6a32f02..4d785bc4da 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -264,7 +264,7 @@ def get_task_names(self): gfs_tasks += ['gempakpgrb2spec'] if self.do_awips: - gfs_tasks += ['awips_20km_1p0deg', 'awips_g2', 'fbwind'] + gfs_tasks += ['awips_20km_1p0deg', 'fbwind'] if self.do_mos: gfs_tasks += ['mos_stn_prep', 'mos_grd_prep', 'mos_ext_stn_prep', 'mos_ext_grd_prep', diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 0a9648ee65..89881af8c9 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -124,7 +124,7 @@ def get_task_names(self): tasks += ['gempak', 'gempakmeta', 'gempakncdcupapgif', 'gempakpgrb2spec'] if self.do_awips: - tasks += ['awips_20km_1p0deg', 'awips_g2', 'fbwind'] + tasks += ['awips_20km_1p0deg', 'fbwind'] if self.do_ocean: tasks += ['ocean_prod'] diff --git a/workflow/prod.yml b/workflow/prod.yml index 64783dd611..55717772b5 100644 --- a/workflow/prod.yml +++ b/workflow/prod.yml @@ -113,17 +113,6 @@ suites: jgfs_atmos_awips_f( 3,27,6 ): edits: TRDRUN: 'NO' - awips_g2: - tasks: - jgfs_atmos_awips_g2_f( 0,64,6 ): - template: jgfs_atmos_awips_g2_master - triggers: - - task: jgfs_atmos_post_f( ) - edits: - FHRGRP: '( )' - FHRLST: 'f( )' - FCSTHR: '( )' - TRDRUN: 'YES' gempak: tasks: jgfs_atmos_gempak: diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 6ee079fdfa..86be494549 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -89,6 +89,27 @@ def waveinit(self): return task + def prep_emissions(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'stage_ic'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('prep_emissions') + task_name = 'prep_emissions' + task_dict = {'task_name': task_name, + 'resources': resources, + 'envars': self.envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/prep_emissions.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + task = rocoto.create_task(task_dict) + + return task + def fcst(self): dependencies = [] dep_dict = {'type': 'task', 'name': f'stage_ic'} @@ -98,6 +119,10 @@ def fcst(self): dep_dict = {'type': 'task', 'name': f'wave_init'} dependencies.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_aero: + dep_dict = {'type': 'task', 'name': f'prep_emissions'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') @@ -125,6 +150,10 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'wave_init'} dependencies.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_aero: + dep_dict = {'type': 'task', 'name': f'prep_emissions'} + dependencies.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) efcsenvars = self.envars.copy() diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index bba7bac3dd..6125a33dec 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1433,47 +1433,6 @@ def awips_20km_1p0deg(self): return task - def awips_g2(self): - - deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.cdump}atmos_prod'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - - awipsenvars = self.envars.copy() - awipsenvar_dict = {'FHRGRP': '#grp#', - 'FHRLST': '#lst#', - 'ROTDIR': self.rotdir} - for key, value in awipsenvar_dict.items(): - awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) - - varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = self._get_awipsgroups(self.cdump, self._configs['awips']) - var_dict = {varname1: varval1, varname2: varval2, varname3: varval3} - - resources = self.get_resource('awips') - - task_name = f'{self.cdump}awips_g2#{varname1}#' - task_dict = {'task_name': task_name, - 'resources': resources, - 'dependency': dependencies, - 'envars': awipsenvars, - 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/awips_g2.sh', - 'job_name': f'{self.pslot}_{task_name}_@H', - 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', - 'maxtries': '&MAXTRIES;' - } - - metatask_dict = {'task_name': f'{self.cdump}awips_g2', - 'task_dict': task_dict, - 'var_dict': var_dict - } - - task = rocoto.create_task(metatask_dict) - - return task - def gempak(self): deps = [] diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 3abae9b5b7..a8b4eb9fac 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -27,7 +27,7 @@ class Tasks: 'verfozn', 'verfrad', 'vminmon', 'metp', 'tracker', 'genesis', 'genesis_fsu', - 'postsnd', 'awips_g2', 'awips_20km_1p0deg', 'fbwind', + 'postsnd', 'awips_20km_1p0deg', 'fbwind', 'gempak', 'gempakmeta', 'gempakmetancdc', 'gempakncdcupapgif', 'gempakpgrb2spec', 'npoess_pgrb2_0p5deg' 'waveawipsbulls', 'waveawipsgridded', 'wavegempak', 'waveinit', 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavepostsbs', 'waveprep',