From fd771cb82ab603a64f16ba8639f3e71c8fc7440d Mon Sep 17 00:00:00 2001 From: Ed Safford <62339196+EdwardSafford-NOAA@users.noreply.github.com> Date: Wed, 12 Oct 2022 16:55:28 -0400 Subject: [PATCH 1/4] Modify default file location for monitor data when using rocoto (#1065) Added a $cyc subdirectory to the default file location for monitor data in rocoto. This ensures no file name collisions occur in the output monitor files. This change was also recently made to GFSv16.3.0 package @ 4335ef2. Additionally a problem with the definition of the previous cycle (m1) for the DA monitors was identified and corrected. This corrects the MinMon's output data. Fixes #1055 --- jobs/rocoto/vrfy.sh | 58 ++++++++++++++++++++------------------ ush/minmon_xtrct_costs.pl | 2 +- ush/minmon_xtrct_gnorms.pl | 4 +-- ush/minmon_xtrct_reduct.pl | 2 +- 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/jobs/rocoto/vrfy.sh b/jobs/rocoto/vrfy.sh index 584961e7ff..50750a94ec 100755 --- a/jobs/rocoto/vrfy.sh +++ b/jobs/rocoto/vrfy.sh @@ -45,6 +45,10 @@ export COMPONENT=${COMPONENT:-atmos} export CDATEm1=$($NDATE -24 $CDATE) export PDYm1=$(echo $CDATEm1 | cut -c1-8) +CDATEm1c=$($NDATE -06 $CDATE) +PDYm1c=$(echo $CDATEm1c | cut -c1-8) +pcyc=$(echo ${CDATEm1c} | cut -c9-10) + export pid=${pid:-$$} export jobid=${job}.${pid} export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" @@ -107,15 +111,15 @@ fi ############################################################### echo echo "=============== START TO RUN RADMON DATA EXTRACTION ===============" -if [ $VRFYRAD = "YES" -a $CDUMP = $CDFNL -a $CDATE != $SDATE ]; then +if [ $VRFYRAD = "YES" -a "${CDUMP}" = "${CDFNL}" -a "${CDATE}" != "${SDATE}" ]; then - export EXP=$PSLOT - export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" - export TANKverf_rad="$TANKverf/stats/$PSLOT/$CDUMP.$PDY" - export TANKverf_radM1="$TANKverf/stats/$PSLOT/$CDUMP.$PDYm1" - export MY_MACHINE=$machine + export EXP=${PSLOT} + export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" + export TANKverf_rad="${TANKverf}/stats/${PSLOT}/${CDUMP}.${PDY}/${cyc}" + export TANKverf_radM1="${TANKverf}/stats/${PSLOT}/${CDUMP}.${PDYm1c}/${pcyc}" + export MY_MACHINE=${machine} - $VRFYRADSH + ${VRFYRADSH} fi @@ -123,15 +127,15 @@ fi ############################################################### echo echo "=============== START TO RUN OZMON DATA EXTRACTION ===============" -if [ $VRFYOZN = "YES" -a $CDUMP = $CDFNL -a $CDATE != $SDATE ]; then +if [ "${VRFYOZN}" = "YES" -a "${CDUMP}" = "${CDFNL}" -a "${CDATE}" != "${SDATE}" ]; then - export EXP=$PSLOT - export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" - export TANKverf_ozn="$TANKverf_ozn/stats/$PSLOT/$CDUMP.$PDY" - export TANKverf_oznM1="$TANKverf_ozn/stats/$PSLOT/$CDUMP.$PDYm1" - export MY_MACHINE=$machine + export EXP=${PSLOT} + export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" + export TANKverf_ozn="${TANKverf_ozn}/stats/${PSLOT}/${CDUMP}.${PDY}/${cyc}" + export TANKverf_oznM1="${TANKverf_ozn}/stats/${PSLOT}/${CDUMP}.${PDYm1c}/${pcyc}" + export MY_MACHINE=${machine} - $VRFYOZNSH + ${VRFYOZNSH} fi @@ -139,14 +143,14 @@ fi ############################################################### echo echo "=============== START TO RUN MINMON ===============" -if [ $VRFYMINMON = "YES" -a $CDATE != $SDATE ]; then +if [ "${VRFYMINMON}" = "YES" -a "${CDATE}" != "${SDATE}" ]; then - export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT" - export M_TANKverfM0="$M_TANKverf/stats/$PSLOT/$CDUMP.$PDY" - export M_TANKverfM1="$M_TANKverf/stats/$PSLOT/$CDUMP.$PDYm1" - export MY_MACHINE=$machine + export COMOUT="${ROTDIR}/${CDUMP}.${PDY}/${cyc}/${COMPONENT}" + export M_TANKverfM0="${M_TANKverf}/stats/${PSLOT}/${CDUMP}.${PDY}/${cyc}" + export M_TANKverfM1="${M_TANKverf}/stats/${PSLOT}/${CDUMP}.${PDYm1c}/${pcyc}" + export MY_MACHINE=${machine} - $VRFYMINSH + ${VRFYMINSH} fi @@ -154,33 +158,33 @@ fi ################################################################################ echo echo "=============== START TO RUN CYCLONE TRACK VERIFICATION ===============" -if [ $VRFYTRAK = "YES" ]; then +if [ ${VRFYTRAK} = "YES" ]; then export COMINsyn=${COMINsyn:-$(compath.py ${envir}/com/gfs/${gfs_ver})/syndat} - $TRACKERSH + ${TRACKERSH} fi ################################################################################ echo echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION ===============" -if [ $VRFYGENESIS = "YES" -a $CDUMP = "gfs" ]; then - $GENESISSH +if [ ${VRFYGENESIS} = "YES" -a "${CDUMP}" = "gfs" ]; then + ${GENESISSH} fi ################################################################################ echo echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION (FSU) ===============" -if [ $VRFYFSU = "YES" -a $CDUMP = "gfs" ]; then - $GENESISFSU +if [ ${VRFYFSU} = "YES" -a "${CDUMP}" = "gfs" ]; then + ${GENESISFSU} fi ############################################################### # Force Exit out cleanly -if [ ${KEEPDATA:-"NO"} = "NO" ] ; then rm -rf $DATAROOT ; fi +if [ ${KEEPDATA:-"NO"} = "NO" ] ; then rm -rf "${DATAROOT}" ; fi exit 0 diff --git a/ush/minmon_xtrct_costs.pl b/ush/minmon_xtrct_costs.pl index 502032da80..1b5d490102 100755 --- a/ush/minmon_xtrct_costs.pl +++ b/ush/minmon_xtrct_costs.pl @@ -208,7 +208,7 @@ #-------------------------- # move files to $M_TANKverf #-------------------------- - my $tankdir = $ENV{"M_TANKverf"}; + my $tankdir = $ENV{"M_TANKverfM0"}; if(! -d $tankdir) { system( "mkdir -p $tankdir" ); } diff --git a/ush/minmon_xtrct_gnorms.pl b/ush/minmon_xtrct_gnorms.pl index 61da749a1d..ecd44232da 100755 --- a/ush/minmon_xtrct_gnorms.pl +++ b/ush/minmon_xtrct_gnorms.pl @@ -414,10 +414,8 @@ sub updateGnormData { #-------------------------- # move files to $M_TANKverf #-------------------------- - my $tankdir = $ENV{"M_TANKverf"}; - print "M_TANKverf = $tankdir \n"; + my $tankdir = $ENV{"M_TANKverfM0"}; if(! -d $tankdir) { - print "making $tankdir\n"; system( "mkdir -p $tankdir" ); } diff --git a/ush/minmon_xtrct_reduct.pl b/ush/minmon_xtrct_reduct.pl index 1b8186b6ad..f6037d3f32 100755 --- a/ush/minmon_xtrct_reduct.pl +++ b/ush/minmon_xtrct_reduct.pl @@ -72,7 +72,7 @@ #---------------------------- # copy outfile to $M_TANKverf #---------------------------- - my $tankdir = $ENV{"M_TANKverf"}; + my $tankdir = $ENV{"M_TANKverfM0"}; if(! -d $tankdir) { system( "mkdir -p $tankdir" ); } From 29cf230c3ed9bad07a0504c3ecd83bce6e7aedb2 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 13 Oct 2022 12:47:50 +0000 Subject: [PATCH 2/4] Resources updates in config.fv3 from WCOSS2 testing - add if-block for WCOSS2 to set export nth_fv3_efcs=1; nth_fv3_efcs is new and used for nth_fv3 in the efcs job - change C384 WRITE_GROUP from 1 to 2 - change C384 to run serially Refs #419 --- parm/config/config.fv3 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index 8ea139e2e8..bc22b555ea 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -86,6 +86,9 @@ case $case_in in export layout_x_gfs=4 export layout_y_gfs=6 export nth_fv3=2 + if [[ "$machine" = "WCOSS2" ]]; then + export nth_fv3_efcs=1 + fi export nth_fv3_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 @@ -105,7 +108,7 @@ case $case_in in export nth_fv3=2 export nth_fv3_gfs=${nth_fv3_gfs_s2sw:-2} export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling - export WRITE_GROUP=1 + export WRITE_GROUP=2 export WRTTASK_PER_GROUP=64 if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi export WRITE_GROUP_GFS=${WRITE_GROUP_GFS_s2sw:-2} @@ -176,11 +179,11 @@ export kchunk3d=1 # Determine whether to use parallel NetCDF based on resolution case ${case_in} in - "C48" | "C96" | "C192") + "C48" | "C96" | "C192" | "C384") export OUTPUT_FILETYPE_ATM="netcdf" export OUTPUT_FILETYPE_SFC="netcdf" ;; - "C384" | "C768" | "C1152" | "C3072") + "C768" | "C1152" | "C3072") export OUTPUT_FILETYPE_ATM="netcdf_parallel" export OUTPUT_FILETYPE_SFC="netcdf_parallel" ;; From 13ccbc9a03a27744201181dcc1d341b0d7a5de3e Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Thu, 13 Oct 2022 12:49:42 +0000 Subject: [PATCH 3/4] WCOSS2 updates from testing in config.resources - set npe_node_analdiag=$npe_analdiag - add WCOSS2 if-block to set memory_echgres="200GB" only on WCOSS2 - add npe_node_cycle back into ecen block - update nth_efcs to use nth_fv3_efcs if set; matching update in config.fv3 Refs #419 --- parm/config/config.resources | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/parm/config/config.resources b/parm/config/config.resources index 30f2aabb44..3fef4ce140 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -290,7 +290,7 @@ elif [ ${step} = "analdiag" ]; then export wtime_analdiag="00:10:00" export npe_analdiag=96 # Should be at least twice npe_ediag export nth_analdiag=1 - export npe_node_analdiag=${npe_node_max} + export npe_node_analdiag=${npe_analdiag} if [[ "${npe_node_analdiag}" -gt "${npe_node_max}" ]]; then export npe_node_analdiag=${npe_node_max} ; fi export memory_analdiag="48GB" @@ -514,7 +514,9 @@ elif [ ${step} = "echgres" ]; then export npe_echgres=3 export nth_echgres=${npe_node_max} export npe_node_echgres=1 - export memory_echgres="200GB" + if [[ "$machine" = "WCOSS2" ]]; then + export memory_echgres="200GB" + fi elif [ ${step} = "init" ]; then @@ -630,7 +632,10 @@ elif [ ${step} = "eupd" ]; then elif [ ${CASE} = "C384" ]; then export npe_eupd=270 export nth_eupd=2 - if [[ "$machine" = "HERA" ]]; then + if [[ "$machine" = "WCOSS2" ]]; then + export npe_eupd=315 + export nth_eupd=14 + elif [[ "$machine" = "HERA" ]]; then export nth_eupd=8 fi if [[ ${machine} = "S4" ]]; then @@ -656,6 +661,7 @@ elif [ ${step} = "ecen" ]; then if [[ ${CASE} = "C384" || ${CASE} = "C192" || ${CASE} = "C96" || ${CASE} = "C48" ]]; then export nth_ecen=2; fi export npe_node_ecen=$(echo "${npe_node_max} / ${nth_ecen}" | bc) export nth_cycle=${nth_ecen} + export npe_node_cycle=$(echo "${npe_node_max} / ${nth_cycle}" | bc) export is_exclusive=True elif [ ${step} = "esfc" ]; then @@ -677,7 +683,7 @@ elif [ ${step} = "efcs" ]; then export wtime_efcs="00:40:00" fi export npe_efcs=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export nth_efcs=${nth_fv3:-2} + export nth_efcs=${nth_fv3_efcs:-${nth_fv3:-2}} export npe_node_efcs=$(echo "${npe_node_max} / ${nth_efcs}" | bc) elif [ ${step} = "epos" ]; then From 1049e5084ba222c12c67bb2d18768f49f4f9c82c Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 13 Oct 2022 13:05:56 +0000 Subject: [PATCH 4/4] Updates from R&D testing - remove nth_fv3_efcs and use nth_fv3=1 for C192 everywhere - set nth_epos=6 on Hera Refs #419 --- parm/config/config.fv3 | 5 +---- parm/config/config.resources | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/parm/config/config.fv3 b/parm/config/config.fv3 index bc22b555ea..46fd18bf07 100755 --- a/parm/config/config.fv3 +++ b/parm/config/config.fv3 @@ -85,10 +85,7 @@ case $case_in in export layout_y=6 export layout_x_gfs=4 export layout_y_gfs=6 - export nth_fv3=2 - if [[ "$machine" = "WCOSS2" ]]; then - export nth_fv3_efcs=1 - fi + export nth_fv3=1 export nth_fv3_gfs=2 export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling export WRITE_GROUP=1 diff --git a/parm/config/config.resources b/parm/config/config.resources index 3fef4ce140..deb9109a9b 100755 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -683,7 +683,7 @@ elif [ ${step} = "efcs" ]; then export wtime_efcs="00:40:00" fi export npe_efcs=$(echo "${layout_x} * ${layout_y} * 6" | bc) - export nth_efcs=${nth_fv3_efcs:-${nth_fv3:-2}} + export nth_efcs=${nth_fv3:-2} export npe_node_efcs=$(echo "${npe_node_max} / ${nth_efcs}" | bc) elif [ ${step} = "epos" ]; then @@ -691,6 +691,9 @@ elif [ ${step} = "epos" ]; then export wtime_epos="00:15:00" export npe_epos=80 export nth_epos=4 + if [[ "$machine" == "HERA" ]]; then + export nth_epos=6 + fi export npe_node_epos=$(echo "${npe_node_max} / ${nth_epos}" | bc) export is_exclusive=True