From 7e648d68eeae154699281b97a1b14c62f57fe4aa Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 15 Jun 2020 15:33:30 -0600 Subject: [PATCH 1/9] Update modulefile for hera.gnu --- modulefiles/hera.gnu/fv3 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modulefiles/hera.gnu/fv3 b/modulefiles/hera.gnu/fv3 index 4e60e56f0b..ab29181dd2 100644 --- a/modulefiles/hera.gnu/fv3 +++ b/modulefiles/hera.gnu/fv3 @@ -23,13 +23,12 @@ module load sutils ## this typically includes compiler, MPI and job scheduler ## module load gnu/9.2.0 -module load openmpi/3.1.4 -module load netcdf/4.7.2 +module use -a /scratch1/BMC/gmtb/software/modulefiles/gnu-9.2.0/mpich-3.3.2 +module load mpich/3.3.2 ## -## use pre-compiled EMSF library and NCEP libraries for above compiler / MPI combination +## use pre-compiled netCDF, EMSF library and NCEP libraries for above compiler / MPI combination ## -module use -a /scratch1/BMC/gmtb/software/modulefiles/gnu-9.2.0/openmpi-3.1.4 module load NCEPlibs/1.0.0 ## From ed03ef6e0a4c1214d6fd72fbeeaca9bf07526dd1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 15 Jun 2020 14:40:29 -0600 Subject: [PATCH 2/9] build.sh: bring in update from public release branch --- build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 85e9547815..c146c00f64 100755 --- a/build.sh +++ b/build.sh @@ -3,8 +3,7 @@ set -eu MYDIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) -export COMPILER=${COMPILER:?"Please set COMPILER environment variable [gnu|intel]"} -export CMAKE_Platform=linux.${COMPILER} +export CMAKE_Platform=${CMAKE_Platform:?"Please set the CMAKE_Platform environment variable, e.g. [macosx.gnu|linux.gnu|linux.intel|hera.intel|...]"} export CMAKE_C_COMPILER=${CMAKE_C_COMPILER:-mpicc} export CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER:-mpicxx} export CMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER:-mpif90} From d09ddee5e7b4238dc261151de5161452b1ae73d5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 15 Jun 2020 14:40:11 -0600 Subject: [PATCH 3/9] cmake/configure_jet.intel.cmake: bugfix for Jet to compile CCPP with multiple SIMD instructions --- cmake/configure_jet.intel.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/configure_jet.intel.cmake b/cmake/configure_jet.intel.cmake index 94d2447f36..0c14a3f3ca 100644 --- a/cmake/configure_jet.intel.cmake +++ b/cmake/configure_jet.intel.cmake @@ -22,9 +22,11 @@ option(INLINE_POST "Enable inline post" OFF) include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) -message("AVX2 is ENABLED on Jet (multi-tagret executable)") +message("AVX2 is ENABLED on Jet (multi-target executable)") string (REPLACE "-xHOST" "-axSSE4.2,AVX,CORE-AVX2" CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}") string (REPLACE "-xHOST" "-axSSE4.2,AVX,CORE-AVX2" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") +# For CCPP +option(SIMDMULTIARCH "Enable multi-target SIMD instruction sets" ON) string(REPLACE "-i_dynamic" "-shared-intel" CMAKE_SHARED_LIBRARY_LINK_Fortran_FLAGS From edde9ff8f1598341abf99f7283404bf2d4f7b142 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 19 Jun 2020 14:03:39 -0600 Subject: [PATCH 4/9] Update .gitmodules and submodule pointer for fv3atm --- .gitmodules | 6 ++++-- FV3 | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index a6770b7e98..4ded960b48 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,9 @@ [submodule "FV3"] path = FV3 - url = https://github.com/NOAA-EMC/fv3atm - branch = develop + #url = https://github.com/NOAA-EMC/fv3atm + #branch = develop + url = https://github.com/climbfuji/fv3atm + branch = dycore_submodule_update_gnu [submodule "NEMS"] path = NEMS url = https://github.com/NOAA-EMC/NEMS diff --git a/FV3 b/FV3 index 02ce4ad481..bd1b738c0f 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 02ce4ad4812bd91356e8a975567e83a346ddb18d +Subproject commit bd1b738c0f35eeea46b9487f443e7df0d4689e12 From 803d9db5bb88a47fcf820663144bec3bce007772 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 19 Jun 2020 15:26:25 -0600 Subject: [PATCH 5/9] Regression test logs for hera.intel --- tests/Compile_hera.intel.log | 1031 ++++++++++++-------------- tests/RegressionTests_hera.intel.log | 108 +-- 2 files changed, 545 insertions(+), 594 deletions(-) diff --git a/tests/Compile_hera.intel.log b/tests/Compile_hera.intel.log index 08df541e12..ce19e95295 100644 --- a/tests/Compile_hera.intel.log +++ b/tests/Compile_hera.intel.log @@ -1,26 +1,26 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' -+ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' -+ export COMPILE_NR=1 -+ COMPILE_NR=1 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_1.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_1.env ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' ++ export COMPILE_NR=10 ++ COMPILE_NR=10 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_10.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_10.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -61,19 +61,19 @@ ++ WLCLK_dflt=15 + export TEST_NAME=compile + TEST_NAME=compile -+ export TEST_NR=1 -+ TEST_NR=1 -+ export JBNME=compile_1 -+ JBNME=compile_1 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_1 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_1 ++ export TEST_NR=10 ++ TEST_NR=10 ++ export JBNME=compile_10 ++ JBNME=compile_10 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_10 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_10 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_1 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_1 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_10 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_10 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -86,51 +86,48 @@ + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629755 -TEST 1 compile is waiting to enter the queue -1 min. TEST 1 compile is pending, Status: PD -2 min. TEST 1 compile is pending, Status: PD -3 min. TEST 1 compile is running, Status: R -4 min. TEST 1 compile is running, Status: R -5 min. TEST 1 compile is running, Status: R -6 min. TEST 1 compile is running, Status: R -7 min. TEST 1 compile is running, Status: R -8 min. TEST 1 compile is running, Status: R -9 min. TEST 1 compile is running, Status: R -10 min. TEST 1 compile is running, Status: R +Job id 8890010 +TEST 10 compile is waiting to enter the queue +1 min. TEST 10 compile is running, Status: R +2 min. TEST 10 compile is running, Status: R +3 min. TEST 10 compile is running, Status: R +4 min. TEST 10 compile is running, Status: R +5 min. TEST 10 compile is running, Status: R +6 min. TEST 10 compile is running, Status: R +7 min. TEST 10 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629755 COMPLETED compile_1 -8629755.bat+ COMPLETED batch -8629755.ext+ COMPLETED extern -11 min. TEST 1 compile is COMPLETED +8890010 COMPLETED compile_10 +8890010.bat+ COMPLETED batch +8890010.ext+ COMPLETED extern +8 min. TEST 10 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=667 -+ echo 'Elapsed time 667 seconds. Compile 1' -Elapsed time 667 seconds. Compile 1 ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 10' +Elapsed time 486 seconds. Compile 10 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' -+ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' -+ export COMPILE_NR=10 -+ COMPILE_NR=10 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_10.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_10.env ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' ++ export COMPILE_NR=11 ++ COMPILE_NR=11 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_11.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_11.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -139,10 +136,10 @@ Elapsed time 667 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -171,19 +168,19 @@ Elapsed time 667 seconds. Compile 1 ++ WLCLK_dflt=15 + export TEST_NAME=compile + TEST_NAME=compile -+ export TEST_NR=10 -+ TEST_NR=10 -+ export JBNME=compile_10 -+ JBNME=compile_10 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_10 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_10 ++ export TEST_NR=11 ++ TEST_NR=11 ++ export JBNME=compile_11 ++ JBNME=compile_11 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_11 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_11 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_10 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_10 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_11 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_11 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -196,46 +193,53 @@ Elapsed time 667 seconds. Compile 1 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629761 -TEST 10 compile is waiting to enter the queue -1 min. TEST 10 compile is pending, Status: PD -2 min. TEST 10 compile is pending, Status: PD -3 min. TEST 10 compile is pending, Status: PD -4 min. TEST 10 compile is running, Status: R -5 min. TEST 10 compile is running, Status: R +Job id 8890065 +TEST 11 compile is waiting to enter the queue +1 min. TEST 11 compile is running, Status: R +2 min. TEST 11 compile is running, Status: R +3 min. TEST 11 compile is running, Status: R +4 min. TEST 11 compile is running, Status: R +5 min. TEST 11 compile is running, Status: R +Slurm unknown status CG. Check sacct ... +8890065 COMPLETED compile_11 +8890065.bat+ COMPLETED batch +8890065.ext+ COMPLETED extern +6 min. TEST 11 compile is COMPLETED +COMPLETED +COMPLETED Slurm unknown status -. Check sacct ... -8629761 COMPLETED compile_10 -8629761.bat+ COMPLETED batch -8629761.ext+ COMPLETED extern -6 min. TEST 10 compile is COMPLETED +8890065 COMPLETED compile_11 +8890065.bat+ COMPLETED batch +8890065.ext+ COMPLETED extern +7 min. TEST 11 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=366 -+ echo 'Elapsed time 366 seconds. Compile 10' -Elapsed time 366 seconds. Compile 10 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 11' +Elapsed time 426 seconds. Compile 11 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' -+ MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' -+ export COMPILE_NR=11 -+ COMPILE_NR=11 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_11.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_11.env ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' ++ export COMPILE_NR=12 ++ COMPILE_NR=12 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_12.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_12.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -244,10 +248,10 @@ Elapsed time 366 seconds. Compile 10 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -276,19 +280,19 @@ Elapsed time 366 seconds. Compile 10 ++ WLCLK_dflt=15 + export TEST_NAME=compile + TEST_NAME=compile -+ export TEST_NR=11 -+ TEST_NR=11 -+ export JBNME=compile_11 -+ JBNME=compile_11 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_11 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_11 ++ export TEST_NR=12 ++ TEST_NR=12 ++ export JBNME=compile_12 ++ JBNME=compile_12 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_12 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_12 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_11 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_11 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_12 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_12 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -301,46 +305,47 @@ Elapsed time 366 seconds. Compile 10 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629925 -TEST 11 compile is waiting to enter the queue -1 min. TEST 11 compile is pending, Status: PD -2 min. TEST 11 compile is pending, Status: PD -3 min. TEST 11 compile is running, Status: R -4 min. TEST 11 compile is running, Status: R -5 min. TEST 11 compile is running, Status: R +Job id 8890152 +TEST 12 compile is waiting to enter the queue +1 min. TEST 12 compile is running, Status: R +2 min. TEST 12 compile is running, Status: R +3 min. TEST 12 compile is running, Status: R +4 min. TEST 12 compile is running, Status: R +5 min. TEST 12 compile is running, Status: R +6 min. TEST 12 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629925 COMPLETED compile_11 -8629925.bat+ COMPLETED batch -8629925.ext+ COMPLETED extern -6 min. TEST 11 compile is COMPLETED +8890152 COMPLETED compile_12 +8890152.bat+ COMPLETED batch +8890152.ext+ COMPLETED extern +7 min. TEST 12 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=367 -+ echo 'Elapsed time 367 seconds. Compile 11' -Elapsed time 367 seconds. Compile 11 ++ elapsed=427 ++ echo 'Elapsed time 427 seconds. Compile 12' +Elapsed time 427 seconds. Compile 12 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' -+ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' -+ export COMPILE_NR=12 -+ COMPILE_NR=12 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_12.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_12.env ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' ++ export COMPILE_NR=13 ++ COMPILE_NR=13 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_13.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_13.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -349,10 +354,10 @@ Elapsed time 367 seconds. Compile 11 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -381,19 +386,19 @@ Elapsed time 367 seconds. Compile 11 ++ WLCLK_dflt=15 + export TEST_NAME=compile + TEST_NAME=compile -+ export TEST_NR=12 -+ TEST_NR=12 -+ export JBNME=compile_12 -+ JBNME=compile_12 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_12 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_12 ++ export TEST_NR=13 ++ TEST_NR=13 ++ export JBNME=compile_13 ++ JBNME=compile_13 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_13 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_13 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_12 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_12 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_13 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_13 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -406,56 +411,47 @@ Elapsed time 367 seconds. Compile 11 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629926 -TEST 12 compile is waiting to enter the queue -1 min. TEST 12 compile is pending, Status: PD -2 min. TEST 12 compile is pending, Status: PD -3 min. TEST 12 compile is running, Status: R -4 min. TEST 12 compile is running, Status: R -5 min. TEST 12 compile is running, Status: R -6 min. TEST 12 compile is running, Status: R -7 min. TEST 12 compile is running, Status: R -8 min. TEST 12 compile is running, Status: R -Slurm unknown status CG. Check sacct ... -8629926 COMPLETED compile_12 -8629926.bat+ COMPLETED batch -8629926.ext+ COMPLETED extern -9 min. TEST 12 compile is COMPLETED -COMPLETED -COMPLETED +Job id 8890151 +TEST 13 compile is waiting to enter the queue +1 min. TEST 13 compile is running, Status: R +2 min. TEST 13 compile is running, Status: R +3 min. TEST 13 compile is running, Status: R +4 min. TEST 13 compile is running, Status: R +5 min. TEST 13 compile is running, Status: R +6 min. TEST 13 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629926 COMPLETED compile_12 -8629926.bat+ COMPLETED batch -8629926.ext+ COMPLETED extern -10 min. TEST 12 compile is COMPLETED +8890151 COMPLETED compile_13 +8890151.bat+ COMPLETED batch +8890151.ext+ COMPLETED extern +7 min. TEST 13 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=607 -+ echo 'Elapsed time 607 seconds. Compile 12' -Elapsed time 607 seconds. Compile 12 ++ elapsed=427 ++ echo 'Elapsed time 427 seconds. Compile 13' +Elapsed time 427 seconds. Compile 13 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' -+ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' -+ export COMPILE_NR=13 -+ COMPILE_NR=13 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_13.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_13.env ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' ++ export COMPILE_NR=1 ++ COMPILE_NR=1 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_1.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_1.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -464,10 +460,10 @@ Elapsed time 607 seconds. Compile 12 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -496,19 +492,19 @@ Elapsed time 607 seconds. Compile 12 ++ WLCLK_dflt=15 + export TEST_NAME=compile + TEST_NAME=compile -+ export TEST_NR=13 -+ TEST_NR=13 -+ export JBNME=compile_13 -+ JBNME=compile_13 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_13 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_13 ++ export TEST_NR=1 ++ TEST_NR=1 ++ export JBNME=compile_1 ++ JBNME=compile_1 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_1 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_13 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_13 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_1 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_1 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -521,47 +517,48 @@ Elapsed time 607 seconds. Compile 12 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8630075 -TEST 13 compile is waiting to enter the queue -1 min. TEST 13 compile is running, Status: R -2 min. TEST 13 compile is running, Status: R -3 min. TEST 13 compile is running, Status: R -4 min. TEST 13 compile is running, Status: R -5 min. TEST 13 compile is running, Status: R -6 min. TEST 13 compile is running, Status: R +Job id 8890014 +TEST 1 compile is waiting to enter the queue +1 min. TEST 1 compile is running, Status: R +2 min. TEST 1 compile is running, Status: R +3 min. TEST 1 compile is running, Status: R +4 min. TEST 1 compile is running, Status: R +5 min. TEST 1 compile is running, Status: R +6 min. TEST 1 compile is running, Status: R +7 min. TEST 1 compile is running, Status: R Slurm unknown status -. Check sacct ... -8630075 COMPLETED compile_13 -8630075.bat+ COMPLETED batch -8630075.ext+ COMPLETED extern -7 min. TEST 13 compile is COMPLETED +8890014 COMPLETED compile_1 +8890014.bat+ COMPLETED batch +8890014.ext+ COMPLETED extern +8 min. TEST 1 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 13' -Elapsed time 426 seconds. Compile 13 ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 1' +Elapsed time 486 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_2.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_2.env ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_2.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_2.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -570,10 +567,10 @@ Elapsed time 426 seconds. Compile 13 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -606,15 +603,15 @@ Elapsed time 426 seconds. Compile 13 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_2 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_2 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_2 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_2 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_2 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_2 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_2 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -627,52 +624,47 @@ Elapsed time 426 seconds. Compile 13 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629753 +Job id 8890009 TEST 2 compile is waiting to enter the queue -1 min. TEST 2 compile is pending, Status: PD -2 min. TEST 2 compile is pending, Status: PD +1 min. TEST 2 compile is running, Status: R +2 min. TEST 2 compile is running, Status: R 3 min. TEST 2 compile is running, Status: R 4 min. TEST 2 compile is running, Status: R 5 min. TEST 2 compile is running, Status: R 6 min. TEST 2 compile is running, Status: R -7 min. TEST 2 compile is running, Status: R -8 min. TEST 2 compile is running, Status: R -9 min. TEST 2 compile is running, Status: R -10 min. TEST 2 compile is running, Status: R -11 min. TEST 2 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629753 COMPLETED compile_2 -8629753.bat+ COMPLETED batch -8629753.ext+ COMPLETED extern -12 min. TEST 2 compile is COMPLETED +8890009 COMPLETED compile_2 +8890009.bat+ COMPLETED batch +8890009.ext+ COMPLETED extern +7 min. TEST 2 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=727 -+ echo 'Elapsed time 727 seconds. Compile 2' -Elapsed time 727 seconds. Compile 2 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 2' +Elapsed time 426 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_3.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_3.env ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_3.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_3.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -681,10 +673,10 @@ Elapsed time 727 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -717,15 +709,15 @@ Elapsed time 727 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_3 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_3 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_3 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_3 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_3 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_3 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_3 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -738,50 +730,47 @@ Elapsed time 727 seconds. Compile 2 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629754 +Job id 8890016 TEST 3 compile is waiting to enter the queue -1 min. TEST 3 compile is pending, Status: PD -2 min. TEST 3 compile is pending, Status: PD +1 min. TEST 3 compile is running, Status: R +2 min. TEST 3 compile is running, Status: R 3 min. TEST 3 compile is running, Status: R 4 min. TEST 3 compile is running, Status: R 5 min. TEST 3 compile is running, Status: R 6 min. TEST 3 compile is running, Status: R -7 min. TEST 3 compile is running, Status: R -8 min. TEST 3 compile is running, Status: R -9 min. TEST 3 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629754 COMPLETED compile_3 -8629754.bat+ COMPLETED batch -8629754.ext+ COMPLETED extern -10 min. TEST 3 compile is COMPLETED +8890016 COMPLETED compile_3 +8890016.bat+ COMPLETED batch +8890016.ext+ COMPLETED extern +7 min. TEST 3 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=607 -+ echo 'Elapsed time 607 seconds. Compile 3' -Elapsed time 607 seconds. Compile 3 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 3' +Elapsed time 426 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_4.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_4.env ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_4.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_4.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -790,10 +779,10 @@ Elapsed time 607 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -826,15 +815,15 @@ Elapsed time 607 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_4 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_4 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_4 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_4 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_4 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_4 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_4 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -847,50 +836,47 @@ Elapsed time 607 seconds. Compile 3 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629758 +Job id 8890012 TEST 4 compile is waiting to enter the queue -1 min. TEST 4 compile is pending, Status: PD -2 min. TEST 4 compile is pending, Status: PD +1 min. TEST 4 compile is running, Status: R +2 min. TEST 4 compile is running, Status: R 3 min. TEST 4 compile is running, Status: R 4 min. TEST 4 compile is running, Status: R 5 min. TEST 4 compile is running, Status: R 6 min. TEST 4 compile is running, Status: R -7 min. TEST 4 compile is running, Status: R -8 min. TEST 4 compile is running, Status: R -9 min. TEST 4 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629758 COMPLETED compile_4 -8629758.bat+ COMPLETED batch -8629758.ext+ COMPLETED extern -10 min. TEST 4 compile is COMPLETED +8890012 COMPLETED compile_4 +8890012.bat+ COMPLETED batch +8890012.ext+ COMPLETED extern +7 min. TEST 4 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=607 -+ echo 'Elapsed time 607 seconds. Compile 4' -Elapsed time 607 seconds. Compile 4 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 4' +Elapsed time 426 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_5.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_5.env ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_5.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_5.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -899,10 +885,10 @@ Elapsed time 607 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -935,15 +921,15 @@ Elapsed time 607 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_5 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_5 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_5 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_5 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_5 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_5 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_5 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -956,52 +942,43 @@ Elapsed time 607 seconds. Compile 4 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629757 +Job id 8890017 TEST 5 compile is waiting to enter the queue -1 min. TEST 5 compile is pending, Status: PD -2 min. TEST 5 compile is pending, Status: PD -3 min. TEST 5 compile is running, Status: R -4 min. TEST 5 compile is running, Status: R -Slurm unknown status CG. Check sacct ... -8629757 COMPLETED compile_5 -8629757.bat+ COMPLETED batch -8629757.ext+ COMPLETED extern -5 min. TEST 5 compile is COMPLETED -COMPLETED -COMPLETED +1 min. TEST 5 compile is running, Status: R +2 min. TEST 5 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629757 COMPLETED compile_5 -8629757.bat+ COMPLETED batch -8629757.ext+ COMPLETED extern -6 min. TEST 5 compile is COMPLETED +8890017 COMPLETED compile_5 +8890017.bat+ COMPLETED batch +8890017.ext+ COMPLETED extern +3 min. TEST 5 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=366 -+ echo 'Elapsed time 366 seconds. Compile 5' -Elapsed time 366 seconds. Compile 5 ++ elapsed=186 ++ echo 'Elapsed time 186 seconds. Compile 5' +Elapsed time 186 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_6.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_6.env ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_6.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_6.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -1010,10 +987,10 @@ Elapsed time 366 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1046,15 +1023,15 @@ Elapsed time 366 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_6 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_6 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_6 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_6 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_6 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_6 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_6 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1067,57 +1044,47 @@ Elapsed time 366 seconds. Compile 5 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629752 +Job id 8890015 TEST 6 compile is waiting to enter the queue -1 min. TEST 6 compile is pending, Status: PD -2 min. TEST 6 compile is pending, Status: PD +1 min. TEST 6 compile is running, Status: R +2 min. TEST 6 compile is running, Status: R 3 min. TEST 6 compile is running, Status: R 4 min. TEST 6 compile is running, Status: R 5 min. TEST 6 compile is running, Status: R 6 min. TEST 6 compile is running, Status: R -7 min. TEST 6 compile is running, Status: R -8 min. TEST 6 compile is running, Status: R -9 min. TEST 6 compile is running, Status: R -Slurm unknown status CG. Check sacct ... -8629752 COMPLETED compile_6 -8629752.bat+ COMPLETED batch -8629752.ext+ COMPLETED extern -10 min. TEST 6 compile is COMPLETED -COMPLETED -COMPLETED Slurm unknown status -. Check sacct ... -8629752 COMPLETED compile_6 -8629752.bat+ COMPLETED batch -8629752.ext+ COMPLETED extern -11 min. TEST 6 compile is COMPLETED +8890015 COMPLETED compile_6 +8890015.bat+ COMPLETED batch +8890015.ext+ COMPLETED extern +7 min. TEST 6 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=667 -+ echo 'Elapsed time 667 seconds. Compile 6' -Elapsed time 667 seconds. Compile 6 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 6' +Elapsed time 426 seconds. Compile 6 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + export COMPILE_NR=7 + COMPILE_NR=7 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_7.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_7.env ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_7.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_7.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -1126,10 +1093,10 @@ Elapsed time 667 seconds. Compile 6 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1162,15 +1129,15 @@ Elapsed time 667 seconds. Compile 6 + TEST_NR=7 + export JBNME=compile_7 + JBNME=compile_7 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_7 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_7 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_7 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_7 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_7 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_7 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_7 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_7 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1183,50 +1150,47 @@ Elapsed time 667 seconds. Compile 6 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629756 +Job id 8890013 TEST 7 compile is waiting to enter the queue -1 min. TEST 7 compile is pending, Status: PD -2 min. TEST 7 compile is pending, Status: PD +1 min. TEST 7 compile is running, Status: R +2 min. TEST 7 compile is running, Status: R 3 min. TEST 7 compile is running, Status: R 4 min. TEST 7 compile is running, Status: R 5 min. TEST 7 compile is running, Status: R 6 min. TEST 7 compile is running, Status: R -7 min. TEST 7 compile is running, Status: R -8 min. TEST 7 compile is running, Status: R -9 min. TEST 7 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629756 COMPLETED compile_7 -8629756.bat+ COMPLETED batch -8629756.ext+ COMPLETED extern -10 min. TEST 7 compile is COMPLETED +8890013 COMPLETED compile_7 +8890013.bat+ COMPLETED batch +8890013.ext+ COMPLETED extern +7 min. TEST 7 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=607 -+ echo 'Elapsed time 607 seconds. Compile 7' -Elapsed time 607 seconds. Compile 7 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 7' +Elapsed time 426 seconds. Compile 7 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + export COMPILE_NR=8 + COMPILE_NR=8 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_8.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_8.env ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_8.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_8.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -1235,10 +1199,10 @@ Elapsed time 607 seconds. Compile 7 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1271,15 +1235,15 @@ Elapsed time 607 seconds. Compile 7 + TEST_NR=8 + export JBNME=compile_8 + JBNME=compile_8 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_8 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_8 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_8 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_8 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_8 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_8 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_8 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_8 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1292,51 +1256,48 @@ Elapsed time 607 seconds. Compile 7 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629759 +Job id 8890018 TEST 8 compile is waiting to enter the queue -1 min. TEST 8 compile is pending, Status: PD -2 min. TEST 8 compile is pending, Status: PD -3 min. TEST 8 compile is pending, Status: PD +1 min. TEST 8 compile is running, Status: R +2 min. TEST 8 compile is running, Status: R +3 min. TEST 8 compile is running, Status: R 4 min. TEST 8 compile is running, Status: R 5 min. TEST 8 compile is running, Status: R 6 min. TEST 8 compile is running, Status: R 7 min. TEST 8 compile is running, Status: R -8 min. TEST 8 compile is running, Status: R -9 min. TEST 8 compile is running, Status: R -10 min. TEST 8 compile is running, Status: R Slurm unknown status -. Check sacct ... -8629759 COMPLETED compile_8 -8629759.bat+ COMPLETED batch -8629759.ext+ COMPLETED extern -11 min. TEST 8 compile is COMPLETED +8890018 COMPLETED compile_8 +8890018.bat+ COMPLETED batch +8890018.ext+ COMPLETED extern +8 min. TEST 8 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=667 -+ echo 'Elapsed time 667 seconds. Compile 8' -Elapsed time 667 seconds. Compile 8 ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 8' +Elapsed time 486 seconds. Compile 8 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 -+ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782 ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + export COMPILE_NR=9 + COMPILE_NR=9 -+ cd /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -+ [[ -e /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_9.env ]] -+ source /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_9.env ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_9.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_9.env ++ export MACHINE_ID=hera.intel ++ MACHINE_ID=hera.intel -++ export PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ PATHRT=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests -++ export PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model -++ PATHTR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION= @@ -1345,10 +1306,10 @@ Elapsed time 667 seconds. Compile 8 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ REGRESSIONTEST_LOG=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/RegressionTests_hera.intel.log -++ export LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel -++ LOG_DIR=/scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/log_hera.intel +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/RegressionTests_hera.intel.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/log_hera.intel + source default_vars.sh ++ '[' hera.intel = wcoss ']' ++ '[' hera.intel = wcoss_cray ']' @@ -1381,15 +1342,15 @@ Elapsed time 667 seconds. Compile 8 + TEST_NR=9 + export JBNME=compile_9 + JBNME=compile_9 -+ export RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_9 -+ RUNDIR=/scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_9 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_9 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_9 + source rt_utils.sh ++ set -eu -++ [[ /scratch1/NCEPDEV/nems/Jun.Wang/nems/vlab/20200610/commit/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/intel/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_9 -+ cd /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/compile_9 ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_9 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/compile_9 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1402,31 +1363,21 @@ Elapsed time 667 seconds. Compile 8 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 8629760 +Job id 8890011 TEST 9 compile is waiting to enter the queue -1 min. TEST 9 compile is pending, Status: PD -2 min. TEST 9 compile is pending, Status: PD -3 min. TEST 9 compile is pending, Status: PD +1 min. TEST 9 compile is running, Status: R +2 min. TEST 9 compile is running, Status: R +3 min. TEST 9 compile is running, Status: R 4 min. TEST 9 compile is running, Status: R 5 min. TEST 9 compile is running, Status: R 6 min. TEST 9 compile is running, Status: R -7 min. TEST 9 compile is running, Status: R -8 min. TEST 9 compile is running, Status: R -9 min. TEST 9 compile is running, Status: R -Slurm unknown status CG. Check sacct ... -8629760 COMPLETED compile_9 -8629760.bat+ COMPLETED batch -8629760.ext+ COMPLETED extern -10 min. TEST 9 compile is COMPLETED -COMPLETED -COMPLETED Slurm unknown status -. Check sacct ... -8629760 COMPLETED compile_9 -8629760.bat+ COMPLETED batch -8629760.ext+ COMPLETED extern -11 min. TEST 9 compile is COMPLETED +8890011 COMPLETED compile_9 +8890011.bat+ COMPLETED batch +8890011.ext+ COMPLETED extern +7 min. TEST 9 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=667 -+ echo 'Elapsed time 667 seconds. Compile 9' -Elapsed time 667 seconds. Compile 9 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 9' +Elapsed time 426 seconds. Compile 9 diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log index 5bead8b996..4300f64eaf 100644 --- a/tests/RegressionTests_hera.intel.log +++ b/tests/RegressionTests_hera.intel.log @@ -1,9 +1,9 @@ -Fri Jun 12 01:05:59 UTC 2020 +Fri Jun 19 20:12:53 UTC 2020 Start Regression test baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_control_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_decomp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_2threads_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_restart_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_restart_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_read_inc_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_read_inc_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_netcdf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGlatlon_netcdf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_nemsio_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_stochy_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_stochy_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_iau_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_iau_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_ca_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_ca_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_ca_prod Checking test 013 fv3_ccpp_ca results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -787,7 +787,7 @@ Test 013 fv3_ccpp_ca PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_lheatstrg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_lheatstrg_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_lheatstrg_prod Checking test 014 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -835,7 +835,7 @@ Test 014 fv3_ccpp_lheatstrg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfdlmprad_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfdlmprad_prod Checking test 015 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -884,7 +884,7 @@ Test 015 fv3_ccpp_gfdlmprad PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_atmwav_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 016 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -933,7 +933,7 @@ Test 016 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_nemsio_c768_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_wrtGauss_nemsio_c768_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_wrtGauss_nemsio_c768_prod Checking test 017 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -982,7 +982,7 @@ Test 017 fv3_ccpp_wrtGauss_nemsio_c768 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_32bit_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_control_32bit_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_control_32bit_prod Checking test 018 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1050,7 +1050,7 @@ Test 018 fv3_ccpp_control_32bit PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_stretched_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_stretched_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_stretched_prod Checking test 019 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1106,7 +1106,7 @@ Test 019 fv3_ccpp_stretched PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_stretched_nest_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_stretched_nest_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_stretched_nest_prod Checking test 020 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1173,7 +1173,7 @@ Test 020 fv3_ccpp_stretched_nest PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_regional_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_regional_control_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_regional_control_prod Checking test 021 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1184,7 +1184,7 @@ Test 021 fv3_ccpp_regional_control PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_regional_restart_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_regional_restart_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_regional_restart_prod Checking test 022 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1193,7 +1193,7 @@ Test 022 fv3_ccpp_regional_restart PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_regional_quilt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_regional_quilt_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_regional_quilt_prod Checking test 023 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1204,7 +1204,7 @@ Test 023 fv3_ccpp_regional_quilt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_regional_c768_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_regional_c768_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_regional_c768_prod Checking test 024 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1215,19 +1215,19 @@ Test 024 fv3_ccpp_regional_c768 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_control_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_control_debug_prod Checking test 025 fv3_ccpp_control_debug results .... Test 025 fv3_ccpp_control_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_stretched_nest_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_stretched_nest_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_stretched_nest_debug_prod Checking test 026 fv3_ccpp_stretched_nest_debug results .... Test 026 fv3_ccpp_stretched_nest_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfdlmp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfdlmp_prod Checking test 027 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1275,7 +1275,7 @@ Test 027 fv3_ccpp_gfdlmp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_gwd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfdlmprad_gwd_prod Checking test 028 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1323,7 +1323,7 @@ Test 028 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_noahmp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 029 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1371,7 +1371,7 @@ Test 029 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmp_hwrfsas_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfdlmp_hwrfsas_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 030 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1419,7 +1419,7 @@ Test 030 fv3_ccpp_gfdlmp_hwrfsas PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_csawmg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_csawmg_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_csawmg_prod Checking test 031 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1467,7 +1467,7 @@ Test 031 fv3_ccpp_csawmg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_satmedmf_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_satmedmf_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_satmedmf_prod Checking test 032 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1515,7 +1515,7 @@ Test 032 fv3_ccpp_satmedmf PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_satmedmfq_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_satmedmfq_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_satmedmfq_prod Checking test 033 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1563,7 +1563,7 @@ Test 033 fv3_ccpp_satmedmfq PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmp_32bit_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfdlmp_32bit_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfdlmp_32bit_prod Checking test 034 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1611,7 +1611,7 @@ Test 034 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_32bit_post_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 035 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1663,7 +1663,7 @@ Test 035 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_cpt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_cpt_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_cpt_prod Checking test 036 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1717,7 +1717,7 @@ Test 036 fv3_ccpp_cpt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gsd_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gsd_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gsd_prod Checking test 037 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1809,7 +1809,7 @@ Test 037 fv3_ccpp_gsd PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_thompson_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_thompson_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_thompson_prod Checking test 038 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1877,7 +1877,7 @@ Test 038 fv3_ccpp_thompson PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_thompson_no_aero_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_thompson_no_aero_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_thompson_no_aero_prod Checking test 039 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1945,7 +1945,7 @@ Test 039 fv3_ccpp_thompson_no_aero PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfs_v15p2_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfs_v15p2_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfs_v15p2_prod Checking test 040 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2013,7 +2013,7 @@ Test 040 fv3_ccpp_gfs_v15p2 PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfs_v16beta_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfs_v16beta_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfs_v16beta_prod Checking test 041 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2081,7 +2081,7 @@ Test 041 fv3_ccpp_gfs_v16beta PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_rrtmgp_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_rrtmgp_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_rrtmgp_prod Checking test 042 fv3_ccpp_rrtmgp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2149,7 +2149,7 @@ Test 042 fv3_ccpp_rrtmgp PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfs_v15p2_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfs_v15p2_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfs_v15p2_debug_prod Checking test 043 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2217,7 +2217,7 @@ Test 043 fv3_ccpp_gfs_v15p2_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfs_v16beta_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfs_v16beta_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfs_v16beta_debug_prod Checking test 044 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2285,7 +2285,7 @@ Test 044 fv3_ccpp_gfs_v16beta_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_rrtmgp_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_rrtmgp_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_rrtmgp_debug_prod Checking test 045 fv3_ccpp_rrtmgp_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2353,7 +2353,7 @@ Test 045 fv3_ccpp_rrtmgp_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gsd_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gsd_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gsd_debug_prod Checking test 046 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2421,7 +2421,7 @@ Test 046 fv3_ccpp_gsd_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_thompson_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_thompson_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_thompson_debug_prod Checking test 047 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2489,7 +2489,7 @@ Test 047 fv3_ccpp_thompson_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_thompson_no_aero_debug_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_thompson_no_aero_debug_prod Checking test 048 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2557,7 +2557,7 @@ Test 048 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfsv16_csawmg_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfsv16_csawmg_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfsv16_csawmg_prod Checking test 049 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2605,7 +2605,7 @@ Test 049 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfsv16_csawmgt_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gfsv16_csawmgt_prod Checking test 050 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2653,7 +2653,7 @@ Test 050 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gocart_clm_ccpp -working dir = /scratch1/NCEPDEV/stmp2/Jun.Wang/FV3_RT/rt_140782/fv3_ccpp_gocart_clm_prod +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_273349/fv3_ccpp_gocart_clm_prod Checking test 051 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2701,5 +2701,5 @@ Test 051 fv3_ccpp_gocart_clm PASS REGRESSION TEST WAS SUCCESSFUL -Fri Jun 12 01:42:57 UTC 2020 -Elapsed time: 00h:37m:02s. Have a nice day! +Fri Jun 19 20:45:04 UTC 2020 +Elapsed time: 00h:32m:12s. Have a nice day! From 99bca7e4814960fd1d39a1281d626830cc52ddd8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 19 Jun 2020 16:01:25 -0600 Subject: [PATCH 6/9] Regression test logs for hera.gnu --- tests/Compile_hera.gnu.log | 528 ++++++++++++++++++++++ tests/RegressionTests_hera.gnu.log | 697 +++++++++++++++++++++++++++++ 2 files changed, 1225 insertions(+) create mode 100644 tests/Compile_hera.gnu.log create mode 100644 tests/RegressionTests_hera.gnu.log diff --git a/tests/Compile_hera.gnu.log b/tests/Compile_hera.gnu.log new file mode 100644 index 0000000000..1828226d3a --- /dev/null +++ b/tests/Compile_hera.gnu.log @@ -0,0 +1,528 @@ ++ SECONDS=0 ++ [[ 4 != 4 ]] ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp' ++ export COMPILE_NR=1 ++ COMPILE_NR=1 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_1.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_1.env +++ export MACHINE_ID=hera.gnu +++ MACHINE_ID=hera.gnu +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ export SCHEDULER=slurm +++ SCHEDULER=slurm +++ export ACCNR=gmtb +++ ACCNR=gmtb +++ export QUEUE=batch +++ QUEUE=batch +++ export PARTITION= +++ PARTITION= +++ export ROCOTO=false +++ ROCOTO=false +++ export ECFLOW=true +++ ECFLOW=true +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu ++ source default_vars.sh +++ '[' hera.gnu = wcoss ']' +++ '[' hera.gnu = wcoss_cray ']' +++ '[' hera.gnu = wcoss_dell_p3 ']' +++ [[ hera.gnu = orion.* ]] +++ [[ hera.gnu = hera.* ]] +++ TASKS_dflt=150 +++ TPN_dflt=40 +++ INPES_dflt=3 +++ JNPES_dflt=8 +++ TASKS_thrd=84 +++ TPN_thrd=20 +++ INPES_thrd=3 +++ JNPES_thrd=4 +++ TASKS_stretch=48 +++ TPN_stretch=12 +++ INPES_stretch=2 +++ JNPES_stretch=4 +++ TASKS_strnest=96 +++ TPN_strnest=12 +++ INPES_strnest=2 +++ JNPES_strnest=4 +++ COMPILER=gnu +++ [[ gnu = gnu ]] +++ WLCLK_dflt=30 ++ export TEST_NAME=compile ++ TEST_NAME=compile ++ export TEST_NR=1 ++ TEST_NR=1 ++ export JBNME=compile_1 ++ JBNME=compile_1 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_1 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_1 ++ source rt_utils.sh +++ set -eu +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ UNIT_TEST=false ++ source atparse.bash ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_1 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_1 ++ [[ slurm = \s\l\u\r\m ]] ++ atparse ++ local __set_x ++ '[' -o xtrace ']' ++ __set_x='set -x' ++ set +x ++ [[ false = \f\a\l\s\e ]] ++ submit_and_wait job_card ++ [[ -z job_card ]] ++ '[' -o xtrace ']' ++ set_x='set -x' ++ set +x +Job id 8891753 +TEST 1 compile is waiting to enter the queue +1 min. TEST 1 compile is running, Status: R +2 min. TEST 1 compile is running, Status: R +Slurm unknown status -. Check sacct ... +8891753 COMPLETED compile_1 +8891753.bat+ COMPLETED batch +8891753.ext+ COMPLETED extern +3 min. TEST 1 compile is COMPLETED +COMPLETED +COMPLETED ++ elapsed=185 ++ echo 'Elapsed time 185 seconds. Compile 1' +Elapsed time 185 seconds. Compile 1 ++ SECONDS=0 ++ [[ 4 != 4 ]] ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' ++ export COMPILE_NR=2 ++ COMPILE_NR=2 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_2.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_2.env +++ export MACHINE_ID=hera.gnu +++ MACHINE_ID=hera.gnu +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ export SCHEDULER=slurm +++ SCHEDULER=slurm +++ export ACCNR=gmtb +++ ACCNR=gmtb +++ export QUEUE=batch +++ QUEUE=batch +++ export PARTITION= +++ PARTITION= +++ export ROCOTO=false +++ ROCOTO=false +++ export ECFLOW=true +++ ECFLOW=true +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu ++ source default_vars.sh +++ '[' hera.gnu = wcoss ']' +++ '[' hera.gnu = wcoss_cray ']' +++ '[' hera.gnu = wcoss_dell_p3 ']' +++ [[ hera.gnu = orion.* ]] +++ [[ hera.gnu = hera.* ]] +++ TASKS_dflt=150 +++ TPN_dflt=40 +++ INPES_dflt=3 +++ JNPES_dflt=8 +++ TASKS_thrd=84 +++ TPN_thrd=20 +++ INPES_thrd=3 +++ JNPES_thrd=4 +++ TASKS_stretch=48 +++ TPN_stretch=12 +++ INPES_stretch=2 +++ JNPES_stretch=4 +++ TASKS_strnest=96 +++ TPN_strnest=12 +++ INPES_strnest=2 +++ JNPES_strnest=4 +++ COMPILER=gnu +++ [[ gnu = gnu ]] +++ WLCLK_dflt=30 ++ export TEST_NAME=compile ++ TEST_NAME=compile ++ export TEST_NR=2 ++ TEST_NR=2 ++ export JBNME=compile_2 ++ JBNME=compile_2 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_2 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_2 ++ source rt_utils.sh +++ set -eu +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ UNIT_TEST=false ++ source atparse.bash ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_2 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_2 ++ [[ slurm = \s\l\u\r\m ]] ++ atparse ++ local __set_x ++ '[' -o xtrace ']' ++ __set_x='set -x' ++ set +x ++ [[ false = \f\a\l\s\e ]] ++ submit_and_wait job_card ++ [[ -z job_card ]] ++ '[' -o xtrace ']' ++ set_x='set -x' ++ set +x +Job id 8891757 +TEST 2 compile is waiting to enter the queue +1 min. TEST 2 compile is pending, Status: PD +2 min. TEST 2 compile is pending, Status: PD +3 min. TEST 2 compile is pending, Status: PD +4 min. TEST 2 compile is pending, Status: PD +5 min. TEST 2 compile is pending, Status: PD +6 min. TEST 2 compile is running, Status: R +7 min. TEST 2 compile is running, Status: R +Slurm unknown status -. Check sacct ... +8891757 COMPLETED compile_2 +8891757.bat+ COMPLETED batch +8891757.ext+ COMPLETED extern +8 min. TEST 2 compile is COMPLETED +COMPLETED +COMPLETED ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 2' +Elapsed time 486 seconds. Compile 2 ++ SECONDS=0 ++ [[ 4 != 4 ]] ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' ++ export COMPILE_NR=3 ++ COMPILE_NR=3 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_3.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_3.env +++ export MACHINE_ID=hera.gnu +++ MACHINE_ID=hera.gnu +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ export SCHEDULER=slurm +++ SCHEDULER=slurm +++ export ACCNR=gmtb +++ ACCNR=gmtb +++ export QUEUE=batch +++ QUEUE=batch +++ export PARTITION= +++ PARTITION= +++ export ROCOTO=false +++ ROCOTO=false +++ export ECFLOW=true +++ ECFLOW=true +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu ++ source default_vars.sh +++ '[' hera.gnu = wcoss ']' +++ '[' hera.gnu = wcoss_cray ']' +++ '[' hera.gnu = wcoss_dell_p3 ']' +++ [[ hera.gnu = orion.* ]] +++ [[ hera.gnu = hera.* ]] +++ TASKS_dflt=150 +++ TPN_dflt=40 +++ INPES_dflt=3 +++ JNPES_dflt=8 +++ TASKS_thrd=84 +++ TPN_thrd=20 +++ INPES_thrd=3 +++ JNPES_thrd=4 +++ TASKS_stretch=48 +++ TPN_stretch=12 +++ INPES_stretch=2 +++ JNPES_stretch=4 +++ TASKS_strnest=96 +++ TPN_strnest=12 +++ INPES_strnest=2 +++ JNPES_strnest=4 +++ COMPILER=gnu +++ [[ gnu = gnu ]] +++ WLCLK_dflt=30 ++ export TEST_NAME=compile ++ TEST_NAME=compile ++ export TEST_NR=3 ++ TEST_NR=3 ++ export JBNME=compile_3 ++ JBNME=compile_3 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_3 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_3 ++ source rt_utils.sh +++ set -eu +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ UNIT_TEST=false ++ source atparse.bash ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_3 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_3 ++ [[ slurm = \s\l\u\r\m ]] ++ atparse ++ local __set_x ++ '[' -o xtrace ']' ++ __set_x='set -x' ++ set +x ++ [[ false = \f\a\l\s\e ]] ++ submit_and_wait job_card ++ [[ -z job_card ]] ++ '[' -o xtrace ']' ++ set_x='set -x' ++ set +x +Job id 8891756 +TEST 3 compile is waiting to enter the queue +1 min. TEST 3 compile is pending, Status: PD +2 min. TEST 3 compile is pending, Status: PD +3 min. TEST 3 compile is pending, Status: PD +4 min. TEST 3 compile is pending, Status: PD +5 min. TEST 3 compile is pending, Status: PD +6 min. TEST 3 compile is running, Status: R +7 min. TEST 3 compile is running, Status: R +8 min. TEST 3 compile is running, Status: R +Slurm unknown status -. Check sacct ... +8891756 COMPLETED compile_3 +8891756.bat+ COMPLETED batch +8891756.ext+ COMPLETED extern +9 min. TEST 3 compile is COMPLETED +COMPLETED +COMPLETED ++ elapsed=546 ++ echo 'Elapsed time 546 seconds. Compile 3' +Elapsed time 546 seconds. Compile 3 ++ SECONDS=0 ++ [[ 4 != 4 ]] ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017 32BIT=Y DEBUG=Y' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017 32BIT=Y DEBUG=Y' ++ export COMPILE_NR=4 ++ COMPILE_NR=4 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_4.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_4.env +++ export MACHINE_ID=hera.gnu +++ MACHINE_ID=hera.gnu +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ export SCHEDULER=slurm +++ SCHEDULER=slurm +++ export ACCNR=gmtb +++ ACCNR=gmtb +++ export QUEUE=batch +++ QUEUE=batch +++ export PARTITION= +++ PARTITION= +++ export ROCOTO=false +++ ROCOTO=false +++ export ECFLOW=true +++ ECFLOW=true +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu ++ source default_vars.sh +++ '[' hera.gnu = wcoss ']' +++ '[' hera.gnu = wcoss_cray ']' +++ '[' hera.gnu = wcoss_dell_p3 ']' +++ [[ hera.gnu = orion.* ]] +++ [[ hera.gnu = hera.* ]] +++ TASKS_dflt=150 +++ TPN_dflt=40 +++ INPES_dflt=3 +++ JNPES_dflt=8 +++ TASKS_thrd=84 +++ TPN_thrd=20 +++ INPES_thrd=3 +++ JNPES_thrd=4 +++ TASKS_stretch=48 +++ TPN_stretch=12 +++ INPES_stretch=2 +++ JNPES_stretch=4 +++ TASKS_strnest=96 +++ TPN_strnest=12 +++ INPES_strnest=2 +++ JNPES_strnest=4 +++ COMPILER=gnu +++ [[ gnu = gnu ]] +++ WLCLK_dflt=30 ++ export TEST_NAME=compile ++ TEST_NAME=compile ++ export TEST_NR=4 ++ TEST_NR=4 ++ export JBNME=compile_4 ++ JBNME=compile_4 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_4 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_4 ++ source rt_utils.sh +++ set -eu +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ UNIT_TEST=false ++ source atparse.bash ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_4 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_4 ++ [[ slurm = \s\l\u\r\m ]] ++ atparse ++ local __set_x ++ '[' -o xtrace ']' ++ __set_x='set -x' ++ set +x ++ [[ false = \f\a\l\s\e ]] ++ submit_and_wait job_card ++ [[ -z job_card ]] ++ '[' -o xtrace ']' ++ set_x='set -x' ++ set +x +Job id 8891754 +TEST 4 compile is waiting to enter the queue +1 min. TEST 4 compile is pending, Status: PD +2 min. TEST 4 compile is running, Status: R +3 min. TEST 4 compile is running, Status: R +Slurm unknown status -. Check sacct ... +8891754 COMPLETED compile_4 +8891754.bat+ COMPLETED batch +8891754.ext+ COMPLETED extern +4 min. TEST 4 compile is COMPLETED +COMPLETED +COMPLETED ++ elapsed=246 ++ echo 'Elapsed time 246 seconds. Compile 4' +Elapsed time 246 seconds. Compile 4 ++ SECONDS=0 ++ [[ 4 != 4 ]] ++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ export RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ RUNDIR_ROOT=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' ++ export COMPILE_NR=5 ++ COMPILE_NR=5 ++ cd /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests ++ [[ -e /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_5.env ]] ++ source /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_5.env +++ export MACHINE_ID=hera.gnu +++ MACHINE_ID=hera.gnu +++ export PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ PATHRT=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests +++ export PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ PATHTR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu +++ export SCHEDULER=slurm +++ SCHEDULER=slurm +++ export ACCNR=gmtb +++ ACCNR=gmtb +++ export QUEUE=batch +++ QUEUE=batch +++ export PARTITION= +++ PARTITION= +++ export ROCOTO=false +++ ROCOTO=false +++ export ECFLOW=true +++ ECFLOW=true +++ export REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ REGRESSIONTEST_LOG=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/RegressionTests_hera.gnu.log +++ export LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu +++ LOG_DIR=/scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/log_hera.gnu ++ source default_vars.sh +++ '[' hera.gnu = wcoss ']' +++ '[' hera.gnu = wcoss_cray ']' +++ '[' hera.gnu = wcoss_dell_p3 ']' +++ [[ hera.gnu = orion.* ]] +++ [[ hera.gnu = hera.* ]] +++ TASKS_dflt=150 +++ TPN_dflt=40 +++ INPES_dflt=3 +++ JNPES_dflt=8 +++ TASKS_thrd=84 +++ TPN_thrd=20 +++ INPES_thrd=3 +++ JNPES_thrd=4 +++ TASKS_stretch=48 +++ TPN_stretch=12 +++ INPES_stretch=2 +++ JNPES_stretch=4 +++ TASKS_strnest=96 +++ TPN_strnest=12 +++ INPES_strnest=2 +++ JNPES_strnest=4 +++ COMPILER=gnu +++ [[ gnu = gnu ]] +++ WLCLK_dflt=30 ++ export TEST_NAME=compile ++ TEST_NAME=compile ++ export TEST_NR=5 ++ TEST_NR=5 ++ export JBNME=compile_5 ++ JBNME=compile_5 ++ export RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_5 ++ RUNDIR=/scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_5 ++ source rt_utils.sh +++ set -eu +++ [[ /scratch1/BMC/gmtb/Dom.Heinzeller/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/gnu/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ UNIT_TEST=false ++ source atparse.bash ++ mkdir -p /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_5 ++ cd /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/compile_5 ++ [[ slurm = \s\l\u\r\m ]] ++ atparse ++ local __set_x ++ '[' -o xtrace ']' ++ __set_x='set -x' ++ set +x ++ [[ false = \f\a\l\s\e ]] ++ submit_and_wait job_card ++ [[ -z job_card ]] ++ '[' -o xtrace ']' ++ set_x='set -x' ++ set +x +Job id 8891755 +TEST 5 compile is waiting to enter the queue +1 min. TEST 5 compile is pending, Status: PD +2 min. TEST 5 compile is pending, Status: PD +3 min. TEST 5 compile is pending, Status: PD +4 min. TEST 5 compile is pending, Status: PD +5 min. TEST 5 compile is pending, Status: PD +6 min. TEST 5 compile is running, Status: R +Slurm unknown status CG. Check sacct ... +8891755 COMPLETED compile_5 +8891755.bat+ COMPLETED batch +8891755.ext+ COMPLETED extern +7 min. TEST 5 compile is COMPLETED +COMPLETED +COMPLETED +Slurm unknown status -. Check sacct ... +8891755 COMPLETED compile_5 +8891755.bat+ COMPLETED batch +8891755.ext+ COMPLETED extern +8 min. TEST 5 compile is COMPLETED +COMPLETED +COMPLETED ++ elapsed=486 ++ echo 'Elapsed time 486 seconds. Compile 5' +Elapsed time 486 seconds. Compile 5 diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log new file mode 100644 index 0000000000..0d35d3906b --- /dev/null +++ b/tests/RegressionTests_hera.gnu.log @@ -0,0 +1,697 @@ +Fri Jun 19 21:24:32 UTC 2020 +Start Regression test + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_gfdlmp_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_gfdlmp_prod +Checking test 001 fv3_ccpp_gfdlmp results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.nemsio .........OK + Comparing phyf024.nemsio .........OK + Comparing dynf000.nemsio .........OK + Comparing dynf024.nemsio .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK +Test 001 fv3_ccpp_gfdlmp PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_gfs_v15p2_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_gfs_v15p2_prod +Checking test 002 fv3_ccpp_gfs_v15p2 results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK +Test 002 fv3_ccpp_gfs_v15p2 PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_gfs_v16beta_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_gfs_v16beta_prod +Checking test 003 fv3_ccpp_gfs_v16beta results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK +Test 003 fv3_ccpp_gfs_v16beta PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_rrtmgp_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_rrtmgp_prod +Checking test 004 fv3_ccpp_rrtmgp results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test 004 fv3_ccpp_rrtmgp PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_gsd_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_gsd_prod +Checking test 005 fv3_ccpp_gsd results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing phyf027.tile1.nc .........OK + Comparing phyf027.tile2.nc .........OK + Comparing phyf027.tile3.nc .........OK + Comparing phyf027.tile4.nc .........OK + Comparing phyf027.tile5.nc .........OK + Comparing phyf027.tile6.nc .........OK + Comparing phyf048.tile1.nc .........OK + Comparing phyf048.tile2.nc .........OK + Comparing phyf048.tile3.nc .........OK + Comparing phyf048.tile4.nc .........OK + Comparing phyf048.tile5.nc .........OK + Comparing phyf048.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing dynf027.tile1.nc .........OK + Comparing dynf027.tile2.nc .........OK + Comparing dynf027.tile3.nc .........OK + Comparing dynf027.tile4.nc .........OK + Comparing dynf027.tile5.nc .........OK + Comparing dynf027.tile6.nc .........OK + Comparing dynf048.tile1.nc .........OK + Comparing dynf048.tile2.nc .........OK + Comparing dynf048.tile3.nc .........OK + Comparing dynf048.tile4.nc .........OK + Comparing dynf048.tile5.nc .........OK + Comparing dynf048.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test 005 fv3_ccpp_gsd PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_thompson_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_thompson_prod +Checking test 006 fv3_ccpp_thompson results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test 006 fv3_ccpp_thompson PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_thompson_no_aero_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_thompson_no_aero_prod +Checking test 007 fv3_ccpp_thompson_no_aero results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf024.tile1.nc .........OK + Comparing phyf024.tile2.nc .........OK + Comparing phyf024.tile3.nc .........OK + Comparing phyf024.tile4.nc .........OK + Comparing phyf024.tile5.nc .........OK + Comparing phyf024.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf024.tile1.nc .........OK + Comparing dynf024.tile2.nc .........OK + Comparing dynf024.tile3.nc .........OK + Comparing dynf024.tile4.nc .........OK + Comparing dynf024.tile5.nc .........OK + Comparing dynf024.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test 007 fv3_ccpp_thompson_no_aero PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_control_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_control_debug_prod +Checking test 008 fv3_ccpp_control_debug results .... +Test 008 fv3_ccpp_control_debug PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_gfs_v15p2_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_gfs_v15p2_debug_prod +Checking test 009 fv3_ccpp_gfs_v15p2_debug results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf006.tile1.nc .........OK + Comparing phyf006.tile2.nc .........OK + Comparing phyf006.tile3.nc .........OK + Comparing phyf006.tile4.nc .........OK + Comparing phyf006.tile5.nc .........OK + Comparing phyf006.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf006.tile1.nc .........OK + Comparing dynf006.tile2.nc .........OK + Comparing dynf006.tile3.nc .........OK + Comparing dynf006.tile4.nc .........OK + Comparing dynf006.tile5.nc .........OK + Comparing dynf006.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test 009 fv3_ccpp_gfs_v15p2_debug PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_gfs_v16beta_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_gfs_v16beta_debug_prod +Checking test 010 fv3_ccpp_gfs_v16beta_debug results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf006.tile1.nc .........OK + Comparing phyf006.tile2.nc .........OK + Comparing phyf006.tile3.nc .........OK + Comparing phyf006.tile4.nc .........OK + Comparing phyf006.tile5.nc .........OK + Comparing phyf006.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf006.tile1.nc .........OK + Comparing dynf006.tile2.nc .........OK + Comparing dynf006.tile3.nc .........OK + Comparing dynf006.tile4.nc .........OK + Comparing dynf006.tile5.nc .........OK + Comparing dynf006.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test 010 fv3_ccpp_gfs_v16beta_debug PASS + + +baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/GNU/fv3_rrtmgp_debug_ccpp +working dir = /scratch1/NCEPDEV/stmp2/Dom.Heinzeller/FV3_RT/rt_225252/fv3_ccpp_rrtmgp_debug_prod +Checking test 011 fv3_ccpp_rrtmgp_debug results .... + Comparing atmos_4xdaily.tile1.nc .........OK + Comparing atmos_4xdaily.tile2.nc .........OK + Comparing atmos_4xdaily.tile3.nc .........OK + Comparing atmos_4xdaily.tile4.nc .........OK + Comparing atmos_4xdaily.tile5.nc .........OK + Comparing atmos_4xdaily.tile6.nc .........OK + Comparing phyf000.tile1.nc .........OK + Comparing phyf000.tile2.nc .........OK + Comparing phyf000.tile3.nc .........OK + Comparing phyf000.tile4.nc .........OK + Comparing phyf000.tile5.nc .........OK + Comparing phyf000.tile6.nc .........OK + Comparing phyf006.tile1.nc .........OK + Comparing phyf006.tile2.nc .........OK + Comparing phyf006.tile3.nc .........OK + Comparing phyf006.tile4.nc .........OK + Comparing phyf006.tile5.nc .........OK + Comparing phyf006.tile6.nc .........OK + Comparing dynf000.tile1.nc .........OK + Comparing dynf000.tile2.nc .........OK + Comparing dynf000.tile3.nc .........OK + Comparing dynf000.tile4.nc .........OK + Comparing dynf000.tile5.nc .........OK + Comparing dynf000.tile6.nc .........OK + Comparing dynf006.tile1.nc .........OK + Comparing dynf006.tile2.nc .........OK + Comparing dynf006.tile3.nc .........OK + Comparing dynf006.tile4.nc .........OK + Comparing dynf006.tile5.nc .........OK + Comparing dynf006.tile6.nc .........OK + Comparing RESTART/coupler.res .........OK + Comparing RESTART/fv_core.res.nc ............SKIP for gnu/pgi compilers + Comparing RESTART/fv_core.res.tile1.nc .........OK + Comparing RESTART/fv_core.res.tile2.nc .........OK + Comparing RESTART/fv_core.res.tile3.nc .........OK + Comparing RESTART/fv_core.res.tile4.nc .........OK + Comparing RESTART/fv_core.res.tile5.nc .........OK + Comparing RESTART/fv_core.res.tile6.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile1.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile2.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile3.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile4.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile5.nc .........OK + Comparing RESTART/fv_srf_wnd.res.tile6.nc .........OK + Comparing RESTART/fv_tracer.res.tile1.nc .........OK + Comparing RESTART/fv_tracer.res.tile2.nc .........OK + Comparing RESTART/fv_tracer.res.tile3.nc .........OK + Comparing RESTART/fv_tracer.res.tile4.nc .........OK + Comparing RESTART/fv_tracer.res.tile5.nc .........OK + Comparing RESTART/fv_tracer.res.tile6.nc .........OK + Comparing RESTART/phy_data.tile1.nc .........OK + Comparing RESTART/phy_data.tile2.nc .........OK + Comparing RESTART/phy_data.tile3.nc .........OK + Comparing RESTART/phy_data.tile4.nc .........OK + Comparing RESTART/phy_data.tile5.nc .........OK + Comparing RESTART/phy_data.tile6.nc .........OK + Comparing RESTART/sfc_data.tile1.nc .........OK + Comparing RESTART/sfc_data.tile2.nc .........OK + Comparing RESTART/sfc_data.tile3.nc .........OK + Comparing RESTART/sfc_data.tile4.nc .........OK + Comparing RESTART/sfc_data.tile5.nc .........OK + Comparing RESTART/sfc_data.tile6.nc .........OK +Test 011 fv3_ccpp_rrtmgp_debug PASS + + +REGRESSION TEST WAS SUCCESSFUL +Fri Jun 19 21:58:27 UTC 2020 +Elapsed time: 00h:33m:56s. Have a nice day! From 0941fb3d757692f0a2430060427c95549ce1dbb8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 19 Jun 2020 16:29:10 -0600 Subject: [PATCH 7/9] Regression test logs for orion.intel --- tests/Compile_orion.intel.log | 835 +++++++++++++------------- tests/RegressionTests_orion.intel.log | 100 +-- 2 files changed, 466 insertions(+), 469 deletions(-) diff --git a/tests/Compile_orion.intel.log b/tests/Compile_orion.intel.log index 972ae0b6cc..b084c7a643 100644 --- a/tests/Compile_orion.intel.log +++ b/tests/Compile_orion.intel.log @@ -1,26 +1,26 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP DEBUG=Y' + export COMPILE_NR=10 + COMPILE_NR=10 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_10.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_10.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_10.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_10.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -64,15 +64,15 @@ + TEST_NR=10 + export JBNME=compile_10 + JBNME=compile_10 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_10 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_10 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_10 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_10 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_10 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_10 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_10 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_10 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -85,42 +85,43 @@ + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90383 +Job id 105662 TEST 10 compile is waiting to enter the queue 1 min. TEST 10 compile is running, Status: R +2 min. TEST 10 compile is running, Status: R Slurm unknown status -. Check sacct ... -90383 COMPLETED compile_10 -90383.batch COMPLETED batch -90383.extern COMPLETED extern -2 min. TEST 10 compile is COMPLETED +105662 COMPLETED compile_10 +105662.batch COMPLETED batch +105662.exte+ COMPLETED extern +3 min. TEST 10 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=126 -+ echo 'Elapsed time 126 seconds. Compile 10' -Elapsed time 126 seconds. Compile 10 ++ elapsed=185 ++ echo 'Elapsed time 185 seconds. Compile 10' +Elapsed time 185 seconds. Compile 10 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + export COMPILE_NR=11 + COMPILE_NR=11 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_11.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_11.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_11.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_11.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -129,10 +130,10 @@ Elapsed time 126 seconds. Compile 10 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -164,15 +165,15 @@ Elapsed time 126 seconds. Compile 10 + TEST_NR=11 + export JBNME=compile_11 + JBNME=compile_11 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_11 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_11 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_11 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_11 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_11 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_11 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_11 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_11 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -185,43 +186,46 @@ Elapsed time 126 seconds. Compile 10 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90390 +Job id 105717 TEST 11 compile is waiting to enter the queue 1 min. TEST 11 compile is running, Status: R 2 min. TEST 11 compile is running, Status: R +3 min. TEST 11 compile is running, Status: R +4 min. TEST 11 compile is running, Status: R +5 min. TEST 11 compile is running, Status: R Slurm unknown status -. Check sacct ... -90390 COMPLETED compile_11 -90390.batch COMPLETED batch -90390.extern COMPLETED extern -3 min. TEST 11 compile is COMPLETED +105717 COMPLETED compile_11 +105717.batch COMPLETED batch +105717.exte+ COMPLETED extern +6 min. TEST 11 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=186 -+ echo 'Elapsed time 186 seconds. Compile 11' -Elapsed time 186 seconds. Compile 11 ++ elapsed=365 ++ echo 'Elapsed time 365 seconds. Compile 11' +Elapsed time 365 seconds. Compile 11 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' + export COMPILE_NR=12 + COMPILE_NR=12 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_12.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_12.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_12.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_12.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -230,10 +234,10 @@ Elapsed time 186 seconds. Compile 11 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -265,15 +269,15 @@ Elapsed time 186 seconds. Compile 11 + TEST_NR=12 + export JBNME=compile_12 + JBNME=compile_12 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_12 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_12 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_12 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_12 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_12 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_12 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_12 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_12 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -286,48 +290,53 @@ Elapsed time 186 seconds. Compile 11 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90398 +Job id 105737 TEST 12 compile is waiting to enter the queue -1 min. TEST 12 compile is running, Status: R -2 min. TEST 12 compile is running, Status: R -3 min. TEST 12 compile is running, Status: R -4 min. TEST 12 compile is running, Status: R -5 min. TEST 12 compile is running, Status: R +1 min. TEST 12 compile is pending, Status: PD +2 min. TEST 12 compile is pending, Status: PD +3 min. TEST 12 compile is pending, Status: PD +4 min. TEST 12 compile is pending, Status: PD +5 min. TEST 12 compile is pending, Status: PD 6 min. TEST 12 compile is running, Status: R 7 min. TEST 12 compile is running, Status: R +8 min. TEST 12 compile is running, Status: R +9 min. TEST 12 compile is running, Status: R +10 min. TEST 12 compile is running, Status: R +11 min. TEST 12 compile is running, Status: R +12 min. TEST 12 compile is running, Status: R Slurm unknown status -. Check sacct ... -90398 COMPLETED compile_12 -90398.batch COMPLETED batch -90398.extern COMPLETED extern -8 min. TEST 12 compile is COMPLETED +105737 COMPLETED compile_12 +105737.batch COMPLETED batch +105737.exte+ COMPLETED extern +13 min. TEST 12 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=486 -+ echo 'Elapsed time 486 seconds. Compile 12' -Elapsed time 486 seconds. Compile 12 ++ elapsed=786 ++ echo 'Elapsed time 786 seconds. Compile 12' +Elapsed time 786 seconds. Compile 12 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=13 + COMPILE_NR=13 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_13.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_13.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_13.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_13.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -336,10 +345,10 @@ Elapsed time 486 seconds. Compile 12 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -371,15 +380,15 @@ Elapsed time 486 seconds. Compile 12 + TEST_NR=13 + export JBNME=compile_13 + JBNME=compile_13 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_13 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_13 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_13 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_13 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_13 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_13 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_13 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_13 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -392,47 +401,52 @@ Elapsed time 486 seconds. Compile 12 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90406 +Job id 105738 TEST 13 compile is waiting to enter the queue -1 min. TEST 13 compile is running, Status: R -2 min. TEST 13 compile is running, Status: R -3 min. TEST 13 compile is running, Status: R -4 min. TEST 13 compile is running, Status: R -5 min. TEST 13 compile is running, Status: R +1 min. TEST 13 compile is pending, Status: PD +2 min. TEST 13 compile is pending, Status: PD +3 min. TEST 13 compile is pending, Status: PD +4 min. TEST 13 compile is pending, Status: PD +5 min. TEST 13 compile is pending, Status: PD 6 min. TEST 13 compile is running, Status: R +7 min. TEST 13 compile is running, Status: R +8 min. TEST 13 compile is running, Status: R +9 min. TEST 13 compile is running, Status: R +10 min. TEST 13 compile is running, Status: R +11 min. TEST 13 compile is running, Status: R Slurm unknown status -. Check sacct ... -90406 COMPLETED compile_13 -90406.batch COMPLETED batch -90406.extern COMPLETED extern -7 min. TEST 13 compile is COMPLETED +105738 COMPLETED compile_13 +105738.batch COMPLETED batch +105738.exte+ COMPLETED extern +12 min. TEST 13 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 13' -Elapsed time 426 seconds. Compile 13 ++ elapsed=726 ++ echo 'Elapsed time 726 seconds. Compile 13' +Elapsed time 726 seconds. Compile 13 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' + export COMPILE_NR=1 + COMPILE_NR=1 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_1.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_1.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_1.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_1.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -441,10 +455,10 @@ Elapsed time 426 seconds. Compile 13 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -476,15 +490,15 @@ Elapsed time 426 seconds. Compile 13 + TEST_NR=1 + export JBNME=compile_1 + JBNME=compile_1 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_1 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_1 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_1 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_1 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_1 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_1 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_1 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -497,7 +511,7 @@ Elapsed time 426 seconds. Compile 13 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90380 +Job id 105666 TEST 1 compile is waiting to enter the queue 1 min. TEST 1 compile is running, Status: R 2 min. TEST 1 compile is running, Status: R @@ -506,38 +520,38 @@ TEST 1 compile is waiting to enter the queue 5 min. TEST 1 compile is running, Status: R 6 min. TEST 1 compile is running, Status: R Slurm unknown status -. Check sacct ... -90380 COMPLETED compile_1 -90380.batch COMPLETED batch -90380.extern COMPLETED extern +105666 COMPLETED compile_1 +105666.batch COMPLETED batch +105666.exte+ COMPLETED extern 7 min. TEST 1 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 1' -Elapsed time 426 seconds. Compile 1 ++ elapsed=425 ++ echo 'Elapsed time 425 seconds. Compile 1' +Elapsed time 425 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_2.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_2.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_2.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_2.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -546,10 +560,10 @@ Elapsed time 426 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -581,15 +595,15 @@ Elapsed time 426 seconds. Compile 1 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_2 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_2 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_2 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_2 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_2 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_2 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_2 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -602,7 +616,7 @@ Elapsed time 426 seconds. Compile 1 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90389 +Job id 105667 TEST 2 compile is waiting to enter the queue 1 min. TEST 2 compile is running, Status: R 2 min. TEST 2 compile is running, Status: R @@ -616,38 +630,38 @@ TEST 2 compile is waiting to enter the queue 10 min. TEST 2 compile is running, Status: R 11 min. TEST 2 compile is running, Status: R Slurm unknown status -. Check sacct ... -90389 COMPLETED compile_2 -90389.batch COMPLETED batch -90389.extern COMPLETED extern +105667 COMPLETED compile_2 +105667.batch COMPLETED batch +105667.exte+ COMPLETED extern 12 min. TEST 2 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=727 -+ echo 'Elapsed time 727 seconds. Compile 2' -Elapsed time 727 seconds. Compile 2 ++ elapsed=725 ++ echo 'Elapsed time 725 seconds. Compile 2' +Elapsed time 725 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_3.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_3.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_3.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_3.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -656,10 +670,10 @@ Elapsed time 727 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -691,15 +705,15 @@ Elapsed time 727 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_3 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_3 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_3 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_3 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_3 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_3 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_3 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -712,7 +726,7 @@ Elapsed time 727 seconds. Compile 2 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90381 +Job id 105664 TEST 3 compile is waiting to enter the queue 1 min. TEST 3 compile is running, Status: R 2 min. TEST 3 compile is running, Status: R @@ -727,45 +741,39 @@ TEST 3 compile is waiting to enter the queue 11 min. TEST 3 compile is running, Status: R 12 min. TEST 3 compile is running, Status: R 13 min. TEST 3 compile is running, Status: R -14 min. TEST 3 compile is running, Status: R -15 min. TEST 3 compile is running, Status: R -16 min. TEST 3 compile is running, Status: R -17 min. TEST 3 compile is running, Status: R -18 min. TEST 3 compile is running, Status: R -19 min. TEST 3 compile is running, Status: R Slurm unknown status -. Check sacct ... -90381 COMPLETED compile_3 -90381.batch COMPLETED batch -90381.extern COMPLETED extern -20 min. TEST 3 compile is COMPLETED +105664 COMPLETED compile_3 +105664.batch COMPLETED batch +105664.exte+ COMPLETED extern +14 min. TEST 3 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1207 -+ echo 'Elapsed time 1207 seconds. Compile 3' -Elapsed time 1207 seconds. Compile 3 ++ elapsed=846 ++ echo 'Elapsed time 846 seconds. Compile 3' +Elapsed time 846 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_4.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_4.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_4.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_4.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -774,10 +782,10 @@ Elapsed time 1207 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -809,15 +817,15 @@ Elapsed time 1207 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_4 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_4 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_4 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_4 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_4 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_4 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_4 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -830,7 +838,7 @@ Elapsed time 1207 seconds. Compile 3 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90385 +Job id 105670 TEST 4 compile is waiting to enter the queue 1 min. TEST 4 compile is running, Status: R 2 min. TEST 4 compile is running, Status: R @@ -843,39 +851,43 @@ TEST 4 compile is waiting to enter the queue 9 min. TEST 4 compile is running, Status: R 10 min. TEST 4 compile is running, Status: R 11 min. TEST 4 compile is running, Status: R +12 min. TEST 4 compile is running, Status: R +13 min. TEST 4 compile is running, Status: R +14 min. TEST 4 compile is running, Status: R +15 min. TEST 4 compile is running, Status: R Slurm unknown status -. Check sacct ... -90385 COMPLETED compile_4 -90385.batch COMPLETED batch -90385.extern COMPLETED extern -12 min. TEST 4 compile is COMPLETED +105670 COMPLETED compile_4 +105670.batch COMPLETED batch +105670.exte+ COMPLETED extern +16 min. TEST 4 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=727 -+ echo 'Elapsed time 727 seconds. Compile 4' -Elapsed time 727 seconds. Compile 4 ++ elapsed=966 ++ echo 'Elapsed time 966 seconds. Compile 4' +Elapsed time 966 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_5.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_5.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_5.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_5.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -884,10 +896,10 @@ Elapsed time 727 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -919,15 +931,15 @@ Elapsed time 727 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_5 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_5 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_5 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_5 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_5 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_5 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_5 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -940,7 +952,7 @@ Elapsed time 727 seconds. Compile 4 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90388 +Job id 105669 TEST 5 compile is waiting to enter the queue 1 min. TEST 5 compile is running, Status: R 2 min. TEST 5 compile is running, Status: R @@ -951,45 +963,39 @@ TEST 5 compile is waiting to enter the queue 7 min. TEST 5 compile is running, Status: R 8 min. TEST 5 compile is running, Status: R 9 min. TEST 5 compile is running, Status: R -10 min. TEST 5 compile is running, Status: R -11 min. TEST 5 compile is running, Status: R -12 min. TEST 5 compile is running, Status: R -13 min. TEST 5 compile is running, Status: R -14 min. TEST 5 compile is running, Status: R -15 min. TEST 5 compile is running, Status: R Slurm unknown status -. Check sacct ... -90388 COMPLETED compile_5 -90388.batch COMPLETED batch -90388.extern COMPLETED extern -16 min. TEST 5 compile is COMPLETED +105669 COMPLETED compile_5 +105669.batch COMPLETED batch +105669.exte+ COMPLETED extern +10 min. TEST 5 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=967 -+ echo 'Elapsed time 967 seconds. Compile 5' -Elapsed time 967 seconds. Compile 5 ++ elapsed=605 ++ echo 'Elapsed time 605 seconds. Compile 5' +Elapsed time 605 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_6.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_6.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_6.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_6.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -998,10 +1004,10 @@ Elapsed time 967 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1033,15 +1039,15 @@ Elapsed time 967 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_6 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_6 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_6 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_6 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_6 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_6 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_6 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1054,7 +1060,7 @@ Elapsed time 967 seconds. Compile 5 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90384 +Job id 105665 TEST 6 compile is waiting to enter the queue 1 min. TEST 6 compile is running, Status: R 2 min. TEST 6 compile is running, Status: R @@ -1062,45 +1068,39 @@ TEST 6 compile is waiting to enter the queue 4 min. TEST 6 compile is running, Status: R 5 min. TEST 6 compile is running, Status: R 6 min. TEST 6 compile is running, Status: R -7 min. TEST 6 compile is running, Status: R -8 min. TEST 6 compile is running, Status: R -9 min. TEST 6 compile is running, Status: R -10 min. TEST 6 compile is running, Status: R -11 min. TEST 6 compile is running, Status: R -12 min. TEST 6 compile is running, Status: R Slurm unknown status -. Check sacct ... -90384 COMPLETED compile_6 -90384.batch COMPLETED batch -90384.extern COMPLETED extern -13 min. TEST 6 compile is COMPLETED +105665 COMPLETED compile_6 +105665.batch COMPLETED batch +105665.exte+ COMPLETED extern +7 min. TEST 6 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=787 -+ echo 'Elapsed time 787 seconds. Compile 6' -Elapsed time 787 seconds. Compile 6 ++ elapsed=425 ++ echo 'Elapsed time 425 seconds. Compile 6' +Elapsed time 425 seconds. Compile 6 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf' + export COMPILE_NR=7 + COMPILE_NR=7 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_7.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_7.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_7.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_7.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -1109,10 +1109,10 @@ Elapsed time 787 seconds. Compile 6 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1144,15 +1144,15 @@ Elapsed time 787 seconds. Compile 6 + TEST_NR=7 + export JBNME=compile_7 + JBNME=compile_7 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_7 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_7 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_7 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_7 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_7 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_7 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_7 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_7 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1165,7 +1165,7 @@ Elapsed time 787 seconds. Compile 6 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90386 +Job id 105663 TEST 7 compile is waiting to enter the queue 1 min. TEST 7 compile is running, Status: R 2 min. TEST 7 compile is running, Status: R @@ -1180,43 +1180,39 @@ TEST 7 compile is waiting to enter the queue 11 min. TEST 7 compile is running, Status: R 12 min. TEST 7 compile is running, Status: R 13 min. TEST 7 compile is running, Status: R -14 min. TEST 7 compile is running, Status: R -15 min. TEST 7 compile is running, Status: R -16 min. TEST 7 compile is running, Status: R -17 min. TEST 7 compile is running, Status: R Slurm unknown status -. Check sacct ... -90386 COMPLETED compile_7 -90386.batch COMPLETED batch -90386.extern COMPLETED extern -18 min. TEST 7 compile is COMPLETED +105663 COMPLETED compile_7 +105663.batch COMPLETED batch +105663.exte+ COMPLETED extern +14 min. TEST 7 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=1087 -+ echo 'Elapsed time 1087 seconds. Compile 7' -Elapsed time 1087 seconds. Compile 7 ++ elapsed=846 ++ echo 'Elapsed time 846 seconds. Compile 7' +Elapsed time 846 seconds. Compile 7 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + export COMPILE_NR=8 + COMPILE_NR=8 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_8.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_8.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_8.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_8.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -1225,10 +1221,10 @@ Elapsed time 1087 seconds. Compile 7 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1260,15 +1256,15 @@ Elapsed time 1087 seconds. Compile 7 + TEST_NR=8 + export JBNME=compile_8 + JBNME=compile_8 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_8 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_8 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_8 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_8 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_8 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_8 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_8 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_8 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1281,7 +1277,7 @@ Elapsed time 1087 seconds. Compile 7 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90387 +Job id 105661 TEST 8 compile is waiting to enter the queue 1 min. TEST 8 compile is running, Status: R 2 min. TEST 8 compile is running, Status: R @@ -1289,39 +1285,40 @@ TEST 8 compile is waiting to enter the queue 4 min. TEST 8 compile is running, Status: R 5 min. TEST 8 compile is running, Status: R 6 min. TEST 8 compile is running, Status: R +7 min. TEST 8 compile is running, Status: R Slurm unknown status -. Check sacct ... -90387 COMPLETED compile_8 -90387.batch COMPLETED batch -90387.extern COMPLETED extern -7 min. TEST 8 compile is COMPLETED +105661 COMPLETED compile_8 +105661.batch COMPLETED batch +105661.exte+ COMPLETED extern +8 min. TEST 8 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 8' -Elapsed time 426 seconds. Compile 8 ++ elapsed=485 ++ echo 'Elapsed time 485 seconds. Compile 8' +Elapsed time 485 seconds. Compile 8 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 -+ RUNDIR_ROOT=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148 ++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ export RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 ++ RUNDIR_ROOT=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v15p2,FV3_GFS_v16beta,FV3_GFS_2017_RRTMGP' + export COMPILE_NR=9 + COMPILE_NR=9 -+ cd /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -+ [[ -e /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_9.env ]] -+ source /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_9.env ++ cd /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests ++ [[ -e /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_9.env ]] ++ source /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_9.env ++ export MACHINE_ID=orion.intel ++ MACHINE_ID=orion.intel -++ export PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ PATHRT=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests -++ export PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model -++ PATHTR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model +++ export PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ PATHRT=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests +++ export PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update +++ PATHTR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update ++ export SCHEDULER=slurm ++ SCHEDULER=slurm -++ export ACCNR=nems -++ ACCNR=nems +++ export ACCNR=gsd-hpcs +++ ACCNR=gsd-hpcs ++ export QUEUE=batch ++ QUEUE=batch ++ export PARTITION=orion @@ -1330,10 +1327,10 @@ Elapsed time 426 seconds. Compile 8 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ REGRESSIONTEST_LOG=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/RegressionTests_orion.intel.log -++ export LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel -++ LOG_DIR=/work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/log_orion.intel +++ export REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ REGRESSIONTEST_LOG=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/RegressionTests_orion.intel.log +++ export LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel +++ LOG_DIR=/work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/log_orion.intel + source default_vars.sh ++ '[' orion.intel = wcoss ']' ++ '[' orion.intel = wcoss_cray ']' @@ -1365,15 +1362,15 @@ Elapsed time 426 seconds. Compile 8 + TEST_NR=9 + export JBNME=compile_9 + JBNME=compile_9 -+ export RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_9 -+ RUNDIR=/work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_9 ++ export RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_9 ++ RUNDIR=/work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_9 + source rt_utils.sh ++ set -eu -++ [[ /work/noaa/nems/junwang/ufs-weather/20200611/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /work/noaa/gmtb/dheinzel/ufs-weather-model/ufs-weather-model-emc-develop-20200616-jetbuild-heragnu-dycore-update/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_9 -+ cd /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/compile_9 ++ mkdir -p /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_9 ++ cd /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/compile_9 + [[ slurm = \s\l\u\r\m ]] + atparse + local __set_x @@ -1386,7 +1383,7 @@ Elapsed time 426 seconds. Compile 8 + '[' -o xtrace ']' + set_x='set -x' + set +x -Job id 90382 +Job id 105668 TEST 9 compile is waiting to enter the queue 1 min. TEST 9 compile is running, Status: R 2 min. TEST 9 compile is running, Status: R @@ -1395,12 +1392,12 @@ TEST 9 compile is waiting to enter the queue 5 min. TEST 9 compile is running, Status: R 6 min. TEST 9 compile is running, Status: R Slurm unknown status -. Check sacct ... -90382 COMPLETED compile_9 -90382.batch COMPLETED batch -90382.extern COMPLETED extern +105668 COMPLETED compile_9 +105668.batch COMPLETED batch +105668.exte+ COMPLETED extern 7 min. TEST 9 compile is COMPLETED COMPLETED COMPLETED -+ elapsed=426 -+ echo 'Elapsed time 426 seconds. Compile 9' -Elapsed time 426 seconds. Compile 9 ++ elapsed=425 ++ echo 'Elapsed time 425 seconds. Compile 9' +Elapsed time 425 seconds. Compile 9 diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log index d1dd23f5cc..88ce13e051 100644 --- a/tests/RegressionTests_orion.intel.log +++ b/tests/RegressionTests_orion.intel.log @@ -1,9 +1,9 @@ -Fri Jun 12 07:46:08 CDT 2020 +Fri Jun 19 16:09:47 CDT 2020 Start Regression test baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_control_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_decomp_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_2threads_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_restart_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_restart_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_read_inc_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_read_inc_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_netcdf_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_nemsio_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_wrtGauss_nemsio_prod Checking test 008 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_stochy_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_stochy_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_stochy_prod Checking test 010 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -603,7 +603,7 @@ Test 010 fv3_ccpp_stochy PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_iau_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_iau_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_iau_prod Checking test 011 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -671,7 +671,7 @@ Test 011 fv3_ccpp_iau PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_ca_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_ca_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_ca_prod Checking test 012 fv3_ccpp_ca results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -739,7 +739,7 @@ Test 012 fv3_ccpp_ca PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfdlmprad_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfdlmprad_prod Checking test 013 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -788,7 +788,7 @@ Test 013 fv3_ccpp_gfdlmprad PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_atmwav_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 014 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -837,7 +837,7 @@ Test 014 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_wrtGauss_nemsio_c768_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_wrtGauss_nemsio_c768_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_wrtGauss_nemsio_c768_prod Checking test 015 fv3_ccpp_wrtGauss_nemsio_c768 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -886,7 +886,7 @@ Test 015 fv3_ccpp_wrtGauss_nemsio_c768 PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_32bit_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_control_32bit_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_control_32bit_prod Checking test 016 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -954,7 +954,7 @@ Test 016 fv3_ccpp_control_32bit PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_stretched_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_stretched_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_stretched_prod Checking test 017 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1010,7 +1010,7 @@ Test 017 fv3_ccpp_stretched PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_stretched_nest_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_stretched_nest_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_stretched_nest_prod Checking test 018 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1077,7 +1077,7 @@ Test 018 fv3_ccpp_stretched_nest PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_regional_control_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_regional_control_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_regional_control_prod Checking test 019 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1088,7 +1088,7 @@ Test 019 fv3_ccpp_regional_control PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_regional_restart_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_regional_restart_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_regional_restart_prod Checking test 020 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1097,7 +1097,7 @@ Test 020 fv3_ccpp_regional_restart PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_regional_quilt_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_regional_quilt_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_regional_quilt_prod Checking test 021 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1108,7 +1108,7 @@ Test 021 fv3_ccpp_regional_quilt PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_regional_c768_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_regional_c768_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_regional_c768_prod Checking test 022 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1119,19 +1119,19 @@ Test 022 fv3_ccpp_regional_c768 PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_control_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_control_debug_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_control_debug_prod Checking test 023 fv3_ccpp_control_debug results .... Test 023 fv3_ccpp_control_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_stretched_nest_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_stretched_nest_debug_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_stretched_nest_debug_prod Checking test 024 fv3_ccpp_stretched_nest_debug results .... Test 024 fv3_ccpp_stretched_nest_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmp_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfdlmp_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfdlmp_prod Checking test 025 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1179,7 +1179,7 @@ Test 025 fv3_ccpp_gfdlmp PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_gwd_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfdlmprad_gwd_prod Checking test 026 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1227,7 +1227,7 @@ Test 026 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_noahmp_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 027 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1275,7 +1275,7 @@ Test 027 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_csawmg_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_csawmg_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_csawmg_prod Checking test 028 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1323,7 +1323,7 @@ Test 028 fv3_ccpp_csawmg PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_satmedmf_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_satmedmf_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_satmedmf_prod Checking test 029 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1371,7 +1371,7 @@ Test 029 fv3_ccpp_satmedmf PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmp_32bit_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfdlmp_32bit_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfdlmp_32bit_prod Checking test 030 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1419,7 +1419,7 @@ Test 030 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfdlmprad_32bit_post_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 031 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1471,7 +1471,7 @@ Test 031 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_cpt_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_cpt_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_cpt_prod Checking test 032 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1525,7 +1525,7 @@ Test 032 fv3_ccpp_cpt PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gsd_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gsd_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gsd_prod Checking test 033 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1617,7 +1617,7 @@ Test 033 fv3_ccpp_gsd PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_thompson_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_thompson_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_thompson_prod Checking test 034 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1685,7 +1685,7 @@ Test 034 fv3_ccpp_thompson PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_thompson_no_aero_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_thompson_no_aero_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_thompson_no_aero_prod Checking test 035 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1753,7 +1753,7 @@ Test 035 fv3_ccpp_thompson_no_aero PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfs_v15p2_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfs_v15p2_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfs_v15p2_prod Checking test 036 fv3_ccpp_gfs_v15p2 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1821,7 +1821,7 @@ Test 036 fv3_ccpp_gfs_v15p2 PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfs_v16beta_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfs_v16beta_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfs_v16beta_prod Checking test 037 fv3_ccpp_gfs_v16beta results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1889,7 +1889,7 @@ Test 037 fv3_ccpp_gfs_v16beta PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_rrtmgp_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_rrtmgp_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_rrtmgp_prod Checking test 038 fv3_ccpp_rrtmgp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1957,7 +1957,7 @@ Test 038 fv3_ccpp_rrtmgp PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfs_v15p2_debug_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfs_v15p2_debug_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfs_v15p2_debug_prod Checking test 039 fv3_ccpp_gfs_v15p2_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2025,7 +2025,7 @@ Test 039 fv3_ccpp_gfs_v15p2_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfs_v16beta_debug_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfs_v16beta_debug_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfs_v16beta_debug_prod Checking test 040 fv3_ccpp_gfs_v16beta_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2093,7 +2093,7 @@ Test 040 fv3_ccpp_gfs_v16beta_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_rrtmgp_debug_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_rrtmgp_debug_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_rrtmgp_debug_prod Checking test 041 fv3_ccpp_rrtmgp_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2161,7 +2161,7 @@ Test 041 fv3_ccpp_rrtmgp_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gsd_debug_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gsd_debug_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gsd_debug_prod Checking test 042 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2229,7 +2229,7 @@ Test 042 fv3_ccpp_gsd_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_thompson_debug_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_thompson_debug_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_thompson_debug_prod Checking test 043 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2297,7 +2297,7 @@ Test 043 fv3_ccpp_thompson_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_thompson_no_aero_debug_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_thompson_no_aero_debug_prod Checking test 044 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2365,7 +2365,7 @@ Test 044 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfsv16_csawmg_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfsv16_csawmg_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfsv16_csawmg_prod Checking test 045 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2413,7 +2413,7 @@ Test 045 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gfsv16_csawmgt_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gfsv16_csawmgt_prod Checking test 046 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2461,7 +2461,7 @@ Test 046 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /work/noaa/fv3-cam/djovic/RT/NEMSfv3gfs/develop-20200611/INTEL/fv3_gocart_clm_ccpp -working dir = /work/noaa/stmp/junwang/stmp/junwang/FV3_RT/rt_436148/fv3_ccpp_gocart_clm_prod +working dir = /work/noaa/stmp/dheinzel/stmp/dheinzel/FV3_RT/rt_117674/fv3_ccpp_gocart_clm_prod Checking test 047 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2509,5 +2509,5 @@ Test 047 fv3_ccpp_gocart_clm PASS REGRESSION TEST WAS SUCCESSFUL -Fri Jun 12 08:20:31 CDT 2020 -Elapsed time: 00h:34m:24s. Have a nice day! +Fri Jun 19 17:21:49 CDT 2020 +Elapsed time: 01h:12m:04s. Have a nice day! From 9ab3ec02bf58375235de662a7e5a29f0efe475e6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 19 Jun 2020 16:30:37 -0600 Subject: [PATCH 8/9] Regression test logs for wcoss_cray and wcoss_dell_p3 --- tests/Compile_wcoss_cray.log | 811 +++++++++++++----------- tests/Compile_wcoss_dell_p3.log | 545 ++++++++-------- tests/RegressionTests_wcoss_cray.log | 90 +-- tests/RegressionTests_wcoss_dell_p3.log | 92 +-- 4 files changed, 793 insertions(+), 745 deletions(-) diff --git a/tests/Compile_wcoss_cray.log b/tests/Compile_wcoss_cray.log index 63bb750fcb..8ee0f219c4 100644 --- a/tests/Compile_wcoss_cray.log +++ b/tests/Compile_wcoss_cray.log @@ -1,22 +1,22 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' -+ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' -+ export COMPILE_NR=1 -+ COMPILE_NR=1 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_1.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_1.env ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' ++ export COMPILE_NR=10 ++ COMPILE_NR=10 ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_10.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_10.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -58,19 +58,19 @@ ++ WLCLK_dflt=15 + export TEST_NAME=compile + TEST_NAME=compile -+ export TEST_NR=1 -+ TEST_NR=1 -+ export JBNME=compile_1 -+ JBNME=compile_1 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_1 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_1 ++ export TEST_NR=10 ++ TEST_NR=10 ++ export JBNME=compile_10 ++ JBNME=compile_10 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_10 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_10 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_1 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_1 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_10 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_10 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -86,45 +86,49 @@ + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266316 -TEST 1 compile is waiting to enter the queue -1 min. TEST 1 compile is running, Status: RUN -2 min. TEST 1 compile is running, Status: RUN -3 min. TEST 1 compile is running, Status: RUN -4 min. TEST 1 compile is running, Status: RUN -5 min. TEST 1 compile is running, Status: RUN -6 min. TEST 1 compile is running, Status: RUN -7 min. TEST 1 compile is running, Status: RUN -8 min. TEST 1 compile is running, Status: RUN -9 min. TEST 1 compile is running, Status: RUN -10 min. TEST 1 compile is running, Status: RUN -11 min. TEST 1 compile is running, Status: RUN -12 min. TEST 1 compile is running, Status: RUN -13 min. TEST 1 compile is running, Status: RUN -14 min. TEST 1 compile is running, Status: RUN -15 min. TEST 1 compile is finished, Status: DONE -+ elapsed=906 -+ echo 'Elapsed time 906 seconds. Compile 1' -Elapsed time 906 seconds. Compile 1 +Job id 402169 +TEST 10 compile is waiting to enter the queue +1 min. TEST 10 compile is waiting in a queue, Status: PEND +2 min. TEST 10 compile is waiting in a queue, Status: PEND +3 min. TEST 10 compile is waiting in a queue, Status: PEND +4 min. TEST 10 compile is waiting in a queue, Status: PEND +5 min. TEST 10 compile is running, Status: RUN +6 min. TEST 10 compile is running, Status: RUN +7 min. TEST 10 compile is running, Status: RUN +8 min. TEST 10 compile is running, Status: RUN +9 min. TEST 10 compile is running, Status: RUN +10 min. TEST 10 compile is running, Status: RUN +11 min. TEST 10 compile is running, Status: RUN +12 min. TEST 10 compile is running, Status: RUN +13 min. TEST 10 compile is running, Status: RUN +14 min. TEST 10 compile is running, Status: RUN +15 min. TEST 10 compile is running, Status: RUN +16 min. TEST 10 compile is running, Status: RUN +17 min. TEST 10 compile is running, Status: RUN +18 min. TEST 10 compile is running, Status: RUN +19 min. TEST 10 compile is finished, Status: DONE ++ elapsed=1146 ++ echo 'Elapsed time 1146 seconds. Compile 10' +Elapsed time 1146 seconds. Compile 10 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' -+ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' -+ export COMPILE_NR=10 -+ COMPILE_NR=10 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_10.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_10.env ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' ++ export COMPILE_NR=11 ++ COMPILE_NR=11 ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_11.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_11.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -137,10 +141,10 @@ Elapsed time 906 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -166,19 +170,19 @@ Elapsed time 906 seconds. Compile 1 ++ WLCLK_dflt=15 + export TEST_NAME=compile + TEST_NAME=compile -+ export TEST_NR=10 -+ TEST_NR=10 -+ export JBNME=compile_10 -+ JBNME=compile_10 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_10 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_10 ++ export TEST_NR=11 ++ TEST_NR=11 ++ export JBNME=compile_11 ++ JBNME=compile_11 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_11 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_11 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_10 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_10 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_11 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_11 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -194,47 +198,49 @@ Elapsed time 906 seconds. Compile 1 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266315 -TEST 10 compile is waiting to enter the queue -1 min. TEST 10 compile is running, Status: RUN -2 min. TEST 10 compile is running, Status: RUN -3 min. TEST 10 compile is running, Status: RUN -4 min. TEST 10 compile is running, Status: RUN -5 min. TEST 10 compile is running, Status: RUN -6 min. TEST 10 compile is running, Status: RUN -7 min. TEST 10 compile is running, Status: RUN -8 min. TEST 10 compile is running, Status: RUN -9 min. TEST 10 compile is running, Status: RUN -10 min. TEST 10 compile is running, Status: RUN -11 min. TEST 10 compile is running, Status: RUN -12 min. TEST 10 compile is running, Status: RUN -13 min. TEST 10 compile is running, Status: RUN -14 min. TEST 10 compile is running, Status: RUN -15 min. TEST 10 compile is running, Status: RUN -16 min. TEST 10 compile is running, Status: RUN -17 min. TEST 10 compile is finished, Status: DONE -+ elapsed=1026 -+ echo 'Elapsed time 1026 seconds. Compile 10' -Elapsed time 1026 seconds. Compile 10 +Job id 402278 +TEST 11 compile is waiting to enter the queue +1 min. TEST 11 compile is waiting in a queue, Status: PEND +2 min. TEST 11 compile is waiting in a queue, Status: PEND +3 min. TEST 11 compile is waiting in a queue, Status: PEND +4 min. TEST 11 compile is waiting in a queue, Status: PEND +5 min. TEST 11 compile is waiting in a queue, Status: PEND +6 min. TEST 11 compile is running, Status: RUN +7 min. TEST 11 compile is running, Status: RUN +8 min. TEST 11 compile is running, Status: RUN +9 min. TEST 11 compile is running, Status: RUN +10 min. TEST 11 compile is running, Status: RUN +11 min. TEST 11 compile is running, Status: RUN +12 min. TEST 11 compile is running, Status: RUN +13 min. TEST 11 compile is running, Status: RUN +14 min. TEST 11 compile is running, Status: RUN +15 min. TEST 11 compile is running, Status: RUN +16 min. TEST 11 compile is running, Status: RUN +17 min. TEST 11 compile is running, Status: RUN +18 min. TEST 11 compile is running, Status: RUN +19 min. TEST 11 compile is finished, Status: DONE ++ elapsed=1146 ++ echo 'Elapsed time 1146 seconds. Compile 11' +Elapsed time 1146 seconds. Compile 11 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' -+ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' -+ export COMPILE_NR=11 -+ COMPILE_NR=11 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_11.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_11.env ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' ++ MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' ++ export COMPILE_NR=1 ++ COMPILE_NR=1 ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_1.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_1.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -247,10 +253,10 @@ Elapsed time 1026 seconds. Compile 10 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -276,19 +282,19 @@ Elapsed time 1026 seconds. Compile 10 ++ WLCLK_dflt=15 + export TEST_NAME=compile + TEST_NAME=compile -+ export TEST_NR=11 -+ TEST_NR=11 -+ export JBNME=compile_11 -+ JBNME=compile_11 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_11 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_11 ++ export TEST_NR=1 ++ TEST_NR=1 ++ export JBNME=compile_1 ++ JBNME=compile_1 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_1 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_11 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_11 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_1 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_1 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -304,46 +310,51 @@ Elapsed time 1026 seconds. Compile 10 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266412 -TEST 11 compile is waiting to enter the queue -1 min. TEST 11 compile is running, Status: RUN -2 min. TEST 11 compile is running, Status: RUN -3 min. TEST 11 compile is running, Status: RUN -4 min. TEST 11 compile is running, Status: RUN -5 min. TEST 11 compile is running, Status: RUN -6 min. TEST 11 compile is running, Status: RUN -7 min. TEST 11 compile is running, Status: RUN -8 min. TEST 11 compile is running, Status: RUN -9 min. TEST 11 compile is running, Status: RUN -10 min. TEST 11 compile is running, Status: RUN -11 min. TEST 11 compile is running, Status: RUN -12 min. TEST 11 compile is running, Status: RUN -13 min. TEST 11 compile is running, Status: RUN -14 min. TEST 11 compile is running, Status: RUN -15 min. TEST 11 compile is running, Status: RUN -16 min. TEST 11 compile is finished, Status: DONE -+ elapsed=966 -+ echo 'Elapsed time 966 seconds. Compile 11' -Elapsed time 966 seconds. Compile 11 +Job id 402173 +TEST 1 compile is waiting to enter the queue +1 min. TEST 1 compile is waiting in a queue, Status: PEND +2 min. TEST 1 compile is waiting in a queue, Status: PEND +3 min. TEST 1 compile is waiting in a queue, Status: PEND +4 min. TEST 1 compile is waiting in a queue, Status: PEND +5 min. TEST 1 compile is waiting in a queue, Status: PEND +6 min. TEST 1 compile is waiting in a queue, Status: PEND +7 min. TEST 1 compile is waiting in a queue, Status: PEND +8 min. TEST 1 compile is running, Status: RUN +9 min. TEST 1 compile is running, Status: RUN +10 min. TEST 1 compile is running, Status: RUN +11 min. TEST 1 compile is running, Status: RUN +12 min. TEST 1 compile is running, Status: RUN +13 min. TEST 1 compile is running, Status: RUN +14 min. TEST 1 compile is running, Status: RUN +15 min. TEST 1 compile is running, Status: RUN +16 min. TEST 1 compile is running, Status: RUN +17 min. TEST 1 compile is running, Status: RUN +18 min. TEST 1 compile is running, Status: RUN +19 min. TEST 1 compile is running, Status: RUN +20 min. TEST 1 compile is running, Status: RUN +21 min. TEST 1 compile is finished, Status: DONE ++ elapsed=1266 ++ echo 'Elapsed time 1266 seconds. Compile 1' +Elapsed time 1266 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_2.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_2.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_2.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_2.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -356,10 +367,10 @@ Elapsed time 966 seconds. Compile 11 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -389,15 +400,15 @@ Elapsed time 966 seconds. Compile 11 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_2 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_2 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_2 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_2 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_2 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_2 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_2 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -413,11 +424,11 @@ Elapsed time 966 seconds. Compile 11 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266319 +Job id 402166 TEST 2 compile is waiting to enter the queue -1 min. TEST 2 compile is running, Status: RUN -2 min. TEST 2 compile is running, Status: RUN -3 min. TEST 2 compile is running, Status: RUN +1 min. TEST 2 compile is waiting in a queue, Status: PEND +2 min. TEST 2 compile is waiting in a queue, Status: PEND +3 min. TEST 2 compile is waiting in a queue, Status: PEND 4 min. TEST 2 compile is running, Status: RUN 5 min. TEST 2 compile is running, Status: RUN 6 min. TEST 2 compile is running, Status: RUN @@ -431,29 +442,35 @@ TEST 2 compile is waiting to enter the queue 14 min. TEST 2 compile is running, Status: RUN 15 min. TEST 2 compile is running, Status: RUN 16 min. TEST 2 compile is running, Status: RUN -17 min. TEST 2 compile is finished, Status: DONE -+ elapsed=1026 -+ echo 'Elapsed time 1026 seconds. Compile 2' -Elapsed time 1026 seconds. Compile 2 +17 min. TEST 2 compile is running, Status: RUN +18 min. TEST 2 compile is running, Status: RUN +19 min. TEST 2 compile is running, Status: RUN +20 min. TEST 2 compile is running, Status: RUN +21 min. TEST 2 compile is running, Status: RUN +22 min. TEST 2 compile is running, Status: RUN +23 min. TEST 2 compile is finished, Status: DONE ++ elapsed=1386 ++ echo 'Elapsed time 1386 seconds. Compile 2' +Elapsed time 1386 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_3.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_3.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_3.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_3.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -466,10 +483,10 @@ Elapsed time 1026 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -499,15 +516,15 @@ Elapsed time 1026 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_3 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_3 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_3 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_3 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_3 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_3 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_3 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -523,11 +540,11 @@ Elapsed time 1026 seconds. Compile 2 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266314 +Job id 402167 TEST 3 compile is waiting to enter the queue -1 min. TEST 3 compile is running, Status: RUN -2 min. TEST 3 compile is running, Status: RUN -3 min. TEST 3 compile is running, Status: RUN +1 min. TEST 3 compile is waiting in a queue, Status: PEND +2 min. TEST 3 compile is waiting in a queue, Status: PEND +3 min. TEST 3 compile is waiting in a queue, Status: PEND 4 min. TEST 3 compile is running, Status: RUN 5 min. TEST 3 compile is running, Status: RUN 6 min. TEST 3 compile is running, Status: RUN @@ -540,29 +557,31 @@ TEST 3 compile is waiting to enter the queue 13 min. TEST 3 compile is running, Status: RUN 14 min. TEST 3 compile is running, Status: RUN 15 min. TEST 3 compile is running, Status: RUN -16 min. TEST 3 compile is finished, Status: DONE -+ elapsed=966 -+ echo 'Elapsed time 966 seconds. Compile 3' -Elapsed time 966 seconds. Compile 3 +16 min. TEST 3 compile is running, Status: RUN +17 min. TEST 3 compile is running, Status: RUN +18 min. TEST 3 compile is finished, Status: DONE ++ elapsed=1086 ++ echo 'Elapsed time 1086 seconds. Compile 3' +Elapsed time 1086 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_4.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_4.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_4.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_4.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -575,10 +594,10 @@ Elapsed time 966 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -608,15 +627,15 @@ Elapsed time 966 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_4 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_4 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_4 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_4 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_4 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_4 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_4 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -632,13 +651,13 @@ Elapsed time 966 seconds. Compile 3 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266313 +Job id 402171 TEST 4 compile is waiting to enter the queue -1 min. TEST 4 compile is running, Status: RUN -2 min. TEST 4 compile is running, Status: RUN -3 min. TEST 4 compile is running, Status: RUN -4 min. TEST 4 compile is running, Status: RUN -5 min. TEST 4 compile is running, Status: RUN +1 min. TEST 4 compile is waiting in a queue, Status: PEND +2 min. TEST 4 compile is waiting in a queue, Status: PEND +3 min. TEST 4 compile is waiting in a queue, Status: PEND +4 min. TEST 4 compile is waiting in a queue, Status: PEND +5 min. TEST 4 compile is waiting in a queue, Status: PEND 6 min. TEST 4 compile is running, Status: RUN 7 min. TEST 4 compile is running, Status: RUN 8 min. TEST 4 compile is running, Status: RUN @@ -648,29 +667,34 @@ TEST 4 compile is waiting to enter the queue 12 min. TEST 4 compile is running, Status: RUN 13 min. TEST 4 compile is running, Status: RUN 14 min. TEST 4 compile is running, Status: RUN -15 min. TEST 4 compile is finished, Status: DONE -+ elapsed=906 -+ echo 'Elapsed time 906 seconds. Compile 4' -Elapsed time 906 seconds. Compile 4 +15 min. TEST 4 compile is running, Status: RUN +16 min. TEST 4 compile is running, Status: RUN +17 min. TEST 4 compile is running, Status: RUN +18 min. TEST 4 compile is running, Status: RUN +19 min. TEST 4 compile is running, Status: RUN +20 min. TEST 4 compile is finished, Status: DONE ++ elapsed=1206 ++ echo 'Elapsed time 1206 seconds. Compile 4' +Elapsed time 1206 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_5.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_5.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_5.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_5.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -683,10 +707,10 @@ Elapsed time 906 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -716,15 +740,15 @@ Elapsed time 906 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_5 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_5 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_5 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_5 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_5 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_5 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_5 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -740,11 +764,11 @@ Elapsed time 906 seconds. Compile 4 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266311 +Job id 402165 TEST 5 compile is waiting to enter the queue -1 min. TEST 5 compile is running, Status: RUN -2 min. TEST 5 compile is running, Status: RUN -3 min. TEST 5 compile is running, Status: RUN +1 min. TEST 5 compile is waiting in a queue, Status: PEND +2 min. TEST 5 compile is waiting in a queue, Status: PEND +3 min. TEST 5 compile is waiting in a queue, Status: PEND 4 min. TEST 5 compile is running, Status: RUN 5 min. TEST 5 compile is running, Status: RUN 6 min. TEST 5 compile is running, Status: RUN @@ -752,29 +776,30 @@ TEST 5 compile is waiting to enter the queue 8 min. TEST 5 compile is running, Status: RUN 9 min. TEST 5 compile is running, Status: RUN 10 min. TEST 5 compile is running, Status: RUN -11 min. TEST 5 compile is finished, Status: DONE -+ elapsed=666 -+ echo 'Elapsed time 666 seconds. Compile 5' -Elapsed time 666 seconds. Compile 5 +11 min. TEST 5 compile is running, Status: RUN +12 min. TEST 5 compile is finished, Status: DONE ++ elapsed=726 ++ echo 'Elapsed time 726 seconds. Compile 5' +Elapsed time 726 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_6.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_6.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_6.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_6.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -787,10 +812,10 @@ Elapsed time 666 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -820,15 +845,15 @@ Elapsed time 666 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_6 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_6 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_6 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_6 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_6 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_6 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_6 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -844,15 +869,15 @@ Elapsed time 666 seconds. Compile 5 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266317 +Job id 402174 TEST 6 compile is waiting to enter the queue -1 min. TEST 6 compile is running, Status: RUN -2 min. TEST 6 compile is running, Status: RUN -3 min. TEST 6 compile is running, Status: RUN -4 min. TEST 6 compile is running, Status: RUN -5 min. TEST 6 compile is running, Status: RUN -6 min. TEST 6 compile is running, Status: RUN -7 min. TEST 6 compile is running, Status: RUN +1 min. TEST 6 compile is waiting in a queue, Status: PEND +2 min. TEST 6 compile is waiting in a queue, Status: PEND +3 min. TEST 6 compile is waiting in a queue, Status: PEND +4 min. TEST 6 compile is waiting in a queue, Status: PEND +5 min. TEST 6 compile is waiting in a queue, Status: PEND +6 min. TEST 6 compile is waiting in a queue, Status: PEND +7 min. TEST 6 compile is waiting in a queue, Status: PEND 8 min. TEST 6 compile is running, Status: RUN 9 min. TEST 6 compile is running, Status: RUN 10 min. TEST 6 compile is running, Status: RUN @@ -860,29 +885,36 @@ TEST 6 compile is waiting to enter the queue 12 min. TEST 6 compile is running, Status: RUN 13 min. TEST 6 compile is running, Status: RUN 14 min. TEST 6 compile is running, Status: RUN -15 min. TEST 6 compile is finished, Status: DONE -+ elapsed=906 -+ echo 'Elapsed time 906 seconds. Compile 6' -Elapsed time 906 seconds. Compile 6 +15 min. TEST 6 compile is running, Status: RUN +16 min. TEST 6 compile is running, Status: RUN +17 min. TEST 6 compile is running, Status: RUN +18 min. TEST 6 compile is running, Status: RUN +19 min. TEST 6 compile is running, Status: RUN +20 min. TEST 6 compile is running, Status: RUN +21 min. TEST 6 compile is running, Status: RUN +22 min. TEST 6 compile is finished, Status: DONE ++ elapsed=1326 ++ echo 'Elapsed time 1326 seconds. Compile 6' +Elapsed time 1326 seconds. Compile 6 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + export COMPILE_NR=7 + COMPILE_NR=7 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_7.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_7.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_7.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_7.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -895,10 +927,10 @@ Elapsed time 906 seconds. Compile 6 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -928,15 +960,15 @@ Elapsed time 906 seconds. Compile 6 + TEST_NR=7 + export JBNME=compile_7 + JBNME=compile_7 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_7 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_7 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_7 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_7 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_7 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_7 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_7 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_7 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -952,14 +984,14 @@ Elapsed time 906 seconds. Compile 6 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266320 +Job id 402172 TEST 7 compile is waiting to enter the queue -1 min. TEST 7 compile is running, Status: RUN -2 min. TEST 7 compile is running, Status: RUN -3 min. TEST 7 compile is running, Status: RUN -4 min. TEST 7 compile is running, Status: RUN -5 min. TEST 7 compile is running, Status: RUN -6 min. TEST 7 compile is running, Status: RUN +1 min. TEST 7 compile is waiting in a queue, Status: PEND +2 min. TEST 7 compile is waiting in a queue, Status: PEND +3 min. TEST 7 compile is waiting in a queue, Status: PEND +4 min. TEST 7 compile is waiting in a queue, Status: PEND +5 min. TEST 7 compile is waiting in a queue, Status: PEND +6 min. TEST 7 compile is waiting in a queue, Status: PEND 7 min. TEST 7 compile is running, Status: RUN 8 min. TEST 7 compile is running, Status: RUN 9 min. TEST 7 compile is running, Status: RUN @@ -968,29 +1000,34 @@ TEST 7 compile is waiting to enter the queue 12 min. TEST 7 compile is running, Status: RUN 13 min. TEST 7 compile is running, Status: RUN 14 min. TEST 7 compile is running, Status: RUN -15 min. TEST 7 compile is finished, Status: DONE -+ elapsed=906 -+ echo 'Elapsed time 906 seconds. Compile 7' -Elapsed time 906 seconds. Compile 7 +15 min. TEST 7 compile is running, Status: RUN +16 min. TEST 7 compile is running, Status: RUN +17 min. TEST 7 compile is running, Status: RUN +18 min. TEST 7 compile is running, Status: RUN +19 min. TEST 7 compile is running, Status: RUN +20 min. TEST 7 compile is finished, Status: DONE ++ elapsed=1206 ++ echo 'Elapsed time 1206 seconds. Compile 7' +Elapsed time 1206 seconds. Compile 7 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + export COMPILE_NR=8 + COMPILE_NR=8 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_8.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_8.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_8.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_8.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1003,10 +1040,10 @@ Elapsed time 906 seconds. Compile 7 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -1036,15 +1073,15 @@ Elapsed time 906 seconds. Compile 7 + TEST_NR=8 + export JBNME=compile_8 + JBNME=compile_8 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_8 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_8 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_8 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_8 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_8 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_8 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_8 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_8 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1060,13 +1097,13 @@ Elapsed time 906 seconds. Compile 7 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266318 +Job id 402170 TEST 8 compile is waiting to enter the queue -1 min. TEST 8 compile is running, Status: RUN -2 min. TEST 8 compile is running, Status: RUN -3 min. TEST 8 compile is running, Status: RUN -4 min. TEST 8 compile is running, Status: RUN -5 min. TEST 8 compile is running, Status: RUN +1 min. TEST 8 compile is waiting in a queue, Status: PEND +2 min. TEST 8 compile is waiting in a queue, Status: PEND +3 min. TEST 8 compile is waiting in a queue, Status: PEND +4 min. TEST 8 compile is waiting in a queue, Status: PEND +5 min. TEST 8 compile is waiting in a queue, Status: PEND 6 min. TEST 8 compile is running, Status: RUN 7 min. TEST 8 compile is running, Status: RUN 8 min. TEST 8 compile is running, Status: RUN @@ -1077,29 +1114,33 @@ TEST 8 compile is waiting to enter the queue 13 min. TEST 8 compile is running, Status: RUN 14 min. TEST 8 compile is running, Status: RUN 15 min. TEST 8 compile is running, Status: RUN -16 min. TEST 8 compile is finished, Status: DONE -+ elapsed=966 -+ echo 'Elapsed time 966 seconds. Compile 8' -Elapsed time 966 seconds. Compile 8 +16 min. TEST 8 compile is running, Status: RUN +17 min. TEST 8 compile is running, Status: RUN +18 min. TEST 8 compile is running, Status: RUN +19 min. TEST 8 compile is running, Status: RUN +20 min. TEST 8 compile is finished, Status: DONE ++ elapsed=1206 ++ echo 'Elapsed time 1206 seconds. Compile 8' +Elapsed time 1206 seconds. Compile 8 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 -+ RUNDIR_ROOT=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614 ++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 ++ RUNDIR_ROOT=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + export COMPILE_NR=9 + COMPILE_NR=9 -+ cd /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_9.env ]] -+ source /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_9.env ++ cd /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_9.env ]] ++ source /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_9.env ++ export MACHINE_ID=wcoss_cray ++ MACHINE_ID=wcoss_cray -++ export PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1112,10 +1153,10 @@ Elapsed time 966 seconds. Compile 8 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_cray.log -++ export LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray -++ LOG_DIR=/gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_cray +++ export REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ REGRESSIONTEST_LOG=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_cray.log +++ export LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray +++ LOG_DIR=/gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_cray + source default_vars.sh ++ '[' wcoss_cray = wcoss ']' ++ '[' wcoss_cray = wcoss_cray ']' @@ -1145,15 +1186,15 @@ Elapsed time 966 seconds. Compile 8 + TEST_NR=9 + export JBNME=compile_9 + JBNME=compile_9 -+ export RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_9 -+ RUNDIR=/gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_9 ++ export RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_9 ++ RUNDIR=/gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_9 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/hps3/emc/global/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/hps3/emc/meso/save/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_9 -+ cd /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/compile_9 ++ mkdir -p /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_9 ++ cd /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/compile_9 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1169,17 +1210,21 @@ Elapsed time 966 seconds. Compile 8 + set +x ESUB: OMP_NUM_THREADS value changed to 1 ESUB: MEMORY LIMIT rusage set to "rusage[mem=500]" - with rusage set in shared job. -Job id 266312 +Job id 402168 TEST 9 compile is waiting to enter the queue -1 min. TEST 9 compile is running, Status: RUN -2 min. TEST 9 compile is running, Status: RUN -3 min. TEST 9 compile is running, Status: RUN -4 min. TEST 9 compile is running, Status: RUN +1 min. TEST 9 compile is waiting in a queue, Status: PEND +2 min. TEST 9 compile is waiting in a queue, Status: PEND +3 min. TEST 9 compile is waiting in a queue, Status: PEND +4 min. TEST 9 compile is waiting in a queue, Status: PEND 5 min. TEST 9 compile is running, Status: RUN 6 min. TEST 9 compile is running, Status: RUN 7 min. TEST 9 compile is running, Status: RUN 8 min. TEST 9 compile is running, Status: RUN -9 min. TEST 9 compile is finished, Status: DONE -+ elapsed=546 -+ echo 'Elapsed time 546 seconds. Compile 9' -Elapsed time 546 seconds. Compile 9 +9 min. TEST 9 compile is running, Status: RUN +10 min. TEST 9 compile is running, Status: RUN +11 min. TEST 9 compile is running, Status: RUN +12 min. TEST 9 compile is running, Status: RUN +13 min. TEST 9 compile is finished, Status: DONE ++ elapsed=786 ++ echo 'Elapsed time 786 seconds. Compile 9' +Elapsed time 786 seconds. Compile 9 diff --git a/tests/Compile_wcoss_dell_p3.log b/tests/Compile_wcoss_dell_p3.log index 86cf00a5fe..a8addb64be 100644 --- a/tests/Compile_wcoss_dell_p3.log +++ b/tests/Compile_wcoss_dell_p3.log @@ -1,22 +1,22 @@ + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_v16_csawmg' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_v16_csawmg' + export COMPILE_NR=10 + COMPILE_NR=10 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_10.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_10.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_10.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_10.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -29,10 +29,10 @@ ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -63,15 +63,15 @@ + TEST_NR=10 + export JBNME=compile_10 + JBNME=compile_10 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_10 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_10 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_10 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_10 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_10 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_10 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_10 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_10 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -86,7 +86,7 @@ + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411611 +Job id 26890272 TEST 10 compile is waiting to enter the queue 1 min. TEST 10 compile is running, Status: RUN 2 min. TEST 10 compile is running, Status: RUN @@ -111,28 +111,28 @@ TEST 10 compile is waiting to enter the queue 21 min. TEST 10 compile is running, Status: RUN 22 min. TEST 10 compile is running, Status: RUN 23 min. TEST 10 compile is finished, Status: DONE -+ elapsed=1388 -+ echo 'Elapsed time 1388 seconds. Compile 10' -Elapsed time 1388 seconds. Compile 10 ++ elapsed=1387 ++ echo 'Elapsed time 1387 seconds. Compile 10' +Elapsed time 1387 seconds. Compile 10 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=11 + COMPILE_NR=11 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_11.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_11.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_11.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_11.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -145,10 +145,10 @@ Elapsed time 1388 seconds. Compile 10 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -179,15 +179,15 @@ Elapsed time 1388 seconds. Compile 10 + TEST_NR=11 + export JBNME=compile_11 + JBNME=compile_11 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_11 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_11 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_11 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_11 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_11 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_11 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_11 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_11 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -202,7 +202,7 @@ Elapsed time 1388 seconds. Compile 10 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411807 +Job id 26890788 TEST 11 compile is waiting to enter the queue 1 min. TEST 11 compile is running, Status: RUN 2 min. TEST 11 compile is running, Status: RUN @@ -226,30 +226,29 @@ TEST 11 compile is waiting to enter the queue 20 min. TEST 11 compile is running, Status: RUN 21 min. TEST 11 compile is running, Status: RUN 22 min. TEST 11 compile is running, Status: RUN -23 min. TEST 11 compile is running, Status: RUN -24 min. TEST 11 compile is finished, Status: DONE -+ elapsed=1447 -+ echo 'Elapsed time 1447 seconds. Compile 11' -Elapsed time 1447 seconds. Compile 11 +23 min. TEST 11 compile is finished, Status: DONE ++ elapsed=1387 ++ echo 'Elapsed time 1387 seconds. Compile 11' +Elapsed time 1387 seconds. Compile 11 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017' + export COMPILE_NR=1 + COMPILE_NR=1 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_1.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_1.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_1.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_1.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -262,10 +261,10 @@ Elapsed time 1447 seconds. Compile 11 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -296,15 +295,15 @@ Elapsed time 1447 seconds. Compile 11 + TEST_NR=1 + export JBNME=compile_1 + JBNME=compile_1 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_1 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_1 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_1 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_1 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_1 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_1 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_1 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_1 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -319,7 +318,7 @@ Elapsed time 1447 seconds. Compile 11 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411609 +Job id 26890278 TEST 1 compile is waiting to enter the queue 1 min. TEST 1 compile is running, Status: RUN 2 min. TEST 1 compile is running, Status: RUN @@ -344,28 +343,28 @@ TEST 1 compile is waiting to enter the queue 21 min. TEST 1 compile is running, Status: RUN 22 min. TEST 1 compile is running, Status: RUN 23 min. TEST 1 compile is finished, Status: DONE -+ elapsed=1388 -+ echo 'Elapsed time 1388 seconds. Compile 1' -Elapsed time 1388 seconds. Compile 1 ++ elapsed=1387 ++ echo 'Elapsed time 1387 seconds. Compile 1' +Elapsed time 1387 seconds. Compile 1 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_gfdlmp WW3=Y' + export COMPILE_NR=2 + COMPILE_NR=2 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_2.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_2.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_2.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_2.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -378,10 +377,10 @@ Elapsed time 1388 seconds. Compile 1 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -412,15 +411,15 @@ Elapsed time 1388 seconds. Compile 1 + TEST_NR=2 + export JBNME=compile_2 + JBNME=compile_2 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_2 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_2 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_2 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_2 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_2 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_2 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_2 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_2 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -435,7 +434,7 @@ Elapsed time 1388 seconds. Compile 1 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411613 +Job id 26890276 TEST 2 compile is waiting to enter the queue 1 min. TEST 2 compile is running, Status: RUN 2 min. TEST 2 compile is running, Status: RUN @@ -460,29 +459,37 @@ TEST 2 compile is waiting to enter the queue 21 min. TEST 2 compile is running, Status: RUN 22 min. TEST 2 compile is running, Status: RUN 23 min. TEST 2 compile is running, Status: RUN -24 min. TEST 2 compile is finished, Status: DONE -+ elapsed=1448 -+ echo 'Elapsed time 1448 seconds. Compile 2' -Elapsed time 1448 seconds. Compile 2 +24 min. TEST 2 compile is running, Status: RUN +25 min. TEST 2 compile is running, Status: RUN +26 min. TEST 2 compile is running, Status: RUN +27 min. TEST 2 compile is running, Status: RUN +28 min. TEST 2 compile is running, Status: RUN +29 min. TEST 2 compile is running, Status: RUN +30 min. TEST 2 compile is running, Status: RUN +31 min. TEST 2 compile is running, Status: RUN +32 min. TEST 2 compile is finished, Status: DONE ++ elapsed=1928 ++ echo 'Elapsed time 1928 seconds. Compile 2' +Elapsed time 1928 seconds. Compile 2 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y' + export COMPILE_NR=3 + COMPILE_NR=3 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_3.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_3.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_3.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_3.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -495,10 +502,10 @@ Elapsed time 1448 seconds. Compile 2 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -529,15 +536,15 @@ Elapsed time 1448 seconds. Compile 2 + TEST_NR=3 + export JBNME=compile_3 + JBNME=compile_3 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_3 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_3 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_3 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_3 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_3 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_3 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_3 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_3 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -552,7 +559,7 @@ Elapsed time 1448 seconds. Compile 2 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411617 +Job id 26890279 TEST 3 compile is waiting to enter the queue 1 min. TEST 3 compile is running, Status: RUN 2 min. TEST 3 compile is running, Status: RUN @@ -578,28 +585,28 @@ TEST 3 compile is waiting to enter the queue 22 min. TEST 3 compile is running, Status: RUN 23 min. TEST 3 compile is running, Status: RUN 24 min. TEST 3 compile is finished, Status: DONE -+ elapsed=1448 -+ echo 'Elapsed time 1448 seconds. Compile 3' -Elapsed time 1448 seconds. Compile 3 ++ elapsed=1447 ++ echo 'Elapsed time 1447 seconds. Compile 3' +Elapsed time 1447 seconds. Compile 3 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y' + export COMPILE_NR=4 + COMPILE_NR=4 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_4.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_4.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_4.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_4.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -612,10 +619,10 @@ Elapsed time 1448 seconds. Compile 3 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -646,15 +653,15 @@ Elapsed time 1448 seconds. Compile 3 + TEST_NR=4 + export JBNME=compile_4 + JBNME=compile_4 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_4 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_4 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_4 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_4 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_4 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_4 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_4 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_4 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -669,7 +676,7 @@ Elapsed time 1448 seconds. Compile 3 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411615 +Job id 26890275 TEST 4 compile is waiting to enter the queue 1 min. TEST 4 compile is running, Status: RUN 2 min. TEST 4 compile is running, Status: RUN @@ -693,30 +700,29 @@ TEST 4 compile is waiting to enter the queue 20 min. TEST 4 compile is running, Status: RUN 21 min. TEST 4 compile is running, Status: RUN 22 min. TEST 4 compile is running, Status: RUN -23 min. TEST 4 compile is running, Status: RUN -24 min. TEST 4 compile is finished, Status: DONE -+ elapsed=1448 -+ echo 'Elapsed time 1448 seconds. Compile 4' -Elapsed time 1448 seconds. Compile 4 +23 min. TEST 4 compile is finished, Status: DONE ++ elapsed=1387 ++ echo 'Elapsed time 1387 seconds. Compile 4' +Elapsed time 1387 seconds. Compile 4 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017,FV3_GFS_2017_stretched 32BIT=Y DEBUG=Y' + export COMPILE_NR=5 + COMPILE_NR=5 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_5.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_5.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_5.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_5.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -729,10 +735,10 @@ Elapsed time 1448 seconds. Compile 4 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -763,15 +769,15 @@ Elapsed time 1448 seconds. Compile 4 + TEST_NR=5 + export JBNME=compile_5 + JBNME=compile_5 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_5 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_5 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_5 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_5 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_5 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_5 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_5 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_5 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -786,7 +792,7 @@ Elapsed time 1448 seconds. Compile 4 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411614 +Job id 26890273 TEST 5 compile is waiting to enter the queue 1 min. TEST 5 compile is running, Status: RUN 2 min. TEST 5 compile is running, Status: RUN @@ -794,30 +800,29 @@ TEST 5 compile is waiting to enter the queue 4 min. TEST 5 compile is running, Status: RUN 5 min. TEST 5 compile is running, Status: RUN 6 min. TEST 5 compile is running, Status: RUN -7 min. TEST 5 compile is running, Status: RUN -8 min. TEST 5 compile is finished, Status: DONE -+ elapsed=487 -+ echo 'Elapsed time 487 seconds. Compile 5' -Elapsed time 487 seconds. Compile 5 +7 min. TEST 5 compile is finished, Status: DONE ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 5' +Elapsed time 426 seconds. Compile 5 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp' + export COMPILE_NR=6 + COMPILE_NR=6 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_6.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_6.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_6.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_6.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -830,10 +835,10 @@ Elapsed time 487 seconds. Compile 5 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -864,15 +869,15 @@ Elapsed time 487 seconds. Compile 5 + TEST_NR=6 + export JBNME=compile_6 + JBNME=compile_6 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_6 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_6 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_6 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_6 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_6 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_6 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_6 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_6 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -887,7 +892,7 @@ Elapsed time 487 seconds. Compile 5 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411616 +Job id 26890274 TEST 6 compile is waiting to enter the queue 1 min. TEST 6 compile is running, Status: RUN 2 min. TEST 6 compile is running, Status: RUN @@ -911,30 +916,29 @@ TEST 6 compile is waiting to enter the queue 20 min. TEST 6 compile is running, Status: RUN 21 min. TEST 6 compile is running, Status: RUN 22 min. TEST 6 compile is running, Status: RUN -23 min. TEST 6 compile is running, Status: RUN -24 min. TEST 6 compile is finished, Status: DONE -+ elapsed=1448 -+ echo 'Elapsed time 1448 seconds. Compile 6' -Elapsed time 1448 seconds. Compile 6 +23 min. TEST 6 compile is finished, Status: DONE ++ elapsed=1387 ++ echo 'Elapsed time 1387 seconds. Compile 6' +Elapsed time 1387 seconds. Compile 6 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq' + export COMPILE_NR=7 + COMPILE_NR=7 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_7.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_7.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_7.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_7.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -947,10 +951,10 @@ Elapsed time 1448 seconds. Compile 6 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -981,15 +985,15 @@ Elapsed time 1448 seconds. Compile 6 + TEST_NR=7 + export JBNME=compile_7 + JBNME=compile_7 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_7 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_7 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_7 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_7 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_7 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_7 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_7 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_7 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1004,7 +1008,7 @@ Elapsed time 1448 seconds. Compile 6 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411612 +Job id 26890277 TEST 7 compile is waiting to enter the queue 1 min. TEST 7 compile is running, Status: RUN 2 min. TEST 7 compile is running, Status: RUN @@ -1037,23 +1041,23 @@ TEST 7 compile is waiting to enter the queue Elapsed time 1568 seconds. Compile 7 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y' + export COMPILE_NR=8 + COMPILE_NR=8 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_8.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_8.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_8.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_8.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1066,10 +1070,10 @@ Elapsed time 1568 seconds. Compile 7 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -1100,15 +1104,15 @@ Elapsed time 1568 seconds. Compile 7 + TEST_NR=8 + export JBNME=compile_8 + JBNME=compile_8 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_8 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_8 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_8 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_8 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_8 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_8 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_8 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_8 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1122,9 +1126,8 @@ Elapsed time 1568 seconds. Compile 7 + '[' -o xtrace ']' + set_x='set -x' + set +x -LSF daemon (LIM) not responding ... still trying ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411631 +Job id 26890280 TEST 8 compile is waiting to enter the queue 1 min. TEST 8 compile is running, Status: RUN 2 min. TEST 8 compile is running, Status: RUN @@ -1153,28 +1156,28 @@ TEST 8 compile is waiting to enter the queue 25 min. TEST 8 compile is running, Status: RUN 26 min. TEST 8 compile is running, Status: RUN 27 min. TEST 8 compile is finished, Status: DONE -+ elapsed=1678 -+ echo 'Elapsed time 1678 seconds. Compile 8' -Elapsed time 1678 seconds. Compile 8 ++ elapsed=1628 ++ echo 'Elapsed time 1628 seconds. Compile 8' +Elapsed time 1628 seconds. Compile 8 + SECONDS=0 + [[ 4 != 4 ]] -+ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 -+ RUNDIR_ROOT=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207 ++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ export RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 ++ RUNDIR_ROOT=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302 + export 'MAKE_OPT=CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + MAKE_OPT='CCPP=Y SUITES=FV3_GSD_v0,FV3_GFS_v15_thompson 32BIT=Y DEBUG=Y' + export COMPILE_NR=9 + COMPILE_NR=9 -+ cd /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -+ [[ -e /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_9.env ]] -+ source /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_9.env ++ cd /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests ++ [[ -e /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_9.env ]] ++ source /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_9.env ++ export MACHINE_ID=wcoss_dell_p3 ++ MACHINE_ID=wcoss_dell_p3 -++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests -++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model -++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model +++ export PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ PATHRT=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests +++ export PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model +++ PATHTR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model ++ export SCHEDULER=lsf ++ SCHEDULER=lsf ++ export ACCNR=GFS-DEV @@ -1187,10 +1190,10 @@ Elapsed time 1678 seconds. Compile 8 ++ ROCOTO=false ++ export ECFLOW=true ++ ECFLOW=true -++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log -++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 -++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/log_wcoss_dell_p3 +++ export REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ REGRESSIONTEST_LOG=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/RegressionTests_wcoss_dell_p3.log +++ export LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 +++ LOG_DIR=/gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/log_wcoss_dell_p3 + source default_vars.sh ++ '[' wcoss_dell_p3 = wcoss ']' ++ '[' wcoss_dell_p3 = wcoss_cray ']' @@ -1221,15 +1224,15 @@ Elapsed time 1678 seconds. Compile 8 + TEST_NR=9 + export JBNME=compile_9 + JBNME=compile_9 -+ export RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_9 -+ RUNDIR=/gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_9 ++ export RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_9 ++ RUNDIR=/gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_9 + source rt_utils.sh ++ set -eu -++ [[ /gpfs/dell2/emc/modeling/noscrub/Jun.Wang/nems/vlab/20200611/ufs-weather-model/tests/run_compile.sh = \.\/\r\t\_\u\t\i\l\s\.\s\h ]] +++ [[ /gpfs/dell2/emc/modeling/noscrub/Dusan.Jovic/ufs/pr151/ufs-weather-model/tests/run_compile.sh = \r\t\_\u\t\i\l\s\.\s\h ]] ++ UNIT_TEST=false + source atparse.bash -+ mkdir -p /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_9 -+ cd /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/compile_9 ++ mkdir -p /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_9 ++ cd /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/compile_9 + [[ lsf = \s\l\u\r\m ]] + [[ lsf = \l\s\f ]] + atparse @@ -1244,7 +1247,7 @@ Elapsed time 1678 seconds. Compile 8 + set_x='set -x' + set +x ESUB: MEMORY LIMIT RLIMIT_RSS set to 8192 - with rusage set in shared job. -Job id 26411610 +Job id 26890281 TEST 9 compile is waiting to enter the queue 1 min. TEST 9 compile is running, Status: RUN 2 min. TEST 9 compile is running, Status: RUN @@ -1253,6 +1256,6 @@ TEST 9 compile is waiting to enter the queue 5 min. TEST 9 compile is running, Status: RUN 6 min. TEST 9 compile is running, Status: RUN 7 min. TEST 9 compile is finished, Status: DONE -+ elapsed=427 -+ echo 'Elapsed time 427 seconds. Compile 9' -Elapsed time 427 seconds. Compile 9 ++ elapsed=426 ++ echo 'Elapsed time 426 seconds. Compile 9' +Elapsed time 426 seconds. Compile 9 diff --git a/tests/RegressionTests_wcoss_cray.log b/tests/RegressionTests_wcoss_cray.log index 4e841d7b4b..a8363635ed 100644 --- a/tests/RegressionTests_wcoss_cray.log +++ b/tests/RegressionTests_wcoss_cray.log @@ -1,9 +1,9 @@ -Fri Jun 12 01:33:44 UTC 2020 +Fri Jun 19 21:11:16 UTC 2020 Start Regression test baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_control_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_decomp_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_2threads_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_restart_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_restart_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_read_inc_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_read_inc_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGauss_netcdf_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGlatlon_netcdf_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGauss_nemsio_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_stochy_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_stochy_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_iau_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_iau_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_lheatstrg_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_lheatstrg_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_lheatstrg_prod Checking test 013 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -767,7 +767,7 @@ Test 013 fv3_ccpp_lheatstrg PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfdlmprad_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfdlmprad_prod Checking test 014 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -816,7 +816,7 @@ Test 014 fv3_ccpp_gfdlmprad PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_atmwav_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -865,7 +865,7 @@ Test 015 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_32bit_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_control_32bit_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_control_32bit_prod Checking test 016 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -933,7 +933,7 @@ Test 016 fv3_ccpp_control_32bit PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_stretched_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_stretched_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_stretched_prod Checking test 017 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -989,7 +989,7 @@ Test 017 fv3_ccpp_stretched PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_stretched_nest_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_stretched_nest_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_stretched_nest_prod Checking test 018 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1056,7 +1056,7 @@ Test 018 fv3_ccpp_stretched_nest PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_regional_control_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_regional_control_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_regional_control_prod Checking test 019 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1067,7 +1067,7 @@ Test 019 fv3_ccpp_regional_control PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_regional_restart_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_regional_restart_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_regional_restart_prod Checking test 020 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1076,7 +1076,7 @@ Test 020 fv3_ccpp_regional_restart PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_regional_quilt_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_regional_quilt_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_regional_quilt_prod Checking test 021 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1087,19 +1087,19 @@ Test 021 fv3_ccpp_regional_quilt PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_control_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_control_debug_prod Checking test 022 fv3_ccpp_control_debug results .... Test 022 fv3_ccpp_control_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_stretched_nest_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_stretched_nest_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_stretched_nest_debug_prod Checking test 023 fv3_ccpp_stretched_nest_debug results .... Test 023 fv3_ccpp_stretched_nest_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmp_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfdlmp_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfdlmp_prod Checking test 024 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1147,7 +1147,7 @@ Test 024 fv3_ccpp_gfdlmp PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_gwd_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfdlmprad_gwd_prod Checking test 025 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1195,7 +1195,7 @@ Test 025 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_noahmp_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 026 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1243,7 +1243,7 @@ Test 026 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmp_hwrfsas_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfdlmp_hwrfsas_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 027 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1291,7 +1291,7 @@ Test 027 fv3_ccpp_gfdlmp_hwrfsas PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_csawmg_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_csawmg_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_csawmg_prod Checking test 028 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1339,7 +1339,7 @@ Test 028 fv3_ccpp_csawmg PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_satmedmf_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_satmedmf_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_satmedmf_prod Checking test 029 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1387,7 +1387,7 @@ Test 029 fv3_ccpp_satmedmf PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_satmedmfq_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_satmedmfq_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_satmedmfq_prod Checking test 030 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1435,7 +1435,7 @@ Test 030 fv3_ccpp_satmedmfq PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmp_32bit_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfdlmp_32bit_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfdlmp_32bit_prod Checking test 031 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1483,7 +1483,7 @@ Test 031 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_32bit_post_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 032 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1535,7 +1535,7 @@ Test 032 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_cpt_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_cpt_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_cpt_prod Checking test 033 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1589,7 +1589,7 @@ Test 033 fv3_ccpp_cpt PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gsd_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gsd_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gsd_prod Checking test 034 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1681,7 +1681,7 @@ Test 034 fv3_ccpp_gsd PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_thompson_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_thompson_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_thompson_prod Checking test 035 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1749,7 +1749,7 @@ Test 035 fv3_ccpp_thompson PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_thompson_no_aero_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_thompson_no_aero_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_thompson_no_aero_prod Checking test 036 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1817,7 +1817,7 @@ Test 036 fv3_ccpp_thompson_no_aero PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gsd_debug_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gsd_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gsd_debug_prod Checking test 037 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1885,7 +1885,7 @@ Test 037 fv3_ccpp_gsd_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_thompson_debug_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_thompson_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_thompson_debug_prod Checking test 038 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1953,7 +1953,7 @@ Test 038 fv3_ccpp_thompson_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_thompson_no_aero_debug_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_thompson_no_aero_debug_prod Checking test 039 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2021,7 +2021,7 @@ Test 039 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfsv16_csawmg_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfsv16_csawmg_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfsv16_csawmg_prod Checking test 040 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2069,7 +2069,7 @@ Test 040 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfsv16_csawmgt_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gfsv16_csawmgt_prod Checking test 041 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2117,7 +2117,7 @@ Test 041 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gocart_clm_ccpp -working dir = /gpfs/hps3/stmp/Jun.Wang/FV3_RT/rt_2614/fv3_ccpp_gocart_clm_prod +working dir = /gpfs/hps3/stmp/Dusan.Jovic/FV3_RT/rt_32387/fv3_ccpp_gocart_clm_prod Checking test 042 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2165,5 +2165,5 @@ Test 042 fv3_ccpp_gocart_clm PASS REGRESSION TEST WAS SUCCESSFUL -Fri Jun 12 02:09:50 UTC 2020 -Elapsed time: 00h:36m:07s. Have a nice day! +Fri Jun 19 22:21:25 UTC 2020 +Elapsed time: 01h:10m:10s. Have a nice day! diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log index 230617ec8c..6114136fd4 100644 --- a/tests/RegressionTests_wcoss_dell_p3.log +++ b/tests/RegressionTests_wcoss_dell_p3.log @@ -1,9 +1,9 @@ -Fri Jun 12 01:31:55 UTC 2020 +Fri Jun 19 21:10:59 UTC 2020 Start Regression test baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_control_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_control_prod Checking test 001 fv3_ccpp_control results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -71,7 +71,7 @@ Test 001 fv3_ccpp_control PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_decomp_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_decomp_prod Checking test 002 fv3_ccpp_decomp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -139,7 +139,7 @@ Test 002 fv3_ccpp_decomp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_2threads_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_2threads_prod Checking test 003 fv3_ccpp_2threads results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -207,7 +207,7 @@ Test 003 fv3_ccpp_2threads PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_restart_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_restart_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_restart_prod Checking test 004 fv3_ccpp_restart results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -275,7 +275,7 @@ Test 004 fv3_ccpp_restart PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_read_inc_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_read_inc_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_read_inc_prod Checking test 005 fv3_ccpp_read_inc results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -343,7 +343,7 @@ Test 005 fv3_ccpp_read_inc PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGauss_netcdf_esmf_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_wrtGauss_netcdf_esmf_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_wrtGauss_netcdf_esmf_prod Checking test 006 fv3_ccpp_wrtGauss_netcdf_esmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -391,7 +391,7 @@ Test 006 fv3_ccpp_wrtGauss_netcdf_esmf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGauss_netcdf_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_wrtGauss_netcdf_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_wrtGauss_netcdf_prod Checking test 007 fv3_ccpp_wrtGauss_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -439,7 +439,7 @@ Test 007 fv3_ccpp_wrtGauss_netcdf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGlatlon_netcdf_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_wrtGlatlon_netcdf_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_wrtGlatlon_netcdf_prod Checking test 008 fv3_ccpp_wrtGlatlon_netcdf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -487,7 +487,7 @@ Test 008 fv3_ccpp_wrtGlatlon_netcdf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGauss_nemsio_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_wrtGauss_nemsio_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_wrtGauss_nemsio_prod Checking test 009 fv3_ccpp_wrtGauss_nemsio results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -535,7 +535,7 @@ Test 009 fv3_ccpp_wrtGauss_nemsio PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_wrtGauss_nemsio_c192_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_wrtGauss_nemsio_c192_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_wrtGauss_nemsio_c192_prod Checking test 010 fv3_ccpp_wrtGauss_nemsio_c192 results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -583,7 +583,7 @@ Test 010 fv3_ccpp_wrtGauss_nemsio_c192 PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_stochy_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_stochy_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_stochy_prod Checking test 011 fv3_ccpp_stochy results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -651,7 +651,7 @@ Test 011 fv3_ccpp_stochy PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_iau_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_iau_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_iau_prod Checking test 012 fv3_ccpp_iau results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -719,7 +719,7 @@ Test 012 fv3_ccpp_iau PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_lheatstrg_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_lheatstrg_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_lheatstrg_prod Checking test 013 fv3_ccpp_lheatstrg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -767,7 +767,7 @@ Test 013 fv3_ccpp_lheatstrg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfdlmprad_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfdlmprad_prod Checking test 014 fv3_ccpp_gfdlmprad results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -816,7 +816,7 @@ Test 014 fv3_ccpp_gfdlmprad PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_atmwav_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfdlmprad_atmwav_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfdlmprad_atmwav_prod Checking test 015 fv3_ccpp_gfdlmprad_atmwav results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -865,7 +865,7 @@ Test 015 fv3_ccpp_gfdlmprad_atmwav PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_32bit_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_control_32bit_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_control_32bit_prod Checking test 016 fv3_ccpp_control_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -933,7 +933,7 @@ Test 016 fv3_ccpp_control_32bit PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_stretched_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_stretched_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_stretched_prod Checking test 017 fv3_ccpp_stretched results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -989,7 +989,7 @@ Test 017 fv3_ccpp_stretched PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_stretched_nest_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_stretched_nest_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_stretched_nest_prod Checking test 018 fv3_ccpp_stretched_nest results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1056,7 +1056,7 @@ Test 018 fv3_ccpp_stretched_nest PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_regional_control_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_regional_control_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_regional_control_prod Checking test 019 fv3_ccpp_regional_control results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1067,7 +1067,7 @@ Test 019 fv3_ccpp_regional_control PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_regional_restart_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_regional_restart_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_regional_restart_prod Checking test 020 fv3_ccpp_regional_restart results .... Comparing atmos_4xdaily.nc .........OK Comparing fv3_history2d.nc .........OK @@ -1076,7 +1076,7 @@ Test 020 fv3_ccpp_regional_restart PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_regional_quilt_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_regional_quilt_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_regional_quilt_prod Checking test 021 fv3_ccpp_regional_quilt results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1087,7 +1087,7 @@ Test 021 fv3_ccpp_regional_quilt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_regional_c768_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_regional_c768_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_regional_c768_prod Checking test 022 fv3_ccpp_regional_c768 results .... Comparing atmos_4xdaily.nc .........OK Comparing dynf000.nc .........OK @@ -1098,19 +1098,19 @@ Test 022 fv3_ccpp_regional_c768 PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_control_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_control_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_control_debug_prod Checking test 023 fv3_ccpp_control_debug results .... Test 023 fv3_ccpp_control_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_stretched_nest_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_stretched_nest_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_stretched_nest_debug_prod Checking test 024 fv3_ccpp_stretched_nest_debug results .... Test 024 fv3_ccpp_stretched_nest_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmp_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfdlmp_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfdlmp_prod Checking test 025 fv3_ccpp_gfdlmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1158,7 +1158,7 @@ Test 025 fv3_ccpp_gfdlmp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_gwd_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfdlmprad_gwd_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfdlmprad_gwd_prod Checking test 026 fv3_ccpp_gfdlmprad_gwd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1206,7 +1206,7 @@ Test 026 fv3_ccpp_gfdlmprad_gwd PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_noahmp_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfdlmprad_noahmp_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfdlmprad_noahmp_prod Checking test 027 fv3_ccpp_gfdlmprad_noahmp results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1254,7 +1254,7 @@ Test 027 fv3_ccpp_gfdlmprad_noahmp PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmp_hwrfsas_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfdlmp_hwrfsas_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfdlmp_hwrfsas_prod Checking test 028 fv3_ccpp_gfdlmp_hwrfsas results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1302,7 +1302,7 @@ Test 028 fv3_ccpp_gfdlmp_hwrfsas PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_csawmg_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_csawmg_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_csawmg_prod Checking test 029 fv3_ccpp_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1350,7 +1350,7 @@ Test 029 fv3_ccpp_csawmg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_satmedmf_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_satmedmf_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_satmedmf_prod Checking test 030 fv3_ccpp_satmedmf results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1398,7 +1398,7 @@ Test 030 fv3_ccpp_satmedmf PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_satmedmfq_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_satmedmfq_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_satmedmfq_prod Checking test 031 fv3_ccpp_satmedmfq results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1446,7 +1446,7 @@ Test 031 fv3_ccpp_satmedmfq PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmp_32bit_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfdlmp_32bit_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfdlmp_32bit_prod Checking test 032 fv3_ccpp_gfdlmp_32bit results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1494,7 +1494,7 @@ Test 032 fv3_ccpp_gfdlmp_32bit PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfdlmprad_32bit_post_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfdlmprad_32bit_post_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfdlmprad_32bit_post_prod Checking test 033 fv3_ccpp_gfdlmprad_32bit_post results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1546,7 +1546,7 @@ Test 033 fv3_ccpp_gfdlmprad_32bit_post PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_cpt_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_cpt_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_cpt_prod Checking test 034 fv3_ccpp_cpt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1600,7 +1600,7 @@ Test 034 fv3_ccpp_cpt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gsd_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gsd_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gsd_prod Checking test 035 fv3_ccpp_gsd results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1692,7 +1692,7 @@ Test 035 fv3_ccpp_gsd PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_thompson_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_thompson_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_thompson_prod Checking test 036 fv3_ccpp_thompson results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1760,7 +1760,7 @@ Test 036 fv3_ccpp_thompson PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_thompson_no_aero_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_thompson_no_aero_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_thompson_no_aero_prod Checking test 037 fv3_ccpp_thompson_no_aero results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1828,7 +1828,7 @@ Test 037 fv3_ccpp_thompson_no_aero PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gsd_debug_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gsd_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gsd_debug_prod Checking test 038 fv3_ccpp_gsd_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1896,7 +1896,7 @@ Test 038 fv3_ccpp_gsd_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_thompson_debug_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_thompson_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_thompson_debug_prod Checking test 039 fv3_ccpp_thompson_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -1964,7 +1964,7 @@ Test 039 fv3_ccpp_thompson_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_thompson_no_aero_debug_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_thompson_no_aero_debug_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_thompson_no_aero_debug_prod Checking test 040 fv3_ccpp_thompson_no_aero_debug results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2032,7 +2032,7 @@ Test 040 fv3_ccpp_thompson_no_aero_debug PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfsv16_csawmg_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfsv16_csawmg_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfsv16_csawmg_prod Checking test 041 fv3_ccpp_gfsv16_csawmg results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2080,7 +2080,7 @@ Test 041 fv3_ccpp_gfsv16_csawmg PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gfsv16_csawmgt_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gfsv16_csawmgt_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gfsv16_csawmgt_prod Checking test 042 fv3_ccpp_gfsv16_csawmgt results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2128,7 +2128,7 @@ Test 042 fv3_ccpp_gfsv16_csawmgt PASS baseline dir = /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200611/fv3_gocart_clm_ccpp -working dir = /gpfs/dell2/ptmp/Jun.Wang/FV3_RT/rt_8207/fv3_ccpp_gocart_clm_prod +working dir = /gpfs/dell2/ptmp/Dusan.Jovic/FV3_RT/rt_53302/fv3_ccpp_gocart_clm_prod Checking test 043 fv3_ccpp_gocart_clm results .... Comparing atmos_4xdaily.tile1.nc .........OK Comparing atmos_4xdaily.tile2.nc .........OK @@ -2176,5 +2176,5 @@ Test 043 fv3_ccpp_gocart_clm PASS REGRESSION TEST WAS SUCCESSFUL -Fri Jun 12 03:29:34 UTC 2020 -Elapsed time: 01h:57m:42s. Have a nice day! +Fri Jun 19 22:08:41 UTC 2020 +Elapsed time: 00h:57m:45s. Have a nice day! From 660bfc54002453b62cc756384ec5701d3dfea36b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 22 Jun 2020 08:05:02 -0600 Subject: [PATCH 9/9] Revert change to .gitmodules and update submodule pointer for fv3aatam --- .gitmodules | 6 ++---- FV3 | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4ded960b48..a6770b7e98 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,7 @@ [submodule "FV3"] path = FV3 - #url = https://github.com/NOAA-EMC/fv3atm - #branch = develop - url = https://github.com/climbfuji/fv3atm - branch = dycore_submodule_update_gnu + url = https://github.com/NOAA-EMC/fv3atm + branch = develop [submodule "NEMS"] path = NEMS url = https://github.com/NOAA-EMC/NEMS diff --git a/FV3 b/FV3 index bd1b738c0f..ea71b79011 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit bd1b738c0f35eeea46b9487f443e7df0d4689e12 +Subproject commit ea71b79011ddbc1e022e203b7e15d6d1c90d314f