Regression test updates: global_4dvar bug fix, oom fix, enhance error…

… checking (NOAA-EMC#532)

regression/global_4dvar.sh

@@ -98,7 +98,7 @@ SINGLEOB=""
 #   bftab_sst= bufr table for sst ONLY needed for sst retrieval (retrieval=.true.)
 #   aeroinfo = text file with information about assimilation of aerosol data
 
-anavinfo=$fixgsi/global_anavinfo.l${LEVS}.txt
+anavinfo=$fixgsi/global_anavinfo_qlqi.l${LEVS}.txt
 berror=$fixgsi/Big_Endian/global_berror.l${LEVS}y${NLAT}.f77
 locinfo=$fixgsi/global_hybens_info.l${LEVS}.txt
 satinfo=$fixgsi/global_satinfo.txt
@@ -311,9 +311,9 @@ $gsi_namelist
 EOF
 cp gsiparm.anl gsiparm.anl.obsvr
 
-echo "run gsi now"
+echo "run gsi observer"
 eval "$APRUN $tmpdir/gsi.x < gsiparm.anl > stdout.obsvr 2>&1"
-rc=$?
+ra=$?
 
 # Run gsi identity model 4dvar under Parallel Operating Environment (poe) on NCEP IBM
 rm -f siganl sfcanl.gsi satbias_out fort.2*
@@ -331,8 +331,8 @@ cat <<EOF > gsiparm.anl
 $gsi_namelist
 EOF
 
-echo "run gsi now"
+echo "run gsi 4dvar"
 eval "$APRUN $tmpdir/gsi.x < gsiparm.anl > stdout 2>&1"
-rc=$?
-
+rb=$?
+rc=$((ra+rb))
 exit $rc

regression/regression_driver.sh

@@ -47,7 +47,6 @@ for jn in `seq ${RSTART} ${REND}`; do
    $scripts/regression_wait.sh ${job[$jn]} ${rcname} $check_resource
    rc=$?
    if [ $rc -ne 0 ]; then
-     rm -f ${rcname}
      exit 1
    fi
    done

regression/regression_param.sh

@@ -150,8 +150,8 @@ case $regtest in
             topts[1]="0:15:00" ; popts[1]="20/1/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="20/2/"  ; ropts[2]="/1"
         elif [[ "$machine" = "Orion" ]]; then
-            topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
-            topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
+            topts[1]="0:15:00" ; popts[1]="20/1/"  ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="20/2/"  ; ropts[2]="/1"
         elif [[ "$machine" = "Jet" ]]; then
             topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"

regression/regression_test.sh

@@ -576,11 +576,20 @@ mkdir -p $vfydir
 
 $ncp $output                        $vfydir/
 
+# Final check for any failed tests
+count=$(grep -i "fail" $output |wc -l)
+if [ $count -gt 0 ]; then
+    (( failed_test = $failed_test + $count ))
+fi
+
+# Remove job log files is no failures detected
 cd $scripts
-rm -f ${exp1}.out
-rm -f ${exp2}.out
-rm -f ${exp3}.out
-rm -f ${exp2_scale}.out
+if [ $count -eq 0 ]; then
+    rm -f ${exp1}.out
+    rm -f ${exp2}.out
+    rm -f ${exp3}.out
+    rm -f ${exp2_scale}.out
+fi
 
 if [[ "$clean" = ".true." ]]; then
    rm -rf $savdir

regression/regression_test_enkf.sh

@@ -34,7 +34,7 @@ maxtime=1200
 maxmem=${maxmem:-3400000} # set in regression_param
 maxmem=$((${memnode:-64}*1024*1024))
 
-# Copy stdout and sanl files 
+# Copy stdout and incr files 
 # from $savdir to $tmpdir
 list="$exp1 $exp2 $exp3"
 for exp in $list; do
@@ -43,7 +43,7 @@ for exp in $list; do
    imem=1
    while [[ $imem -le $nmem ]]; do
       member="_mem"`printf %03i $imem`
-      $ncp $savdir/$exp/sanl_${global_adate}_fhr06$member $tmpdir/sanl$member.$exp
+      $ncp $savdir/$exp/incr_${global_adate}_fhr06$member $tmpdir/incr$member.$exp
       (( imem = $imem + 1 ))
    done
 done
@@ -282,10 +282,13 @@ nmem=10
 imem=1
 while [[ $imem -le $nmem ]]; do
    member="_mem"`printf %03i $imem`
-   if ! cmp -s sanl$member.${exp1} sanl$member.${exp2} 
-then
-   echo 'sanl'$member'.'${exp1}' sanl'$member'.'${exp2}' are NOT identical'
-fi
+   ncdump incr$member.${exp1} > incr$member.${exp1}.out
+   ncdump incr$member.${exp2} > incr$member.${exp2}.out
+   if [ ! diff incr$member.${exp1}.out incr$member.${exp2}.out ]; then
+       echo 'incr'$member'.'${exp1}' incr'$member'.'${exp2}' are NOT identical'
+   else
+       rm -f incr$member.${exp1}.out incr$member.${exp2}.out
+   fi
    (( imem = $imem + 1 ))
 done
 echo
@@ -379,11 +382,14 @@ else
    imem=1
    while [[ $imem -le $nmem ]]; do
       member="_mem"`printf %03i $imem`
-      if ! cmp -s sanl$member.${exp1} sanl$member.${exp3}
-      then
-      echo 'sanl'$member'.'${exp1}' sanl'$member'.'${exp3}' are NOT identical'
+      ncdump incr$member.${exp1} > incr$member.${exp1}.out
+      ncdump incr$member.${exp3} > incr$member.${exp3}.out
+      if [ ! diff incr$member.${exp1}.out incr$member.${exp3}.out ]; then
+          echo 'incr'$member'.'${exp1}' incr'$member'.'${exp3}' are NOT identical'
+      else
+          rm -f incr$member.${exp1}.out incr$member.${exp3}.out
       fi
-   (( imem = $imem + 1 ))
+      (( imem = $imem + 1 ))
    done
    echo
 } >> $output
@@ -411,11 +417,20 @@ mkdir -p $vfydir
 
 $ncp $output                        $vfydir/
 
+# Final check for any failed tests
+count=$(grep -i "fail" $output |wc -l)
+if [ $count -gt 0 ]; then
+    (( failed_test = $failed_test + $count ))
+fi
+
+# Remove job log files is no failures detected
 cd $scripts
-rm -f ${exp1}.out
-rm -f ${exp2}.out
-rm -f ${exp3}.out
-rm -f ${exp2_scale}.out
+if [ $count -eq 0 ]; then
+    rm -f ${exp1}.out
+    rm -f ${exp2}.out
+    rm -f ${exp3}.out
+    rm -f ${exp2_scale}.out
+fi
 
 if [[ "$clean" = ".true." ]]; then
    rm -rf $savdir