Refactor bit-for-bit diagnostics (CICE-Consortium#300)

* implement bfb for mpi comm global sums, fix compile issue with cpps, remove unused cpps * add set_nml.reprosum * update global reductions for serial mode to leverage bfbflag in the same way as mpi mode * update and fix bfb compare feature * add comparelog.csh script and logbfb test * update travis gcc, fix tdir feature with tests * update Makefile for c compiles, update Macros files to support c compile and serial/parallel compiles better and cleaner * modify report results script to address grep for bfbcomp cases, add first_suite.ts * fix Macros.onyx_intel, accidently removed debug flags in earlier commit * update travis c build * fix script logic for more threads than tasks per node * update scripts to address CICE-Consortium#299, poll_queue script and CICE-Consortium#301, single test bgen default use * add QSTAT variables to fram and cesium
lettie-roach · Mar 27, 2019 · d417b97 · d417b97
1 parent 6f1321f
commit d417b97
Show file tree

Hide file tree

Showing 64 changed files with 5,160 additions and 1,810 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -11,6 +11,7 @@ addons:
       - pkg-config
       - netcdf-bin libnetcdf-dev #libnetcdff-dev (only required on Debian)
       - gfortran
+      - gcc
       - openmpi-bin libopenmpi-dev
       - wget
       #- lftp

diff --git a/cice.setup b/cice.setup
@@ -327,6 +327,16 @@ if (-e $tsfile) then
   exit -1
 endif
 
+set remote = `git remote -v | grep -i origin | grep -i push | sed "s|.*\(https.*\)\s.*|\1|g"`
+set branch = `git status | grep -i "on branch" | sed 's|^.*ranch\s*\(\S*\)$|\1|g'`
+set hash     = `git log | grep -i commit | head -1 | cut -d " " -f 2-`
+set hashuser = `git log | grep -i author | head -1 | cut -d : -f 2-`
+set hashdate = `git log | grep -i date   | head -1 | cut -d : -f 2-`
+set cdate = `date -u "+%Y-%m-%d"`
+set ctime = `date -u "+%H:%M:%S"`
+set vers = ${ICE_VERSION}
+set shhash = `echo ${hash} | cut -c 1-10`
+
 if ( ${dosuite} == 0 ) then
   set teststring = "${test} ${grid} ${pesx} ${sets}"
   if ( $bfbcomp != ${spval} ) then
@@ -368,25 +378,12 @@ else
   endif
   cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir}
   cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ${tsdir}
-
-  if ($report == 1) then
-    cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir}
-  endif
+  cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir}
 
 cat >! ${tsdir}/suite.run << EOF0
 #!/bin/csh -f
 EOF0
 
-  set remote = `git remote -v | grep -i origin | grep -i push | sed "s|.*\(https.*\)\s.*|\1|g"`
-  set branch = `git status | grep -i "on branch" | sed 's|^.*ranch\s*\(\S*\)$|\1|g'`
-  set hash     = `git log | grep -i commit | head -1 | cut -d " " -f 2-`
-  set hashuser = `git log | grep -i author | head -1 | cut -d : -f 2-`
-  set hashdate = `git log | grep -i date   | head -1 | cut -d : -f 2-`
-  set cdate = `date -u "+%Y-%m-%d"`
-  set ctime = `date -u "+%H:%M:%S"`
-  set vers = ${ICE_VERSION}
-  set shhash = `echo ${hash} | cut -c 1-10`
-
 cat >! ${tsdir}/results.csh << EOF0
 #!/bin/csh -f
 rm -f results.log
@@ -435,11 +432,26 @@ set bfbcomp_base = "$bfbcomp"
 foreach compiler ( $ncompilers )
   set machcomp = ${machine}_${compiler}
   foreach line ( "`cat $tsfile`" )
+
     # Check if line is a comment line
     if ( $line:q =~ '#'* || $line:q =~ '$'* || $line:q =~ '!'* ) then
       echo "skipping line: $line"
       continue
+    # Check if line is a sleep line, can only happen with suites
+    else if ( $line:q =~ 'sleep'*) then
+cat >> ${tsdir}/suite.run << EOF
+echo "-------test--------------"
+echo "$line:q"
+$line:q
+
+EOF
+      echo "adding sleep line: $line"
+      echo "" 
+      echo "---" 
+      echo "" 
+      continue
     endif
+
     # Obtain the test name, sets, grid, and PE information from .ts file
     set test = `echo $line | cut -d' ' -f1`
     set grid = `echo $line | cut -d' ' -f2`
@@ -461,7 +473,7 @@ foreach compiler ( $ncompilers )
 
     set fbfbcomp = ${spval}
     if ($bfbcomp != ${spval}) then
-      set fbfbcomp = ${machcomp}_${bfbcomp}.${testid}
+      set fbfbcomp = ${machcomp}_${bfbcomp}
     endif
 
     set testname_noid = ${spval}
@@ -480,11 +492,7 @@ foreach compiler ( $ncompilers )
       # soptions starts with _
       set testname_noid = "${machcomp}_${test}_${grid}_${pesx}${soptions}"
       set testname_base = "${machcomp}_${test}_${grid}_${pesx}${soptions}.${testid}"
-      if (${dosuite} == 1) then
-        set testname = "${tsdir}/${testname_base}"
-      else
-        set testname = "${testname_base}"
-      endif
+      set testname = "${tsdir}/${testname_base}"
       set case = ${testname}
     endif
 
@@ -504,7 +512,7 @@ foreach compiler ( $ncompilers )
     # Setup case directory, copy files to case directory
 
     mkdir -p ${case}
-    echo "`date`${0} $initargv[*]" >> ${case}/README.case
+    echo "`date` ${0} $initargv[*]" >> ${case}/README.case
 
     cd ${case}
     set casedir = `pwd`
@@ -690,6 +698,7 @@ setenv ICE_SPVAL    ${spval}
 setenv ICE_QUIETMODE  ${quietmode}
 setenv ICE_TEST     ${test}
 setenv ICE_TESTNAME ${testname_noid}
+setenv ICE_TESTID   ${testid}
 setenv ICE_BFBCOMP  ${fbfbcomp}
 setenv ICE_ACCOUNT  ${acct}
 setenv ICE_QUEUE    ${queue}
@@ -702,7 +711,7 @@ EOF1
       foreach name ($setsxorig)
         if (-e ${ICE_SCRIPTS}/options/set_files.${name}) then
           echo "adding options files from set_files.${name}"
-          echo "`date`${0} adding options files from set_files.${name}" >> ${casedir}/README.case
+          echo "`date` ${0} adding options files from set_files.${name}" >> ${casedir}/README.case
           set setsnew = `cat ${ICE_SCRIPTS}/options/set_files.${name}`
           foreach nset ($setsnew)
             if ($nset !~ "#*") then
@@ -720,9 +729,10 @@ EOF1
 
     if (${docase} == 0) then
       # from test options to casescr in case any test time changes are applied
+      cp -f -p ${ICE_SCRIPTS}/tests/comparebfb.csh ${casescr}
+      cp -f -p ${ICE_SCRIPTS}/tests/comparelog.csh ${casescr}
       if (-e ${ICE_SCRIPTS}/tests/test_${test}.files) then
         cp -f -p ${ICE_SCRIPTS}/tests/test_${test}.files ${casescr}
-        cp -f -p ${ICE_SCRIPTS}/tests/comparebfb.csh ${casescr}
         foreach file (`cat ${casescr}/test_${test}.files`)
           if (-e ${ICE_SCRIPTS}/options/$file) then
             cp -f -p ${ICE_SCRIPTS}/options/$file ${casescr}
@@ -749,7 +759,7 @@ cat >> ${fimods} << EOF2
 
 EOF2
         echo "adding namelist mods set_nml.${name}"
-        echo "`date`${0} adding namelist modes set_nml.${name}" >> ${casedir}/README.case
+        echo "`date` ${0} adding namelist modes set_nml.${name}" >> ${casedir}/README.case
         set found = 1
       endif
       if (-e ${ICE_SCRIPTS}/options/set_env.${name}) then
@@ -767,7 +777,7 @@ cat >> ${fimods} << EOF2
 EOF2
 
         echo "adding env mods set_env.${name}"
-        echo "`date`${0} adding namelist modes set_env.${name}" >> ${casedir}/README.case
+        echo "`date` ${0} adding namelist modes set_env.${name}" >> ${casedir}/README.case
         set found = 1
       endif
       if (${found} == 0) then
@@ -833,7 +843,7 @@ source ./cice.settings
 set ciceexe = "../ciceexe.\${ICE_COMPILER}.\${ICE_COMMDIR}.\${ICE_BLDDEBUG}.\${ICE_THREADED}.\${ICE_IOTYPE}"
 ./cice.build \${ciceexe}
 if !(-e \${ciceexe}) cp -p \${ICE_RUNDIR}/cice \${ciceexe}
-./cice.submit
+./cice.submit | tee -a ../suite.jobs
 cd ..
 
 EOF
@@ -887,6 +897,12 @@ echo "\$pends of \$total tests PENDING"
 exit \$failures
 EOF
 
+  if ($?ICE_MACHINE_QSTAT) then
+cat >! ${tsdir}/poll_queue.env << EOF0
+setenv ICE_MACHINE_QSTAT ${ICE_MACHINE_QSTAT}
+EOF0
+  endif
+
   # build and submit tests
   cd ${tsdir}
   ./suite.run | tee suite.log

diff --git a/cicecore/cicedynB/general/ice_flux.F90 b/cicecore/cicedynB/general/ice_flux.F90
@@ -709,6 +709,11 @@ subroutine init_coupler_flux
                            + vatm(:,:,:)**2)  ! wind speed, (m/s)
       Cdn_atm(:,:,:) = (vonkar/log(zref/iceruf)) &
                      * (vonkar/log(zref/iceruf)) ! atmo drag for RASM
+      alvdr_init(:,:,:) = c0
+      alidr_init(:,:,:) = c0
+      alvdf_init(:,:,:) = c0
+      alidf_init(:,:,:) = c0
+
 
       end subroutine init_coupler_flux
 

diff --git a/cicecore/cicedynB/general/ice_init.F90 b/cicecore/cicedynB/general/ice_init.F90
@@ -237,7 +237,7 @@ subroutine input_data
       diagfreq = 24          ! how often diag output is written
       print_points = .false. ! if true, print point data
       print_global = .true.  ! if true, print global diagnostic data
-      bfbflag = .false.      ! if true, do bit-for-bit computations
+      bfbflag = 'off'        ! off = optimized
       diag_type = 'stdout'
       diag_file = 'ice_diag.d'
       histfreq(1) = '1'      ! output frequency option for different streams
@@ -921,7 +921,7 @@ subroutine input_data
          write(nu_diag,1020) ' diagfreq                  = ', diagfreq
          write(nu_diag,1010) ' print_global              = ', print_global
          write(nu_diag,1010) ' print_points              = ', print_points
-         write(nu_diag,1010) ' bfbflag                   = ', bfbflag
+         write(nu_diag,1030) ' bfbflag                   = ', bfbflag
          write(nu_diag,1020) ' numin                     = ', numin
          write(nu_diag,1020) ' numax                     = ', numax
          write(nu_diag,1050) ' histfreq                  = ', histfreq(:)

diff --git a/cicecore/cicedynB/general/ice_step_mod.F90 b/cicecore/cicedynB/general/ice_step_mod.F90
@@ -98,10 +98,10 @@ subroutine prep_radiation (iblk)
 
       call ice_timer_start(timer_sw)      ! shortwave
 
-      alvdr_init(:,:,:) = c0
-      alvdf_init(:,:,:) = c0
-      alidr_init(:,:,:) = c0
-      alidf_init(:,:,:) = c0
+      alvdr_init(:,:,iblk) = c0
+      alvdf_init(:,:,iblk) = c0
+      alidr_init(:,:,iblk) = c0
+      alidf_init(:,:,iblk) = c0
 
          this_block = get_block(blocks_ice(iblk),iblk)         
          ilo = this_block%ilo