Port to Izumi (#356)

* izumi initial port

* update izumi port and add nothread_suite

* update kind conversions in ice_dyn_evp_1d.F90

cice.setup

@@ -493,14 +493,10 @@ EOF
       set blckx = `echo ${pesx} | cut -d x -f 3`
       set blcky = `echo ${pesx} | cut -d x -f 4`
       set mblck = `echo ${pesx} | cut -d x -f 5`
-      if ($?ICE_MACHINE_MAXPES) then
-        @ pesreq = ${task} * ${thrd}
-        if (${pesreq} > ${ICE_MACHINE_MAXPES}) then
-          @ task = ${ICE_MACHINE_MAXPES} / ${thrd}
-          @ mblck = ${mblck} * ((${pesreq} / ${ICE_MACHINE_MAXPES}) + 1)
-        endif
+      if (${task} == 0 || ${thrd} == 0 || ${blckx} == 0 || ${blcky} == 0 || ${mblck} == 0) then
+        echo "${0}: ERROR in -p argument, cannot have zeros"
+        exit -1
       endif
-      set pesx = ${task}x${thrd}x${blckx}x${blcky}x${mblck}
     else
       set chck = `echo ${pesx} | sed  's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'`
       if (${chck} == OK) then
@@ -509,13 +505,10 @@ EOF
         set blckx = `echo ${pesx} | cut -d x -f 3`
         set blcky = `echo ${pesx} | cut -d x -f 4`
         set mblck = 0
-        if ($?ICE_MACHINE_MAXPES) then
-          @ pesreq = ${task} * ${thrd}
-          if (${pesreq} > ${ICE_MACHINE_MAXPES}) then
-            @ task = ${ICE_MACHINE_MAXPES} / ${thrd}
-          endif
+        if (${task} == 0 || ${thrd} == 0 || ${blckx} == 0 || ${blcky} == 0) then
+          echo "${0}: ERROR in -p argument, cannot have zeros"
+          exit -1
         endif
-        set pesx = ${task}x${thrd}x${blckx}x${blcky}
       else
         set chck = `echo ${pesx} | sed  's/^[0-9][0-9]*x[0-9][0-9]*$/OK/'`
         if (${chck} == OK) then
@@ -524,20 +517,73 @@ EOF
           set blckx = 0
           set blcky = 0
           set mblck = 0
-          if ($?ICE_MACHINE_MAXPES) then
-            @ pesreq = ${task} * ${thrd}
-            if (${pesreq} > ${ICE_MACHINE_MAXPES}) then
-              @ task = ${ICE_MACHINE_MAXPES} / ${thrd}
-            endif
+          if (${task} == 0 || ${thrd} == 0) then
+            echo "${0}: ERROR in -p argument, cannot have zeros"
+            exit -1
           endif
-          set pesx = ${task}x${thrd}
         else
-          echo "${0}: ERROR in -p argument, ${pesx}, must be [m]x[n], [m]x[n]x[bx]x[by], or [m]x[n]x[bx]x[by]x[mb] "
-          exit -1
+          set chck = `echo ${pesx} | sed  's/^[0-9][0-9]*$/OK/'`
+          if (${chck} == OK) then
+            set task = `echo ${pesx} | cut -d x -f 1`
+            set thrd = 1
+            set blckx = 0
+            set blcky = 0
+            set mblck = 0
+            if (${task} == 0) then
+              echo "${0}: ERROR in -p argument, cannot have zeros"
+              exit -1
+            endif
+          else
+            echo "${0}: ERROR in -p argument, ${pesx}, must be [m], [m]x[n], [m]x[n]x[bx]x[by], or [m]x[n]x[bx]x[by]x[mb] "
+            exit -1
+          endif
+        endif
+      endif
+    endif
+
+    @ blkreq = ${task} * ${mblck}
+    # check max threads, reduce threads, increase tasks
+    if ($?ICE_MACHINE_MAXTHREADS) then
+      if (${thrd} > ${ICE_MACHINE_MAXTHREADS}) then
+        @ pesreq = ${task} * ${thrd}
+        @ task = ${pesreq} / ${ICE_MACHINE_MAXTHREADS}
+        @ thrd = ${ICE_MACHINE_MAXTHREADS}
+        @ peschk = ${task} * ${thrd}
+        if (${pesreq} > ${peschk}) then
+          @ task = ${task} + 1
         endif
+#        echo "tcx1 reset to $task $thrd $mblck"
+      endif
+    endif
+
+    # check max pes, reduce tasks
+    if ($?ICE_MACHINE_MAXPES) then
+      @ pesreq = ${task} * ${thrd}
+      if (${pesreq} > ${ICE_MACHINE_MAXPES}) then
+        @ task = ${ICE_MACHINE_MAXPES} / ${thrd}
+#        echo "tcx2 reset to $task $thrd $mblck"
       endif
     endif
 
+    # check max blocks and adjust as needed
+    if ($mblck > 0) then
+      @ mblck = ${blkreq} / ${task}
+      @ blkchk = ${task} * ${mblck}
+      if (${blkreq} > ${blkchk}) then
+        @ mblck = $mblck + 1
+      endif
+#      echo "tcx3 reset to $task $thrd $mblck"
+    endif
+
+    # update pesx based on use defined settings and machine limits to reflect actual value
+    set pesx = ${task}x${thrd}x${blckx}x${blcky}x${mblck}
+    if (${mblck} == 0) then
+      set pesx = ${task}x${thrd}x${blckx}x${blcky}
+    endif
+    if (${blckx} == 0 || ${blcky} == 0) then
+      set pesx = ${task}x${thrd}
+    endif
+
     set testname_noid = ${spval}
     # create case for test cases
     if (${docase} == 0) then

cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90

@@ -47,7 +47,7 @@ module dmi_omp
 #if defined (_OPENMP)
   ! Please note, this constant will create a compiler info for a constant
   ! expression in IF statements:
-  real(kind=dbl_kind), private :: rdomp_iam, rdomp_nt
+  real(kind=dbl_kind) :: rdomp_iam, rdomp_nt
   !$OMP THREADPRIVATE(domp_iam,domp_nt,rdomp_iam,rdomp_nt) 
 #endif
 
@@ -70,9 +70,9 @@ subroutine domp_init(nt_out)
     !$OMP PARALLEL DEFAULT(none)
 #if defined (_OPENMP)
     domp_iam  = omp_get_thread_num()
-    rdomp_iam = real(domp_iam,8)
+    rdomp_iam = real(domp_iam,dbl_kind)
     domp_nt   = omp_get_num_threads()
-    rdomp_nt  = real(domp_nt,8)
+    rdomp_nt  = real(domp_nt,dbl_kind)
 #else
     domp_iam  = 0
     domp_nt   = 1
@@ -139,9 +139,9 @@ subroutine domp_get_domain_rlu(lower,upper,d_lower,d_upper)
 
 #if defined (_OPENMP)
     if (omp_in_parallel()) then
-      dlen    = real(upper-lower+1, 8)
-      d_lower = lower    + floor((rdomp_iam*dlen+p5)/rdomp_nt, 4)
-      d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, 4)
+      dlen    = real(upper-lower+1, dbl_kind)
+      d_lower = lower    + floor((rdomp_iam*dlen+p5)/rdomp_nt, JPIM)
+      d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, JPIM)
     endif
 #endif
 

configuration/scripts/cice.batch.csh

@@ -79,6 +79,18 @@ cat >> ${jobfile} << EOFB
 #PBS -l nodes=1:ppn=24
 EOFB
 
+else if (${ICE_MACHINE} =~ izumi*) then
+if (${runlength} > 2) set queue = "medium"
+cat >> ${jobfile} << EOFB
+#PBS -j oe 
+###PBS -m ae 
+#PBS -V
+#PBS -q ${queue}
+#PBS -N ${ICE_CASENAME}
+#PBS -l nodes=${nnodes}:ppn=${taskpernode}
+#PBS -l walltime=${batchtime}
+EOFB
+
 else if (${ICE_MACHINE} =~ thunder* || ${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad*  || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr*) then
 cat >> ${jobfile} << EOFB
 #PBS -N ${shortcase}

configuration/scripts/cice.launch.csh

@@ -34,7 +34,7 @@ EOFR
 endif
 
 #=======
-else if (${ICE_MACHINE} =~ hobart*) then
+else if (${ICE_MACHINE} =~ hobart* || ${ICE_MACHINE} =~ izumi*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE

configuration/scripts/machines/Macros.izumi_gnu

@@ -0,0 +1,52 @@
+#==============================================================================
+# Makefile macros for NCAR izumi, gnu compiler
+#==============================================================================
+
+CPP        := /usr/bin/cpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2
+
+FIXEDFLAGS := -ffixed-line-length-132
+FREEFLAGS  := -ffree-form
+FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS   += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+else
+  FFLAGS   += -O2
+endif
+
+SCC   := gcc 
+SFC   := gfortran
+MPICC := mpicc
+MPIFC := mpif90
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+INCLDIR := -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif
+
+## if using parallel I/O, load all 3 libraries.  PIO must be first!
+ifeq ($(IO_TYPE), pio)
+   PIO_PATH:=
+   INCLDIR += -I
+   SLIBS   := $(SLIB) -L$(PIO_PATH) -lpiofS
+endif
+

configuration/scripts/machines/Macros.izumi_intel

@@ -0,0 +1,52 @@
+#==============================================================================
+# Makefile macros for NCAR izumi, intel compiler
+#==============================================================================
+
+CPP        := /usr/bin/cpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -qno-opt-dynamic-align -fp-model precise
+
+FIXEDFLAGS := -fixed -132
+FREEFLAGS  := -free
+FFLAGS     := -qno-opt-dynamic-align  -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS   += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
+else
+  FFLAGS   += -O2
+endif
+
+SCC   := icc 
+SFC   := ifort
+MPICC := mpicc
+MPIFC := mpif90
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+INCLDIR := -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -mkl
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+
+## if using parallel I/O, load all 3 libraries.  PIO must be first!
+ifeq ($(IO_TYPE), pio)
+   PIO_PATH:=
+   INCLDIR += -I
+   SLIBS   := $(SLIB) -L$(PIO_PATH) -lpiofS
+endif
+

configuration/scripts/machines/Macros.izumi_nag

@@ -0,0 +1,53 @@
+#==============================================================================
+# Makefile macros for NCAR izumi, NAG compiler
+#==============================================================================
+
+CPP        := /usr/bin/cpp
+CPPDEFS    := -DFORTRANUNDERSCORE -DNO_R16 $(ICE_CPPDEFS)
+CFLAGS     := -c
+
+FIXEDFLAGS := -fixed
+FREEFLAGS  := -free
+FFLAGS     := -Wp,-macro=no_com -convert=BIG_ENDIAN -wmismatch=mpi_bcast,mpi_isend,mpi_irecv,mpi_send,mpi_recv,mpi_allreduce
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS   += -O0 -C=all -g -time -f2003 -ieee=stop
+else
+  FFLAGS   += -O2 -ieee=full
+endif
+
+SCC   := cc
+SFC   := nagfor
+MPICC := mpicc
+MPIFC := mpif90
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+INCLDIR := -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true)
+   LDFLAGS += -openmp
+   CFLAGS += -fopenmp
+   FFLAGS += -openmp
+else
+   FFLAGS += -gline
+endif
+
+## if using parallel I/O, load all 3 libraries.  PIO must be first!
+ifeq ($(IO_TYPE), pio)
+   PIO_PATH:=
+   INCLDIR += -I
+   SLIBS   := $(SLIB) -L$(PIO_PATH) -lpiof
+endif

configuration/scripts/machines/Macros.izumi_pgi

@@ -0,0 +1,52 @@
+#==============================================================================
+# Makefile macros for NCAR izumi, pgi compiler
+#==============================================================================
+
+CPP        := pgcc -E
+CPPDEFS    := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -Kieee
+
+FIXEDFLAGS := -Mextend
+FREEFLAGS  := -Mfree
+FFLAGS     := -Kieee -Mbyteswapio -traceback
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS   += -O0 -g -Mbounds -Mchkptr
+else
+  FFLAGS   += -O2
+endif
+
+SCC   := pgcc 
+SFC   := pgf90
+MPICC := mpicc
+MPIFC := mpif90
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+INCLDIR := -I$(NETCDF_PATH)/include
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -mp
+   CFLAGS += -mp
+   FFLAGS += -mp
+endif
+
+## if using parallel I/O, load all 3 libraries.  PIO must be first!
+ifeq ($(IO_TYPE), pio)
+   PIO_PATH:=
+   INCLDIR += -I
+   SLIBS   := $(SLIB) -L$(PIO_PATH) -lpiofS
+endif
+