Port to Perlmutter gnu, intel, cray. (#467)

CICE-Consortium · Oct 13, 2023 · aea58f3 · aea58f3
1 parent 182030d
commit aea58f3
Show file tree

Hide file tree

Showing 8 changed files with 290 additions and 0 deletions.
diff --git a/configuration/scripts/icepack.batch.csh b/configuration/scripts/icepack.batch.csh
@@ -127,6 +127,23 @@ cat >> ${jobfile} << EOFB
 ###SBATCH --mail-user username@domain.com
 EOFB
 
+else if (${ICE_MACHINE} =~ perlmutter*) then
+@ nthrds2 = ${nthrds} * 2
+cat >> ${jobfile} << EOFB
+#SBATCH -J ${ICE_CASENAME}
+#SBATCH -A ${acct}
+#SBATCH --qos ${ICE_MACHINE_QUEUE}
+#SBATCH --time ${ICE_RUNLENGTH}
+#SBATCH --nodes ${nnodes}
+#SBATCH --ntasks ${ncores}
+#SBATCH --cpus-per-task ${nthrds2}
+#SBATCH --constraint cpu
+###SBATCH -e filename
+###SBATCH -o filename
+###SBATCH --mail-type FAIL
+###SBATCH --mail-user username@domain.com
+EOFB
+
 else if (${ICE_MACHINE} =~ compy*) then
 cat >> ${jobfile} << EOFB
 #SBATCH -J ${ICE_CASENAME}

diff --git a/configuration/scripts/icepack.launch.csh b/configuration/scripts/icepack.launch.csh
@@ -14,6 +14,13 @@ cat >> ${jobfile} << EOFR
 aprun -n 1 -N 1 -d 1 ./icepack >&! \$ICE_RUNLOG_FILE
 EOFR
 
+#==========================================
+
+elseif (${ICE_MACHINE} =~ perlmutter) then
+cat >> ${jobfile} << EOFR
+srun --cpu-bind=cores ./icepack >&! \$ICE_RUNLOG_FILE
+EOFR
+
 #==========================================
 else
 cat >> ${jobfile} << EOFR

diff --git a/configuration/scripts/machines/Macros.perlmutter_cray b/configuration/scripts/machines/Macros.perlmutter_cray
@@ -0,0 +1,38 @@
+#==============================================================================
+# Makefile macros for NERSC perlmutter, cray compiler
+#==============================================================================
+
+CPP        := ftn -e P
+CPPDEFS    := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS}
+CFLAGS     := -c -O2
+
+FIXEDFLAGS := -132
+FREEFLAGS  := 
+FFLAGS     := -hbyteswapio
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -hfp0 -g -Rbcdps -Ktrap=fp
+else
+  FFLAGS     += -O2 -hfp0   # -eo
+endif
+
+SCC   := cc
+SFC   := ftn
+CC := $(SCC)
+FC := $(SFC)
+LD := $(FC)
+
+NETCDF_PATH := $(NETCDF_DIR)
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true)
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif
+
diff --git a/configuration/scripts/machines/Macros.perlmutter_gnu b/configuration/scripts/machines/Macros.perlmutter_gnu
@@ -0,0 +1,38 @@
+#==============================================================================
+# Makefile macros for NERSC perlmutter, gnu compiler
+#==============================================================================
+
+CPP        := ftn -E
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c
+
+FIXEDFLAGS := -ffixed-line-length-132
+FREEFLAGS  := -ffree-form
+FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS   += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow --std f2008
+#  FFLAGS   += -O0 -g -fcheck=all -finit-real=snan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+  CFLAGS   += -O0
+endif
+
+SCC   := gcc
+SFC   := gfortran
+CC := $(SCC)
+FC := $(SFC)
+LD := $(FC)
+
+NETCDF_PATH := $(NETCDF_DIR)
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true)
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif
+
diff --git a/configuration/scripts/machines/Macros.perlmutter_intel b/configuration/scripts/machines/Macros.perlmutter_intel
@@ -0,0 +1,40 @@
+#==============================================================================
+# Makefile macros for NERSC perlmutter, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -march=core-avx2
+
+FIXEDFLAGS := -fixed -132
+FREEFLAGS  := -free
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := icx
+SFC   := ifort
+CC := $(SCC)
+FC := $(SFC)
+LD := $(FC)
+
+NETCDF_PATH := $(NETCDF_DIR)
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF_PATH)/lib
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true)
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+
diff --git a/configuration/scripts/machines/env.perlmutter_cray b/configuration/scripts/machines/env.perlmutter_cray
@@ -0,0 +1,50 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+#module unload PrgEnv-aocc
+#module unload PrgEnv-cray
+#module unload PrgEnv-gnu
+#module unload PrgEnv-intel
+#module unload PrgEnv-nvidia
+#module unload cpe
+#module load cpe/23.03
+#module unload gpu
+module load cpu
+module load PrgEnv-cray
+module unload cce
+module load cce/15.0.1
+module unload cray-netcdf
+module unload cray-hdf5
+module load cray-hdf5/1.12.2.3
+module load cray-netcdf/4.9.0.3
+#module unload cray-pals
+#module load cray-pals/1.2.2
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+setenv ICE_MACHINE_MACHNAME perlmutter
+setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect"
+setenv ICE_MACHINE_ENVNAME cray
+setenv ICE_MACHINE_ENVINFO "Cray clang/Fortran 15.0.1, netcdf4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $SCRATCH/ICEPACK_RUNS
+setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium
+setenv ICE_MACHINE_BASELINE $SCRATCH/ICEPACK_BASELINE
+setenv ICE_MACHINE_SUBMIT "sbatch"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "shared"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 1
+setenv ICE_MACHINE_QSTAT "squeue --jobs= "
diff --git a/configuration/scripts/machines/env.perlmutter_gnu b/configuration/scripts/machines/env.perlmutter_gnu
@@ -0,0 +1,50 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+#module unload PrgEnv-aocc
+#module unload PrgEnv-cray
+#module unload PrgEnv-gnu
+#module unload PrgEnv-intel
+#module unload PrgEnv-nvidia
+#module unload cpe
+#module load cpe/23.03
+#module unload gpu
+module load cpu
+module load PrgEnv-gnu
+module unload gcc
+module load gcc/11.2.0
+module unload cray-netcdf
+module unload cray-hdf5
+module load cray-hdf5/1.12.2.3
+module load cray-netcdf/4.9.0.3
+#module unload cray-pals
+#module load cray-pals/1.2.2
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+setenv ICE_MACHINE_MACHNAME perlmutter
+setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect"
+setenv ICE_MACHINE_ENVNAME gnu
+setenv ICE_MACHINE_ENVINFO "gcc/gfortran 11.2.0 20210728, netcdf4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $SCRATCH/ICEPACK_RUNS
+setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium
+setenv ICE_MACHINE_BASELINE $SCRATCH/ICEPACK_BASELINE
+setenv ICE_MACHINE_SUBMIT "sbatch"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "shared"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 1
+setenv ICE_MACHINE_QSTAT "squeue --jobs= "
diff --git a/configuration/scripts/machines/env.perlmutter_intel b/configuration/scripts/machines/env.perlmutter_intel
@@ -0,0 +1,50 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+#module unload PrgEnv-aocc
+#module unload PrgEnv-cray
+#module unload PrgEnv-gnu
+#module unload PrgEnv-intel
+#module unload PrgEnv-nvidia
+#module unload cpe
+#module load cpe/23.03
+#module unload gpu
+module load cpu
+module load PrgEnv-intel
+module unload intel
+module load intel/2023.1.0
+module unload cray-netcdf
+module unload cray-hdf5
+module load cray-hdf5/1.12.2.3
+module load cray-netcdf/4.9.0.3
+#module unload cray-pals
+#module load cray-pals/1.2.2
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+setenv ICE_MACHINE_MACHNAME perlmutter
+setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "ifort 2021.9.0 20230302, netcdf4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $SCRATCH/ICEPACK_RUNS
+setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium
+setenv ICE_MACHINE_BASELINE $SCRATCH/ICEPACK_BASELINE
+setenv ICE_MACHINE_SUBMIT "sbatch"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "shared"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 1
+setenv ICE_MACHINE_QSTAT "squeue --jobs= "