From f3c94d2f1040ff432d864ba9f9a74918a0fbf8a1 Mon Sep 17 00:00:00 2001 From: abouchat <50667934+abouchat@users.noreply.github.com> Date: Thu, 16 May 2019 16:32:46 -0400 Subject: [PATCH] Add machine files for millikan (#313) --- configuration/scripts/cice.batch.csh | 13 ++++ configuration/scripts/cice.launch.csh | 12 ++++ .../scripts/machines/Macros.millikan_intel | 72 +++++++++++++++++++ .../scripts/machines/env.millikan_intel | 25 +++++++ 4 files changed, 122 insertions(+) create mode 100644 configuration/scripts/machines/Macros.millikan_intel create mode 100644 configuration/scripts/machines/env.millikan_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index c7c410e55..07b1f06e8 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -150,6 +150,19 @@ cat >> ${jobfile} << EOFB #SBATCH --qos=standby EOFB +else if (${ICE_MACHINE} =~ millikan*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -A ${acct} +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +###SBATCH --mail-type END,FAIL +###SBATCH --mail-user=amelie.bouchat@canada.ca +#SBATCH --qos=standby +EOFB + else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFB # nothing to do diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index b5f243159..86ede7049 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -141,6 +141,18 @@ mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR endif +#======= +else if (${ICE_MACHINE} =~ millikan*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + #======= else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFR diff --git a/configuration/scripts/machines/Macros.millikan_intel b/configuration/scripts/machines/Macros.millikan_intel new file mode 100644 index 000000000..1d74158c9 --- /dev/null +++ b/configuration/scripts/machines/Macros.millikan_intel @@ -0,0 +1,72 @@ +#============================================================================== +# Makefile macros for "millikan" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 +#-xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + +INCLDIR := $(INCLDIR) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_PNETCDF := $(PNETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl +SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf + +ifeq ($(compile_threaded), true) + LDFLAGS += -openmp + CFLAGS += -openmp + FFLAGS += -openmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include + SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof + + CPPDEFS := $(CPPDEFS) -Dncdf +endif + +ifeq ($(IO_TYPE), netcdf) + CPPDEFS := $(CPPDEFS) -Dncdf +endif diff --git a/configuration/scripts/machines/env.millikan_intel b/configuration/scripts/machines/env.millikan_intel new file mode 100644 index 000000000..fc81ac097 --- /dev/null +++ b/configuration/scripts/machines/env.millikan_intel @@ -0,0 +1,25 @@ +#!/bin/csh -f + +set ssmuse=/fs/ssm/main/env/20180430/all/bin/ssmuse-csh # package loader +source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156 # intel compiler +source /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156/intelcomp_2016.1.156_multi/bin/compilervars.csh intel64 # should be sourced by above domain, but bug in csh script +source $ssmuse -d /fs/ssm/main/opt/openmpi/openmpi-1.6.5/intelcomp-2016.1.156 # openmpi +source $ssmuse -d /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150 # netcdf (and openmpi) + +setenv ICE_MACHINE_ENVNAME millikan +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /users/dor/armn/amb/data/local/runs +setenv ICE_MACHINE_INPUTDATA /users/dor/armn/amb/data/local/forcing +setenv ICE_MACHINE_BASELINE /users/dor/armn/amb/data/local/baseline +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 36 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_BLDTHRDS 4 +setenv ICE_MACHINE_QSTAT "qstat " + +if (-e ~/.cice_proj) then + set account_name = `head -1 ~/.cice_proj` + setenv CICE_ACCT ${account_name} +endif