From c334aee2e27e27767a1758a0acf1a12fa69c8565 Mon Sep 17 00:00:00 2001 From: Philippe Blain Date: Thu, 2 Jun 2022 11:26:36 -0400 Subject: [PATCH] machines: eccc: update our machine files (#725) A few notes: - We do need to request memory even on machines where we have excluseive node access. 20 GB was chosen rather arbitrarily. - We set umask to 022 to make the .o and .e files readable by group and others. - We use the minimal SSM package for the compiler and Intel MPI, but we keep the setup using environment modules commented if ever we need to weak things (i.e. I_MPI_LIBRARY_KIND) - We set OMP_STACKSIZE. Since d1e972a (Update OMP (CICE-Consortium#680), 2022-02-18), OpenMP threading is active in 'ice_transport_remap.F90', and the default OpenMP stack size needs to be adjusted to avoid stack overflows. We set it to a 64 Mb size as used for other machines. Also, remove dead code setting 'CICE_ACCT'. This variable was last used in 98e03077 (Update scripts, rename variables from CICE_ to ICE_ to be more reusable in icepack., 2017-09-15), and so did not do anything for any of the machines that were using it after that commit. Remove code in machines env files that was setting it based on '~/.cice_proj'. A few notes specific to 'gpsc3': - Since we use an '--export' directive to choose which environment variables are exported to the job environment by SLURM, SSMUSE_BASE and SSMUSE_PATH are not present in the environnement and loading domains without their full paths fails on csh, so use a full path. - We use the compiler package from main/opt instead of eccc/all/opt since we do not need the EC-specific variables to be set (and it also leads to job failures since BASE_ARCH is not defined). --- configuration/scripts/cice.batch.csh | 37 ++++++++++++++ configuration/scripts/cice.launch.csh | 37 ++++++++++++++ ...cros.millikan_intel => Macros.gpsc3_intel} | 33 +++++-------- ...{Macros.cesium_intel => Macros.ppp3_intel} | 33 +++++-------- .../scripts/machines/Macros.ppp5_intel | 49 +++++++++++++++++++ .../scripts/machines/Macros.ppp6_gnu | 44 +++++++++++++++++ .../scripts/machines/Macros.ppp6_gnu-impi | 44 +++++++++++++++++ .../scripts/machines/Macros.ppp6_intel | 49 +++++++++++++++++++ .../scripts/machines/Macros.ppp6_intel19 | 49 +++++++++++++++++++ .../scripts/machines/Macros.robert_intel | 49 +++++++++++++++++++ .../scripts/machines/Macros.underhill_intel | 49 +++++++++++++++++++ .../scripts/machines/env.cesium_intel | 30 ------------ configuration/scripts/machines/env.fram_intel | 5 -- .../scripts/machines/env.gpsc3_intel | 37 ++++++++++++++ .../scripts/machines/env.millikan_intel | 30 ------------ configuration/scripts/machines/env.ppp3_intel | 35 +++++++++++++ configuration/scripts/machines/env.ppp5_intel | 38 ++++++++++++++ configuration/scripts/machines/env.ppp6_gnu | 29 +++++++++++ .../scripts/machines/env.ppp6_gnu-impi | 38 ++++++++++++++ configuration/scripts/machines/env.ppp6_intel | 38 ++++++++++++++ .../scripts/machines/env.ppp6_intel19 | 39 +++++++++++++++ .../scripts/machines/env.robert_intel | 39 +++++++++++++++ .../scripts/machines/env.underhill_intel | 39 +++++++++++++++ 23 files changed, 761 insertions(+), 109 deletions(-) rename configuration/scripts/machines/{Macros.millikan_intel => Macros.gpsc3_intel} (55%) rename configuration/scripts/machines/{Macros.cesium_intel => Macros.ppp3_intel} (55%) create mode 100644 configuration/scripts/machines/Macros.ppp5_intel create mode 100644 configuration/scripts/machines/Macros.ppp6_gnu create mode 100644 configuration/scripts/machines/Macros.ppp6_gnu-impi create mode 100644 configuration/scripts/machines/Macros.ppp6_intel create mode 100644 configuration/scripts/machines/Macros.ppp6_intel19 create mode 100644 configuration/scripts/machines/Macros.robert_intel create mode 100644 configuration/scripts/machines/Macros.underhill_intel delete mode 100755 configuration/scripts/machines/env.cesium_intel create mode 100644 configuration/scripts/machines/env.gpsc3_intel delete mode 100755 configuration/scripts/machines/env.millikan_intel create mode 100644 configuration/scripts/machines/env.ppp3_intel create mode 100644 configuration/scripts/machines/env.ppp5_intel create mode 100644 configuration/scripts/machines/env.ppp6_gnu create mode 100644 configuration/scripts/machines/env.ppp6_gnu-impi create mode 100644 configuration/scripts/machines/env.ppp6_intel create mode 100644 configuration/scripts/machines/env.ppp6_intel19 create mode 100644 configuration/scripts/machines/env.robert_intel create mode 100644 configuration/scripts/machines/env.underhill_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 7a1334532..43ce00010 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -250,8 +250,45 @@ cat >> ${jobfile} << EOFB #PBS -j oe #PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds} #PBS -l walltime=${batchtime} +#PBS -W umask=022 EOFB +else if (${ICE_MACHINE} =~ robert* || ${ICE_MACHINE} =~ underhill* || ${ICE_MACHINE} =~ ppp6* || ${ICE_MACHINE} =~ ppp5*) then +cat >> ${jobfile} << EOFB +#PBS -N ${ICE_CASENAME} +#PBS -j oe +#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=20gb +#PBS -l walltime=${batchtime} +#PBS -W umask=022 +EOFB + +else if (${ICE_MACHINE} =~ ppp3*) then +cat >> ${jobfile} << EOFB +#PBS -N ${ICE_CASENAME} +#PBS -j oe +#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}:mem=20gb:res_tmpfs=1000:res_image=eccc/eccc_all_ppp_ubuntu-18.04-amd64_latest +#PBS -l walltime=${batchtime} +#PBS -W umask=022 +#PBS -q development +#PBS -o ${ICE_CASEDIR} +#PBS -S /bin/csh +EOFB + +else if (${ICE_MACHINE} =~ gpsc3*) then +cat >> ${jobfile} << EOFB +#SBATCH --export=USER,LOGNAME,HOME,MAIL,PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +#SBATCH -J ${ICE_CASENAME} +#SBATCH -A ${acct} +#SBATCH --partition ${queue} +#SBATCH --time ${batchtime} +#SBATCH --nodes ${nnodes} +#SBATCH --ntasks ${ntasks} +#SBATCH --cpus-per-task ${nthrds} +#SBATCH --mem-per-cpu=5G +#SBATCH --comment="image=eccc/eccc_all_default_ubuntu-18.04-amd64_latest" +EOFB + + else if (${ICE_MACHINE} =~ freya* ) then cat >> ${jobfile} << EOFB #PBS -N ${ICE_CASENAME} diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 9a557ec44..a4b6ca37d 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -159,6 +159,43 @@ aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_F EOFR endif +#======= +else if (${ICE_MACHINE} =~ ppp5* || ${ICE_MACHINE} =~ ppp6* || ${ICE_MACHINE} =~ robert* || ${ICE_MACHINE} =~ underhill*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ ppp3*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +rumpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ gpsc3*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + + #======= else if (${ICE_MACHINE} =~ freya*) then if (${ICE_COMMDIR} =~ serial*) then diff --git a/configuration/scripts/machines/Macros.millikan_intel b/configuration/scripts/machines/Macros.gpsc3_intel similarity index 55% rename from configuration/scripts/machines/Macros.millikan_intel rename to configuration/scripts/machines/Macros.gpsc3_intel index 4a3b21093..916996617 100644 --- a/configuration/scripts/machines/Macros.millikan_intel +++ b/configuration/scripts/machines/Macros.gpsc3_intel @@ -1,5 +1,5 @@ #============================================================================== -# Makefile macros for "millikan" +# Makefile macros for "gpsc3" #============================================================================== # For use with intel compiler #============================================================================== @@ -11,12 +11,12 @@ CFLAGS := -c -O2 -fp-model precise FIXEDFLAGS := -132 FREEFLAGS := -FR -FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -no-wrap-margin +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin #-xHost -FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created +#-init=snan,arrays # -heap-arrays 1024 else FFLAGS += -O2 @@ -36,25 +36,14 @@ else endif LD:= $(FC) -NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ - -PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs - -#PNETCDF_PATH := $(PNETCDF) -#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib - -INCLDIR := $(INCLDIR) - -LIB_NETCDF := $(NETCDF_PATH)/lib -LIB_PNETCDF := $(PNETCDF_PATH)/lib -LIB_MPI := $(IMPILIBDIR) - -#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl -SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif ifeq ($(ICE_THREADED), true) - LDFLAGS += -openmp - CFLAGS += -openmp - FFLAGS += -openmp + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp endif diff --git a/configuration/scripts/machines/Macros.cesium_intel b/configuration/scripts/machines/Macros.ppp3_intel similarity index 55% rename from configuration/scripts/machines/Macros.cesium_intel rename to configuration/scripts/machines/Macros.ppp3_intel index 2ad3ff1f3..143445478 100644 --- a/configuration/scripts/machines/Macros.cesium_intel +++ b/configuration/scripts/machines/Macros.ppp3_intel @@ -1,5 +1,5 @@ #============================================================================== -# Makefile macros for "cesium" +# Makefile macros for "ppp3" #============================================================================== # For use with intel compiler #============================================================================== @@ -11,12 +11,12 @@ CFLAGS := -c -O2 -fp-model precise FIXEDFLAGS := -132 FREEFLAGS := -FR -FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -no-wrap-margin +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin #-xHost -FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created +#-init=snan,arrays # -heap-arrays 1024 else FFLAGS += -O2 @@ -36,25 +36,14 @@ else endif LD:= $(FC) -NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ - -PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs - -#PNETCDF_PATH := $(PNETCDF) -#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib - -INCLDIR := $(INCLDIR) - -LIB_NETCDF := $(NETCDF_PATH)/lib -LIB_PNETCDF := $(PNETCDF_PATH)/lib -LIB_MPI := $(IMPILIBDIR) - -#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl -SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -llapack -lblas +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif ifeq ($(ICE_THREADED), true) - LDFLAGS += -openmp - CFLAGS += -openmp - FFLAGS += -openmp + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp endif diff --git a/configuration/scripts/machines/Macros.ppp5_intel b/configuration/scripts/machines/Macros.ppp5_intel new file mode 100644 index 000000000..0a45703b9 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp5_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "ppp5" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.ppp6_gnu b/configuration/scripts/machines/Macros.ppp6_gnu new file mode 100644 index 000000000..c55406d39 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp6_gnu @@ -0,0 +1,44 @@ +#============================================================================== +# Makefile macros for ECCC ppp6 +#============================================================================== +# For use with GNU compiler +#============================================================================== + +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 +#-xHost + +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := gcc +SFC := gfortran +MPICC := mpicc +MPIFC := mpifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.ppp6_gnu-impi b/configuration/scripts/machines/Macros.ppp6_gnu-impi new file mode 100644 index 000000000..413154223 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp6_gnu-impi @@ -0,0 +1,44 @@ +#============================================================================== +# Makefile macros for ECCC ppp6 +#============================================================================== +# For use with GNU compiler +#============================================================================== + +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 +#-xHost + +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := gcc +SFC := gfortran +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.ppp6_intel b/configuration/scripts/machines/Macros.ppp6_intel new file mode 100644 index 000000000..ece398a49 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp6_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "ppp6" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.ppp6_intel19 b/configuration/scripts/machines/Macros.ppp6_intel19 new file mode 100644 index 000000000..ece398a49 --- /dev/null +++ b/configuration/scripts/machines/Macros.ppp6_intel19 @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "ppp6" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.robert_intel b/configuration/scripts/machines/Macros.robert_intel new file mode 100644 index 000000000..e54a645c9 --- /dev/null +++ b/configuration/scripts/machines/Macros.robert_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "robert" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.underhill_intel b/configuration/scripts/machines/Macros.underhill_intel new file mode 100644 index 000000000..70a889009 --- /dev/null +++ b/configuration/scripts/machines/Macros.underhill_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for "underhill" +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 -no-wrap-margin +#-xHost + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -check nooutput_conversion -debug-parameters all +#-init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/env.cesium_intel b/configuration/scripts/machines/env.cesium_intel deleted file mode 100755 index 329bdf32d..000000000 --- a/configuration/scripts/machines/env.cesium_intel +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/csh -f - -set ssmuse=/fs/ssm/main/env/20180430/all/bin/ssmuse-csh # package loader -source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156 # intel compiler -source /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156/intelcomp_2016.1.156_multi/bin/compilervars.csh intel64 # should be sourced by above domain, but bug in csh script -source $ssmuse -d /fs/ssm/main/opt/openmpi/openmpi-1.6.5/intelcomp-2016.1.156 # openmpi -source $ssmuse -d /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150 # netcdf (and openmpi) - -# May be needed for OpenMP memory -#setenv OMP_STACKSIZE 64M - -setenv ICE_MACHINE_MACHNAME cesium -setenv ICE_MACHINE_MACHINFO "cesium" -setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_ENVINFO "intel 2016.1.156, openmpi 1.6.5" -setenv ICE_MACHINE_MAKE colormake-short -setenv ICE_MACHINE_WKDIR /users/dor/afsg/phb/local/CICEDIRS/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /users/dor/afsg/phb/local/FORCING -setenv ICE_MACHINE_BASELINE /users/dor/afsg/phb/local/CICEDIRS/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT "qsub" -setenv ICE_MACHINE_TPNODE 36 -setenv ICE_MACHINE_ACCT P0000000 -setenv ICE_MACHINE_QUEUE "debug" -setenv ICE_MACHINE_BLDTHRDS 4 -setenv ICE_MACHINE_QSTAT "qstat " - -if (-e ~/.cice_proj) then - set account_name = `head -1 ~/.cice_proj` - setenv CICE_ACCT ${account_name} -endif diff --git a/configuration/scripts/machines/env.fram_intel b/configuration/scripts/machines/env.fram_intel index 35de927e7..6ca8285bc 100755 --- a/configuration/scripts/machines/env.fram_intel +++ b/configuration/scripts/machines/env.fram_intel @@ -24,8 +24,3 @@ setenv ICE_MACHINE_TPNODE 36 setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat " - -if (-e ~/.cice_proj) then - set account_name = `head -1 ~/.cice_proj` - setenv CICE_ACCT ${account_name} -endif diff --git a/configuration/scripts/machines/env.gpsc3_intel b/configuration/scripts/machines/env.gpsc3_intel new file mode 100644 index 000000000..2c8d49275 --- /dev/null +++ b/configuration/scripts/machines/env.gpsc3_intel @@ -0,0 +1,37 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# SSM package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler +source $ssmuse -d /fs/ssm/main/intelcomp/intelpsxe-cluster-19.0.3.199 +setenv FOR_DUMP_CORE_FILE 1 +# OpenMPI +source $ssmuse -d /fs/ssm/hpco/exp/openmpi/openmpi-3.1.2--hpcx-2.4.0-mofed-4.6--csh--intel-19.0.3.199 +source $ssmuse -d /fs/ssm/main/opt/openmpi-setup/openmpi-setup-0.3 +# NetCDF +source $ssmuse -d /fs/ssm/hpco/exp/hdf5-netcdf4/serial/static/intel-19.0.3.199/02 + +endif + +setenv ICE_MACHINE_MACHNAME gpsc3 +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site3/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site3/cice/baseline/ +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 44 +setenv ICE_MACHINE_ACCT "eccc_cmdd" +setenv ICE_MACHINE_QUEUE "standard" +setenv ICE_MACHINE_BLDTHRDS 24 +setenv ICE_MACHINE_QSTAT "squeue" + +setenv SLURM_EXPORT_ENV "ALL" diff --git a/configuration/scripts/machines/env.millikan_intel b/configuration/scripts/machines/env.millikan_intel deleted file mode 100755 index 350ea5b6e..000000000 --- a/configuration/scripts/machines/env.millikan_intel +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/csh -f - -set ssmuse=/fs/ssm/main/env/20180430/all/bin/ssmuse-csh # package loader -source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156 # intel compiler -source /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156/intelcomp_2016.1.156_multi/bin/compilervars.csh intel64 # should be sourced by above domain, but bug in csh script -source $ssmuse -d /fs/ssm/main/opt/openmpi/openmpi-1.6.5/intelcomp-2016.1.156 # openmpi -source $ssmuse -d /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150 # netcdf (and openmpi) - -# May be needed for OpenMP memory -#setenv OMP_STACKSIZE 64M - -setenv ICE_MACHINE_MACHNAME millikan -setenv ICE_MACHINE_MACHINFO "millikan" -setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_ENVINFO "intel 2016.1.156, openmpi 1.6.5, netcdf" -setenv ICE_MACHINE_MAKE make -setenv ICE_MACHINE_WKDIR /users/dor/armn/amb/data/local/runs -setenv ICE_MACHINE_INPUTDATA /users/dor/armn/amb/data/local/forcing -setenv ICE_MACHINE_BASELINE /users/dor/armn/amb/data/local/baseline -setenv ICE_MACHINE_SUBMIT "qsub" -setenv ICE_MACHINE_TPNODE 36 -setenv ICE_MACHINE_ACCT P0000000 -setenv ICE_MACHINE_QUEUE "debug" -setenv ICE_MACHINE_BLDTHRDS 4 -setenv ICE_MACHINE_QSTAT "qstat " - -if (-e ~/.cice_proj) then - set account_name = `head -1 ~/.cice_proj` - setenv CICE_ACCT ${account_name} -endif diff --git a/configuration/scripts/machines/env.ppp3_intel b/configuration/scripts/machines/env.ppp3_intel new file mode 100644 index 000000000..cf9fbe4a7 --- /dev/null +++ b/configuration/scripts/machines/env.ppp3_intel @@ -0,0 +1,35 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# SSM package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler +source $ssmuse -d comm/eccc/all/opt/intelcomp/intelpsxe-cluster-19.0.3.199 +setenv FOR_DUMP_CORE_FILE 1 +# OpenMPI +source $ssmuse -d /fs/ssm/hpco/exp/openmpi/openmpi-3.1.2--hpcx-2.4.0-mofed-4.6--csh--intel-19.0.3.199 +source $ssmuse -d main/opt/openmpi-setup/openmpi-setup-0.3 +# NetCDF +source $ssmuse -d hpco/exp/hdf5-netcdf4/serial/static/intel-19.0.3.199/02 + +endif + +setenv ICE_MACHINE_MACHNAME ppp3 +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site3/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site3/cice/baseline/ +setenv ICE_MACHINE_SUBMIT "jobsub" +setenv ICE_MACHINE_TPNODE 36 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "development" +setenv ICE_MACHINE_BLDTHRDS 24 +setenv ICE_MACHINE_QSTAT "jobctl-qstat " diff --git a/configuration/scripts/machines/env.ppp5_intel b/configuration/scripts/machines/env.ppp5_intel new file mode 100644 index 000000000..79dbf2a1b --- /dev/null +++ b/configuration/scripts/machines/env.ppp5_intel @@ -0,0 +1,38 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# Package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler + Intel MPI +source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# module use /home/sice500/modulefiles +# module load -s icc mpi +setenv FOR_DUMP_CORE_FILE 1 +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp5 +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/ppp5/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall5/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/ppp5/cice/baselines/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.ppp6_gnu b/configuration/scripts/machines/env.ppp6_gnu new file mode 100644 index 000000000..39cc27740 --- /dev/null +++ b/configuration/scripts/machines/env.ppp6_gnu @@ -0,0 +1,29 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# OpenMPI +source /usr/mpi/gcc/openmpi-4.1.2a1/bin/mpivars.csh + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp6 +setenv ICE_MACHINE_ENVNAME gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.ppp6_gnu-impi b/configuration/scripts/machines/env.ppp6_gnu-impi new file mode 100644 index 000000000..f2a523bf1 --- /dev/null +++ b/configuration/scripts/machines/env.ppp6_gnu-impi @@ -0,0 +1,38 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# GNU compilers + Intel MPI +source $ssmuse -x /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2 +setenv I_MPI_DEBUG_COREDUMP 1 +setenv I_MPI_F77 gfortran +setenv I_MPI_F90 gfortran +setenv I_MPI_FC gfortran +setenv I_MPI_CC gcc +setenv I_MPI_CXX g++ + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp6 +setenv ICE_MACHINE_ENVNAME gnu-impi +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.ppp6_intel b/configuration/scripts/machines/env.ppp6_intel new file mode 100644 index 000000000..dfaeb855f --- /dev/null +++ b/configuration/scripts/machines/env.ppp6_intel @@ -0,0 +1,38 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# Package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler + Intel MPI +source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# module use /home/sice500/modulefiles +# module load -s icc mpi +setenv FOR_DUMP_CORE_FILE 1 +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp6 +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/ppp6/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/ppp6/cice/baselines/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.ppp6_intel19 b/configuration/scripts/machines/env.ppp6_intel19 new file mode 100644 index 000000000..d41242630 --- /dev/null +++ b/configuration/scripts/machines/env.ppp6_intel19 @@ -0,0 +1,39 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler +source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelpsxe-cluster-19.0.3.199 +setenv FOR_DUMP_CORE_FILE 1 +# Intel MPI +source $ssmuse -d /fs/ssm/hpco/exp/intelpsxe-impi-19.0.3.199 +setenv FI_PROVIDER verbs +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d hpco/exp/hdf5-netcdf4/serial/static/intel-19.0.3.199/02 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME ppp6 +setenv ICE_MACHINE_ENVNAME intel19 +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/site6/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/site6/cice/baseline/ +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_BLDTHRDS 24 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.robert_intel b/configuration/scripts/machines/env.robert_intel new file mode 100644 index 000000000..43c11a529 --- /dev/null +++ b/configuration/scripts/machines/env.robert_intel @@ -0,0 +1,39 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# Package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler + Intel MPI +source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# module use /home/sice500/modulefiles +# module load -s icc mpi +setenv FOR_DUMP_CORE_FILE 1 +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME robert +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/robert/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall6/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/robert/cice/baselines/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat diff --git a/configuration/scripts/machines/env.underhill_intel b/configuration/scripts/machines/env.underhill_intel new file mode 100644 index 000000000..90192853e --- /dev/null +++ b/configuration/scripts/machines/env.underhill_intel @@ -0,0 +1,39 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +# Package loader +source /fs/ssm/main/opt/ssmuse/ssmuse-1.11/ssmuse_1.11_all/bin/ssmuse-boot.csh >& /dev/null +set ssmuse=`which ssmuse-csh` +# Intel compiler + Intel MPI +source $ssmuse -d /fs/ssm/main/opt/intelcomp/inteloneapi-2022.1.2/intelcomp+mpi+mkl +# module use /home/sice500/modulefiles +# module load -s icc mpi +setenv FOR_DUMP_CORE_FILE 1 +setenv I_MPI_DEBUG_COREDUMP 1 +# NetCDF +source $ssmuse -d main/opt/hdf5-netcdf4/serial/shared/inteloneapi-2022.1.2/01 + +# OpenMP +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME underhill +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/underhill/cice/runs/ +setenv ICE_MACHINE_INPUTDATA /space/hall5/sitestore/eccc/cmd/e/sice500/ +setenv ICE_MACHINE_BASELINE ~/data/underhill/cice/baselines/ +setenv ICE_MACHINE_SUBMIT qsub +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_TPNODE 80 +setenv ICE_MACHINE_ACCT unused +setenv ICE_MACHINE_QUEUE development +setenv ICE_MACHINE_BLDTHRDS 40 +setenv ICE_MACHINE_QSTAT qstat