diff --git a/.gitmodules b/.gitmodules index 34d37ff395..29d0969982 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,15 @@ [submodule "FV3"] path = FV3 - url = https://github.com/NOAA-EMC/fv3atm - branch = develop + #url = https://github.com/NOAA-EMC/fv3atm + #branch = develop + url = https://github.com/climbfuji/fv3atm + branch = remove_cheyenne_pgi [submodule "NEMS"] path = NEMS - url = https://github.com/NOAA-EMC/NEMS - branch = develop + #url = https://github.com/NOAA-EMC/NEMS + #branch = develop + url = https://github.com/climbfuji/NEMS + branch = add_stampede_config [submodule "FMS"] path = FMS url = https://github.com/NOAA-GFDL/FMS diff --git a/FV3 b/FV3 index 6590d4bd22..925f9928bb 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 6590d4bd225651b4570570eec287016264873a29 +Subproject commit 925f9928bbade62ff8645dc933e361dc355d9b9c diff --git a/NEMS b/NEMS index 19844f9c1e..91f35ec187 160000 --- a/NEMS +++ b/NEMS @@ -1 +1 @@ -Subproject commit 19844f9c1e6e6993c4cdf920f6f226db847d7b9b +Subproject commit 91f35ec187bc9700fc0d0d95e7df3c4ee44ca7c0 diff --git a/cmake/configure_stampede.intel.cmake b/cmake/configure_stampede.intel.cmake new file mode 100644 index 0000000000..534d580aa7 --- /dev/null +++ b/cmake/configure_stampede.intel.cmake @@ -0,0 +1,35 @@ +message("") +message("Setting configuration for $ENV{CMAKE_Platform}") +message("") + +get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME) +get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) +get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME) +message("C compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})") +message("CXX compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})") +message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})") +message("") + +option(DEBUG "Enable DEBUG mode" OFF) +option(REPRO "Enable REPRO mode" OFF) +option(VERBOSE "Enable VERBOSE mode" OFF) +option(32BIT "Enable 32BIT (single precision arithmetic in dycore)" OFF) +option(OPENMP "Enable OpenMP threading" ON) +option(AVX2 "Enable AVX2 instruction set" ON) + +option(INLINE_POST "Enable inline post" OFF) + +include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake ) + +set(NEMSIO_INC $ENV{NEMSIO_INC}) +set(POST_INC $ENV{POST_INC}) +set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB}) + +set(ESMF_MOD ${ESMF_F90COMPILEPATHS}) +set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}") + +set(NETCDF_INC_DIR $ENV{NETCDF}/include) +set(NETCDF_LIBDIR $ENV{NETCDF}/lib) +set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf) + +message("") diff --git a/conf/before_components.mk b/conf/before_components.mk index d617d27ba0..6eee9d190c 100644 --- a/conf/before_components.mk +++ b/conf/before_components.mk @@ -18,14 +18,6 @@ CHOSEN_MODULE=$(BUILD_TARGET)/fv3 CONFIGURE_NEMS_FILE=configure.fv3.$(BUILD_TARGET) -# ---------------------------------------------------------------------- -# Exit for systems that are currently not supported -ifeq ($(BUILD_TARGET),theia.pgi) - $(error NEMSfv3gfs currently not supported on $(BUILD_TARGET)) -else ifeq ($(BUILD_TARGET),cheyenne.pgi) - $(error NEMSfv3gfs currently not supported on $(BUILD_TARGET)) -endif - # ---------------------------------------------------------------------- # Copy the executable and modules.nems files into the tests/ directory # if a TEST_BUILD_NAME is specified. diff --git a/conf/configure.fv3.cheyenne.pgi b/conf/configure.fv3.cheyenne.pgi deleted file mode 100644 index d54faa1d6d..0000000000 --- a/conf/configure.fv3.cheyenne.pgi +++ /dev/null @@ -1,193 +0,0 @@ -## NEMS configuration file -## -## Platform: Cheyenne -## Compiler: PGI with SGI MPT - -SHELL=/bin/sh - -################################################################################ -## Include the common configuration parts - -ifdef InNemsMakefile -include $(TOP)/conf/configure.nems.NUOPC -endif - -###################### PHYS_MODE ##### CHEM_MODE ############################### -# -# -# - -PHYS_MODE =compile -CHEM_MODE =compile -ifeq ($(PHYS_MODE),compile) - PHYS_LIB = $(TOP)/atmos/gsm/gsmphys - PHYS_INC = $(TOP)/atmos/gsm/gsmphys - PHYS_DIR = $(TOP)/atmos/gsm/gsmphys -endif -ifeq ($(CHEM_MODE),compile) - CHEM_LIB = $(TOP)/chem - CHEM_INC = $(TOP)/chem/gocart/src/Config/ - CHEM_DIR = $(TOP)/chem - CHEM_MOD = $(TOP)/chem/gocart/${ARCH}/include - ESMADIR = chem/gocart -endif - -############ -# commands # -############ -CPP = mpicc -E -SFC = pgfortran -SCC = pgcc -CCOMP = pgcc -DM_FC = mpif90 -f90=$(SFC) -DM_CC = mpicc -cc=$(SCC) -DMPI2_SUPPORT -FC = $(DM_FC) -CC = $(DM_CC) -DFSEEKO64_OK -LD = $(FC) - -######### -# flags # -######### -# default is 64-bit OpenMP non-hydrostatic build using AVX2 -DEBUG = -REPRO = -VERBOSE = -OPENMP = Y -AVX2 = Y -HYDRO = N -CCPP = N - -include $(ESMFMKFILE) -ESMF_INC = $(ESMF_F90COMPILEPATHS) - -NEMSIOINC = -I$(NEMSIO_INC) -NCEPLIBS = $(NEMSIO_LIB) $(BACIO_LIB4) $(SP_LIBd) $(W3EMC_LIBd) $(W3NCO_LIBd) - -############################################## -# Need to use at least GNU Make version 3.81 # -############################################## -need := 3.81 -ok := $(filter $(need),$(firstword $(sort $(MAKE_VERSION) $(need)))) -ifneq ($(need),$(ok)) -$(error Need at least make version $(need). Load module gmake/3.81) -endif - -NETCDF_ROOT = $(NETCDF) -INCLUDE = -I$(NETCDF_ROOT)/include $(NEMSIOINC) -NETCDF_INC = -I$(NETCDF_ROOT)/include -ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),) - NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf -else - NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf -endif - -FPPFLAGS := -Mpreprocess $(INCLUDE) -CFLAGS := $(INCLUDE) - -FFLAGS := $(INCLUDE) -Mcray=pointer -Kieee -byteswapio - -CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP -CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_INLINE_POST - -ifeq ($(HYDRO),Y) -CPPDEFS += -else -CPPDEFS += -DMOIST_CAPPA -DUSE_COND -endif - -ifeq ($(NAM_phys),Y) -CPPDEFS += -DNAM_phys -endif - -ifeq ($(32BIT),Y) -CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8 -FFLAGS += -i4 -r4 -else -FFLAGS += -i4 -r8 -Mfprelaxed=div -Mfprelaxed=sqrt -endif - -ifeq ($(AVX2),Y) -FFLAGS += -tp=haswell -CFLAGS += -tp=haswell -else -FFLAGS += -tp=x64 -CFLAGS += -tp=x64 -endif - -ifeq ($(MULTI_GASES),Y) -CPPDEFS += -DMULTI_GASES -endif - -FFLAGS_OPT = -O1 -FFLAGS_REPRO = -O1 -g -traceback -FFLAGS_DEBUG = -g -O0 -Ktrap=fp -Mbounds -traceback - -TRANSCENDENTALS := -FFLAGS_OPENMP = -mp -FFLAGS_VERBOSE = -v -V - -CFLAGS += -D__IFC - -CFLAGS_OPT = -O2 -CFLAGS_REPRO = -O2 -CFLAGS_OPENMP = -mp -CFLAGS_DEBUG = -O0 -g -traceback - -# Optional Testing compile flags. Mutually exclusive from DEBUG, REPRO, and OPT -# *_TEST will match the production if no new option(s) is(are) to be tested. -FFLAGS_TEST = -O3 -CFLAGS_TEST = -O2 - -LDFLAGS := -LDFLAGS_OPENMP := -mp -LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M - -# start with blank LIBS -LIBS := - -ifeq ($(REPRO),Y) -CPPDEFS += -DREPRO -CFLAGS += $(CFLAGS_REPRO) -FFLAGS += $(FFLAGS_REPRO) -FAST := -else ifeq ($(DEBUG),Y) -CPPDEFS += -DDEBUG -CFLAGS += $(CFLAGS_DEBUG) -FFLAGS += $(FFLAGS_DEBUG) -FAST := -else ifeq ($(TEST),Y) -CFLAGS += $(CFLAGS_TEST) -FFLAGS += $(FFLAGS_TEST) -FAST := -else -CFLAGS += $(CFLAGS_OPT) -FFLAGS += $(FFLAGS_OPT) -FAST := $(TRANSCENDENTALS) -endif - -ifeq ($(OPENMP),Y) -CPPDEFS += -DOPENMP -CFLAGS += $(CFLAGS_OPENMP) -FFLAGS += $(FFLAGS_OPENMP) -LDFLAGS += $(LDFLAGS_OPENMP) -endif - -ifeq ($(VERBOSE),Y) -CFLAGS += $(CFLAGS_VERBOSE) -FFLAGS += $(FFLAGS_VERBOSE) -LDFLAGS += $(LDFLAGS_VERBOSE) -endif - -ifeq ($(CCPP),Y) -CPPDEFS += -DCCPP -CFLAGS += -I$(PATH_CCPP)/include -FFLAGS += -I$(PATH_CCPP)/include -LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2 -endif - -LDFLAGS += $(LIBS) - -ifdef InNemsMakefile -FFLAGS += $(ESMF_INC) -EXTLIBS = $(NCEPLIBS) $(ESMF_LIB) $(LDFLAGS) $(NETCDF_LIB) -endif diff --git a/conf/configure.fv3.stampede.intel b/conf/configure.fv3.stampede.intel index 6a68807262..772bc4ea6a 100644 --- a/conf/configure.fv3.stampede.intel +++ b/conf/configure.fv3.stampede.intel @@ -71,9 +71,9 @@ NETCDF_ROOT = $(NETCDF) INCLUDE = -I$(NETCDF_ROOT)/include NETCDF_INC = -I$(NETCDF_ROOT)/include ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),) - NETCDF_LIB += -L$(NETCDF_ROOT)/lib -lnetcdff -lnetcdf + NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf else - NETCDF_LIB += -L$(NETCDF_ROOT)/lib -lnetcdff -lnetcdf + NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf endif FPPFLAGS := -fpp -Wp,-w $(INCLUDE) diff --git a/doc/README_CENTOS.txt b/doc/README_CENTOS.txt deleted file mode 100644 index d819ac4c79..0000000000 --- a/doc/README_CENTOS.txt +++ /dev/null @@ -1,235 +0,0 @@ -# Dom Heinzeller (dom.heinzeller@noaa.gov), 08/21/2019 - -In order to build and run the FV3 trunk (August 2019) with possible CCPP extensions by GMTB on CentOS Linux, -the following installation steps are recommended. The version numbers correspond to the default versions in -August 2019 and will change to newer versions in the future. Unless problems occur during the manual builds in -step 4, these differences can be ignored. It is also assumed that the bash shell is used in the following. - -1. Install CentOS 7 (minimal install or more) or start up cloud instance (e.g. Amazon AWS) with Centos 7 - -2. Install standard packages as root (su) - - yum update - yum install -y net-tools - yum install -y gcc-gfortran - yum install -y gcc-c++ - yum install -y screen - yum install -y wget - yum install -y cmake - yum install -y bzip2 - yum install -y texinfo - yum install -y autogen - yum install -y dejagnu - yum install -y byacc - yum install -y curl-devel - yum install -y m4 - yum install -y git - yum install -y libxml2-devel - -3. Install thirdparty libraries - - mkdir -p /usr/local/src && cd /usr/local/src - - # gcc-8.3.0 - wget http://ftp.mirrorservice.org/sites/sourceware.org/pub/gcc/releases/gcc-8.3.0/gcc-8.3.0.tar.gz - tar -xvf gcc-8.3.0.tar.gz - cd gcc-8.3.0 - ./contrib/download_prerequisites # if this hangs, need to change the ftp URLs to https URLs in the script (behind firewall?) - ./configure \ - --disable-multilib \ - --enable-languages=c,c++,fortran \ - --prefix=/usr/local 2>&1 | tee log.config - make 2>&1 | tee log.make - make install 2>&1 | tee log.install - cd .. - rm -fr gcc-8.3.0 - - export LD_LIBRARY_PATH="/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH" - - # mpich-3.3.1 - wget http://www.mpich.org/static/downloads/3.3.1/mpich-3.3.1.tar.gz - tar -xvzf mpich-3.3.1.tar.gz - cd mpich-3.3.1 - ./configure \ - --prefix=/usr/local 2>&1 | tee log.config - make 2>&1 | tee log.make - make install 2>&1 | tee log.install - cd .. - rm -fr mpich-3.3.1 - - # netcdf-4.7.0 - wget https://www.zlib.net/zlib-1.2.11.tar.gz - wget https://support.hdfgroup.org/ftp/lib-external/szip/2.1.1/src/szip-2.1.1.tar.gz - # go to https://www.hdfgroup.org/downloads/hdf5/source-code/ and download hdf5-1.10.5.tar.bz2 - wget https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-c-4.7.0.tar.gz - wget https://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-fortran-4.4.5.tar.gz - # - tar -xvzf zlib-1.2.11.tar.gz - cd zlib-1.2.11 - ./configure \ - --prefix=/usr/local \ - --static 2>&1 | tee log.config - make 2>&1 | tee log.make - make install 2>&1 | tee log.install - make distclean 2>&1 | tee log.distclean - ./configure \ - --prefix=/usr/local 2>&1 | tee log.config - make 2>&1 | tee log.make - make install 2>&1 | tee log.install - cd .. - rm -fr zlib-1.2.11 - # - tar -xvzf szip-2.1.1.tar.gz - cd szip-2.1.1 - ./configure \ - --prefix=/usr/local 2>&1 | tee log.config - make 2>&1 | tee log.make - make install 2>&1 | tee log.install - cd .. - rm -fr szip-2.1.1 - # - tar -xvjf hdf5-1.10.5.tar.bz2 - cd hdf5-1.10.5 - CC=mpicc \ - CXX=mpicxx \ - FC=mpif90 \ - ./configure \ - --prefix=/usr/local \ - --enable-parallel \ - --with-zlib=/usr/local \ - --with-szlib=/usr/local 2>&1 | tee log.config - make 2>&1 | tee log.make - make install 2>&1 | tee log.install - cd .. - rm -fr hdf5-1.10.5 - # - tar -xvzf netcdf-c-4.7.0.tar.gz - cd netcdf-c-4.7.0 - CC=mpicc \ - CXX=mpicxx \ - FC=mpif90 \ - ./configure \ - --prefix=/usr/local \ - --enable-parallel-tests 2>&1 | tee log.config - make 2>&1 | tee log.make - make install 2>&1 | tee log.install - cd .. - rm -fr netcdf-c-4.7.0 - # - tar -xvzf netcdf-fortran-4.4.5.tar.gz - cd netcdf-fortran-4.4.5 - CC=mpicc \ - CXX=mpicxx \ - FC=mpif90 \ - ./configure \ - --prefix=/usr/local \ - --enable-parallel-tests 2>&1 | tee log.config - make 2>&1 | tee log.make - make install 2>&1 | tee log.install - # - export NETCDF=/usr/local - - # NCEP libraries - git clone https://github.com/NCAR/NCEPlibs.git - mv NCEPlibs NCEPlibs-20190820 - cd NCEPlibs-20190820 - mkdir /usr/local/NCEPlibs-20190820 - ./make_ncep_libs.sh -s linux -c gnu -d /usr/local/NCEPlibs-20190820 -o 1 - cd .. - rm -fr NCEPlibs-20190820 - - export NCEPLIBS_DIR=/usr/local/NCEPlibs-20190820 - - # Download ESMF_8_0_0_beta_snapshot_50 from https://sourceforge.net/p/esmf/esmf/ref/master/tags/ - # to /home/ubuntu/src (creates a directory esmf-esmf-...), rename it to esmf-8.0.0_bs50 and tar it - # up for later use - cd /usr/local/src - mv esmf-esmf-... esmf-8.0.0_bs50 - tar -cvzf esmf-8.0.0_bs50.tar.gz esmf-8.0.0_bs50 - # - # Install esmf-8.0.0_bs50 - tar -xvzf esmf-8.0.0_bs50.tar.gz - cd esmf-8.0.0_bs50 - export ESMF_DIR=`pwd` - export ESMF_INSTALL_PREFIX=/usr/local/esmf-8.0.0_bs50 - export ESMF_CXXCOMPILER=mpicxx - export ESMF_CXXLINKER=mpicxx - export ESMF_F90COMPILER=mpif90 - export ESMF_F90LINKER=mpif90 - export ESMF_COMM=mpich3 - export ESMF_MPIRUN=mpiexec - export ESMF_NETCDF=nc-config - export ESMF_INSTALL_BINDIR=bin - export ESMF_INSTALL_LIBDIR=lib - export ESMF_INSTALL_MODDIR=mod - make info 2>&1 | tee log.info - make 2>&1 | tee log.make - # "make check" is optional and can take very long time - make check 2>&1 | tee log.check - # SYSTEM TESTS SUMMARY - # Found 45 multi-processor system tests, 44 passed and 1 failed. - # UNIT TESTS SUMMARY - # Found 3466 non-exhaustive multi-processor unit tests, 3466 passed and 0 failed. - # --> ignore and proceed - make install 2>&1 | tee log.install - make installcheck 2>&1 | tee log.installcheck - cd .. - rm -fr esmf-8.0.0_bs50 - export -n ESMF_DIR - export -n ESMF_INSTALL_PREFIX - export -n ESMF_CXXCOMPILER - export -n ESMF_CXXLINKER - export -n ESMF_F90COMPILER - export -n ESMF_F90LINKER - export -n ESMF_COMM - export -n ESMF_MPIRUN - export -n ESMF_NETCDF - export -n ESMF_INSTALL_BINDIR - export -n ESMF_INSTALL_LIBDIR - export -n ESMF_INSTALL_MODDIR - - export ESMFMKFILE=/usr/local/esmf-8.0.0_bs50/lib/esmf.mk - -4. Compile NEMSfv3gfs as normal user - - mkdir ~/scratch - - # clone this branch of NEMSfv3gfs to ~/scratch/NEMSfv3gfs - - cd ~/scratch/NEMSfv3gfs/tests - - export LD_LIBRARY_PATH="/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH" - export NCEPLIBS_DIR=/usr/local/NCEPlibs-20190820 - export NETCDF=/usr/local - export ESMFMKFILE=/usr/local/esmf-8.0.0_bs50/lib/esmf.mk - export CC=mpicc - export CXX=mpicxx - export F77=mpif77 - export F90=mpif90 - export FC=mpif90 - - ./compile.sh $PWD/../FV3 linux.gnu 'CCPP=N' 2>&1 | tee log.compile # without CCPP - ./compile.sh $PWD/../FV3 linux.gnu 'CCPP=Y' 2>&1 | tee log.compile # with CCPP, dynamic mode - ./compile.sh $PWD/../FV3 linux.gnu 'CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp' 2>&1 | tee log.compile # with CCPP, static mode, GFS suite - -5. Set up the run directory using the template on Theia or Cheyenne at some location on your machine: - - a) copy the contents of the run directory templates to where you want to run the model, change to this directory - (these folders are read-only, i.e. users might have to add the write-flag after copying/rsyncing them) - - theia: /scratch4/BMC/gmtb/Dom.Heinzeller/rundirs/20190811/linux/fv3_gfdlmp/ - cheyenne: /glade/p/ral/jntp/GMTB/NEMSfv3gfs/rundirs/20190811/linux/fv3_gfdlmp/ - - b) edit run_linux.sh, set variables and change the variable FV3_BUILD_DIR to the top-level directory of your FV3-build - - c) set the environment variables as above (consider creating a shell script to source?) and run the model - - export NCEPLIBS_DIR=/usr/local/NCEPlibs-20190820 - export NETCDF=/usr/local - export ESMFMKFILE=/usr/local/esmf-8.0.0_bs50/lib/esmf.mk - - ./run_linux.sh 2>&1 | tee run_linux.log - # or, with N OpenMP threads (use N=1 for the dynamic CCPP build) - OMP_NUM_THREADS=N ./run_linux.sh 2>&1 | tee run_linux.log - - d) go and get yourself a cup of coffee ... diff --git a/doc/README_CHEYENNE.txt b/doc/README_CHEYENNE.txt deleted file mode 100644 index 333825d281..0000000000 --- a/doc/README_CHEYENNE.txt +++ /dev/null @@ -1,67 +0,0 @@ -################################################################################ -# Cheyenne/Intel -################################################################################ - -# OFFICIAL BASELINE -# /glade/p/ral/jntp/GMTB/NEMSfv3gfs/RT/trunk-20190912/INTEL - -# OWN BASELINE -# /glade/scratch/$USER/FV3_RT/REGRESSION_TEST_INTEL - -# RUN DIRS -# /glade/scratch/$USER/FV3_RT/rt_... - -export NEMS_COMPILER=intel # optional, default is intel if not set -export ACCNR=... # set account name/number for job scheduler - -# from the top-level directory of NEMSfv3gfs -cd tests - -# Test standard compile -./compile.sh $PWD/../FV3 cheyenne.intel 2>&1 | tee log.compile - -# Regression tests against official baseline -./rt.sh -f 2>&1 | tee rt.log - -# Create new baseline -./rt.sh -f -c 2>&1 | tee rt_create.log - -# Regression tests against new baseline -./rt.sh -f -m 2>&1 | tee rt_verify.log - -################################################################################ -# Cheyenne/GNU -################################################################################ - -# OFFICIAL BASELINE -# /glade/p/ral/jntp/GMTB/NEMSfv3gfs/RT/trunk-20190912/GNU - -# OWN BASELINE -# /glade/scratch/$USER/FV3_RT/REGRESSION_TEST_GNU - -# RUN DIRS -# /glade/scratch/$USER/FV3_RT/rt_... - -export NEMS_COMPILER=gnu -export ACCNR=... # set account name/number for job scheduler - -# from the top-level directory of NEMSfv3gfs -cd tests - -# Test standard compile -./compile.sh $PWD/../FV3 cheyenne.gnu 2>&1 | tee log.compile - -# Regression tests against official baseline -./rt.sh -l rt_gnu.conf 2>&1 | tee rt_gnu.log - -# Create new baseline -./rt.sh -l rt_gnu.conf -c 2>&1 | tee rt_gnu_create.log - -# Regression tests against new baseline -./rt.sh -l rt_gnu.conf -m 2>&1 | tee rt_gnu_verify.log - -################################################################################ -# Cheyenne/PGI -################################################################################ - -no longer supported diff --git a/doc/README_HERA.txt b/doc/README_HERA.txt deleted file mode 100644 index bf42209187..0000000000 --- a/doc/README_HERA.txt +++ /dev/null @@ -1,80 +0,0 @@ -################################################################################ -# Hera/Intel -################################################################################ - -# OFFICIAL BASELINE -# /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/trunk-20190824 - -# OWN BASELINE -# /scratch1/NCEPDEV/stmp4/$USER/FV3_RT - -# RUN DIRS - substitute $USER with your username -# /scratch1/NCEPDEV/stmp2/$USER/FV3_RT/rt_... - -export NEMS_COMPILER=intel # optional, default is intel if not set -export ACCNR=... # set account name/number for job scheduler - -# from the top-level directory of NEMSfv3gfs -cd tests - -# Test standard compile -./compile.sh $PWD/../FV3 hera.intel 2>&1 | tee log.compile - -# Regression tests against official baseline -./rt.sh -f 2>&1 | tee rt_full.log - -################################################################################ -# Hera/GNU -################################################################################ - -# OFFICIAL BASELINE -# N/A - -# OWN BASELINE -# /scratch1/NCEPDEV/stmp4/$USER/FV3_RT/REGRESSION_TEST_GNU - -# RUN DIRS -# /scratch1/NCEPDEV/stmp2/$USER/FV3_RT/rt_... - -export NEMS_COMPILER=gnu -export ACCNR=... # set account name/number for job scheduler - -# from the top-level directory of NEMSfv3gfs -cd tests - -# Test standard compile -./compile.sh $PWD/../FV3 hera.gnu 2>&1 | tee log.compile - -# Create new baseline -./rt.sh -f -l rt_gnu_pgi.conf -c fv3 2>&1 | tee rt_create_sub.log - -# Regression tests against new baseline -./rt.sh -f -l rt_gnu_pgi.conf -m 2>&1 | tee rt_sub.log - -################################################################################ -# Hera/PGI -################################################################################ - -# OFFICIAL BASELINE -# N/A - -# OWN BASELINE -# /scratch1/NCEPDEV/stmp4/$USER/FV3_RT/REGRESSION_TEST_PGI - -# RUN DIRS -# /scratch1/NCEPDEV/stmp2/$USER/FV3_RT/rt_... - -export NEMS_COMPILER=pgi -export ACCNR=... # set account name/number for job scheduler - -# from the top-level directory of NEMSfv3gfs -cd tests - -# Test standard compile -./compile.sh $PWD/../FV3 hera.pgi 2>&1 | tee log.compile - -# Create new baseline -./rt.sh -f -l rt_gnu_pgi.conf -c fv3 2>&1 | tee rt_create_sub.log - -# Regression tests against new baseline -./rt.sh -f -l rt_gnu_pgi.conf -m 2>&1 | tee rt_sub.log diff --git a/doc/README_MACOSX_clanggfortran.txt b/doc/README_MACOSX_clanggfortran.txt deleted file mode 100644 index b6effc1b6d..0000000000 --- a/doc/README_MACOSX_clanggfortran.txt +++ /dev/null @@ -1,229 +0,0 @@ -# Dom Heinzeller (dom.heinzeller@noaa.gov), 08/12/2019 - -Target systems: macOS High Sierra / macOS Mojave with LLVM clang + GNU gfortran compilers and mpich MPI library. - -In order to build and run the FV3 trunk (August 2019) with possible CCPP extensions by GMTB on macOS, -the following installation steps are recommended. The version numbers for the "brew" correspond to the default versions -in August 2019 and will change to newer versions in the future. Unless problems occur during the manual builds in -steps 6-11, these differences can be ignored. It is also assumed that the bash shell is used in the following. - -1. Install homebrew (enter sudo password when requested) - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - -2. Create /usr/local/src, /usr/local/esmf-8.0.0_bs50 and /usr/local/NCEPlibs-20190811. Change permissions to your user name / user group - - # change to root mode - sudo su - # /usr/local/src - mkdir /usr/local/src - chown YOUR_USERNAME /usr/local/src - chgrp YOUR_GROUPNAME /usr/local/src - # /usr/local/esmf-8.0.0_bs50 - mkdir /usr/local/esmf-8.0.0_bs50 - chown YOUR_USERNAME /usr/local/esmf-8.0.0_bs50 - chgrp YOUR_GROUPNAME /usr/local/esmf-8.0.0_bs50 - # /usr/local/NCEPlibs-20190811 - mkdir /usr/local/NCEPlibs-20190811 - chown YOUR_USERNAME /usr/local/NCEPlibs-20190811 - chgrp YOUR_GROUPNAME /usr/local/NCEPlibs-20190811 - # leave root mode - exit - -3. Use homebrew to install the following packages - - a) Install gcc-9.1.0, gfortran-9.1.0 - brew install gcc@9 - - b) Install clang-8.0.1 with openmp support - brew install llvm - - c) Install mpich-3.3.1 - brew install mpich - - d) Install netcdf-4.6.3 - brew install netcdf - - e) Install libpng-1.6.37 - brew install libpng - - f) Install udunits-2.2.27 - brew install udunits - - g) Install cmake-3.15.2 - brew install cmake - - h) Install xquartz-2.7.11 (optional, only needed for ncview) - brew cask install xquartz - - i) Install ncview-2.1.7 (optional) - brew install ncview - -4. Create a shell setup script ~/setenv_develop_nemsfv3gfs.sh to set the required paths for compiling NEMSfv3gfs. Contents: - -#################################################################################### -#!/bin/bash - -echo "Setting environment variables for develop-NEMSfv3gfs" - -export PATH="/usr/local/opt/llvm/bin:$PATH" -export CPPFLAGS="-I/usr/local/opt/llvm/include" -export LDFLAGS="-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib" -export LIBS_OPENMP="-L/usr/local/opt/llvm/lib -lomp" - -export CC=/usr/local/opt/llvm/bin/clang -export CXX=/usr/local/opt/llvm/bin/clang++ -export FC=/usr/local/bin/gfortran -export F90=/usr/local/bin/gfortran -export F77=/usr/local/bin/gfortran - -export MPICC="mpicc -cc=${CC}" -export MPICXX="mpicxx -cxx=${CXX}" -export MPIFORT=/usr/local/bin/mpifort -export MPIF77=/usr/local/bin/mpif77 -export MPIF90=/usr/local/bin/mpif90 - -export HDF5=/usr/local -export NETCDF=/usr/local -export ESMFMKFILE=/usr/local/esmf-8.0.0_bs50/lib/esmf.mk -export NCEPLIBS_DIR=/usr/local/NCEPlibs-20190811 -export MKL_DIR=/opt/intel/compilers_and_libraries_2019.4.233/mac/mkl - -#################################################################################### - -5. Source this shell script for compiling the libraries and the model below - . ~/setenv_develop_nemsfv3gfs.sh - -6. Install ESMF 8.0.0_bs50 - - # Download ESMF_8_0_0_beta_snapshot_50 from https://sourceforge.net/p/esmf/esmf/ref/master/tags/ - # to /usr/local/src (creates a directory esmf-esmf-...), rename it to esmf-8.0.0_bs50 and tar it - # up for later use - cd /usr/local/src - mv esmf-esmf-... esmf-8.0.0_bs50 - tar -cvzf esmf-8.0.0_bs50.tar.gz esmf-8.0.0_bs50 - - # Compile and install ESMF - cd esmf-8.0.0_bs50 - export ESMF_DIR=`pwd` - export ESMF_COMPILER=gfortranclang - export ESMF_CXXCOMPILER=$MPICXX - export ESMF_CXXLINKER=$MPICXX - export ESMF_F90COMPILER=$MPIF90 - export ESMF_F90LINKER=$MPIF90 - export ESMF_BOPT=O - export ESMF_OPTLEVEL=2 - export ESMF_COMM=mpich - export ESMF_MPIRUN=mpirun - export ESMF_NETCDF=1 - export ESMF_NETCDF_INCLUDE=$NETCDF/include - export ESMF_NETCDF_LIBPATH=$NETCDF/lib - export ESMF_NETCDF_LIBS="-lnetcdff -lnetcdf -lmpichf90" - export ESMF_NETCDF=split - export ESMF_INSTALL_PREFIX=/usr/local/esmf-8.0.0_bs50 - export ESMF_INSTALL_BINDIR=bin - export ESMF_INSTALL_LIBDIR=lib - export ESMF_INSTALL_MODDIR=mod - # - make info 2>&1 | tee log.info - make 2>&1 | tee log.make - # "make check" is optional and can take very long time - make check 2>&1 | tee log.check - # SYSTEM TESTS SUMMARY - # Found 45 multi-processor system tests, 38 passed and 7 failed. - # UNIT TESTS SUMMARY - # Found 3466 non-exhaustive multi-processor unit tests, 3394 passed and 72 failed. - # --> ignore and proceed - make install 2>&1 | tee log.install - make installcheck 2>&1 | tee log.installcheck - # - cd $ESMF_INSTALL_PREFIX - # - # Fix wrong path to libesmf.dylib in ESMF binaries - this will hopefully be addressed in future ESMF releases - # - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortranclang.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_Info - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortranclang.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_InfoC - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortranclang.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_WebServController - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortranclang.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_Regrid - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortranclang.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_RegridWeightGen - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortranclang.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_Scrip2Unstruct - # - # Fix wrong ID in libesmf.dylib - this will hopefully be addressed in future ESMF releases - # - install_name_tool -id $ESMF_INSTALL_PREFIX/lib/libesmf.dylib lib/libesmf.dylib - install_name_tool -id $ESMF_INSTALL_PREFIX/lib/libesmf_fullylinked.dylib lib/libesmf_fullylinked.dylib - # - # Clean up - cd /usr/local/src - rm -fr esmf-8.0.0_bs50 - export -n ESMF_DIR - export -n ESMF_COMPILER - export -n ESMF_CXXCOMPILER - export -n ESMF_CXXLINKER - export -n ESMF_F90COMPILER - export -n ESMF_F90LINKER - export -n ESMF_BOPT - export -n ESMF_OPTLEVEL - export -n ESMF_COMM - export -n ESMF_MPIRUN - export -n ESMF_NETCDF - export -n ESMF_NETCDF_INCLUDE - export -n ESMF_NETCDF_LIBPATH - export -n ESMF_NETCDF_LIBS - export -n ESMF_NETCDF - export -n ESMF_INSTALL_PREFIX - export -n ESMF_INSTALL_BINDIR - export -n ESMF_INSTALL_LIBDIR - export -n ESMF_INSTALL_MODDIR - -7. Build external NCEP libraries (use date tag 20190811 to allow for different versions in the future) - - # Obtain source code from gitub and build in /usr/local/src - cd /usr/local/src - git clone https://github.com/NCAR/NCEPlibs.git NCEPlibs-20190811 - cd NCEPlibs-20190811 - # Requires exporting CC, F90, MPIF90 (done by setenv_develop_nemsfv3gfs.sh) - ./make_ncep_libs.sh -s macosx -c gnu -d /usr/local/NCEPlibs-20190811 -o 1 2>&1 | tee log.make - -8. Download and install Intel Math Kernel Library MKL (full package) from https://software.intel.com/en-us/mkl - to /opt/intel (default location) using the installer script (requires sudo/root access) - -9. Pro-tip (optional): have your computer remember your GitHub username and password. - See https://help.github.com/en/articles/caching-your-github-password-in-git. - - git config --global credential.helper osxkeychain - -10. Download the model from GitHub: - cd $HOME - mkdir NEMSfv3gfs - cd NEMSfv3gfs - git clone --branch=gmtb/develop https://github.com/NCAR/NEMSfv3gfs NEMSfv3gfs-gmtb-develop-20190811 - cd NEMSfv3gfs-gmtb-develop-20190811 - git submodule init - git submodule update - -11. Build model. Change to top-level directory of NEMSfv3gfs-gmtb-develop-20190811 - - . ~/setenv_develop_nemsfv3gfs.sh - cd tests - # Note: omit '32BIT=Y' to compile dynamics in double precision (slower to run) - ./compile.sh $PWD/../FV3 macosx.gnu '32BIT=Y CCPP=N' 2>&1 | tee log.compile # without CCPP - ./compile.sh $PWD/../FV3 macosx.gnu '32BIT=Y CCPP=Y' 2>&1 | tee log.compile # with CCPP, dynamic mode - ./compile.sh $PWD/../FV3 macosx.gnu '32BIT=Y CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp' 2>&1 | tee log.compile # with CCPP, static mode, GFS suite - -12. Set up the run directory using the template on Theia or Cheyenne at some location on your machine: - - a) copy the contents of the run directory templates to where you want to run the model, change to this directory - (these folders are read-only, i.e. users might have to add the write-flag after copying/rsyncing them) - - theia: /scratch4/BMC/gmtb/Dom.Heinzeller/rundirs/20190811/macosx/fv3_gfdlmp/ - cheyenne: /glade/p/ral/jntp/GMTB/NEMSfv3gfs/rundirs/20190811/macosx/fv3_gfdlmp/ - - b) edit run_macosx.sh, set variables and change the variable FV3_BUILD_DIR to the top-level directory of your FV3-build - - c) source ~/setenv_develop_nemsfv3gfs.sh and execute the model run using the wrapper run_macosx.sh - . ~/setenv_develop_nemsfv3gfs.sh - ./run_macosx.sh 2>&1 | tee run_macosx.log - # or, with N OpenMP threads (use N=1 for the dynamic CCPP build) - OMP_NUM_THREADS=N ./run_macosx.sh 2>&1 | tee run_macosx.log - - d) go and get yourself a cup of coffee ... diff --git a/doc/README_MACOSX_gccgfortran.txt b/doc/README_MACOSX_gccgfortran.txt deleted file mode 100644 index 7d7b81e791..0000000000 --- a/doc/README_MACOSX_gccgfortran.txt +++ /dev/null @@ -1,221 +0,0 @@ -# Dom Heinzeller (dom.heinzeller@noaa.gov), 08/12/2019 - -Target systems: macOS High Sierra / macOS Mojave with GNU gcc+gfortran compilers and mpich MPI library. - -In order to build and run the FV3 trunk (August 2019) with possible CCPP extensions by GMTB on macOS, -the following installation steps are recommended. The version numbers for the "brew" correspond to the default versions -in August 2019 and will change to newer versions in the future. Unless problems occur during the manual builds in -steps 6-11, these differences can be ignored. It is also assumed that the bash shell is used in the following. - -1. Install homebrew (enter sudo password when requested) - /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - -2. Create /usr/local/src, /usr/local/esmf-8.0.0_bs50 and /usr/local/NCEPlibs-20190811. Change permissions to your user name / user group - - # change to root mode - sudo su - # /usr/local/src - mkdir /usr/local/src - chown YOUR_USERNAME /usr/local/src - chgrp YOUR_GROUPNAME /usr/local/src - # /usr/local/esmf-8.0.0_bs50 - mkdir /usr/local/esmf-8.0.0_bs50 - chown YOUR_USERNAME /usr/local/esmf-8.0.0_bs50 - chgrp YOUR_GROUPNAME /usr/local/esmf-8.0.0_bs50 - # /usr/local/NCEPlibs-20190811 - mkdir /usr/local/NCEPlibs-20190811 - chown YOUR_USERNAME /usr/local/NCEPlibs-20190811 - chgrp YOUR_GROUPNAME /usr/local/NCEPlibs-20190811 - # leave root mode - exit - -3. Use homebrew to install the following packages - - a) Install gcc-8.3.0, gfortran-8.3.0 - brew install gcc@8 - - b) Install mpich-3.3.1 - brew install mpich - - c) Install netcdf-4.6.3 - brew install netcdf - - d) Install libpng-1.6.37 - brew install libpng - - e) Install udunits-2.2.27 - brew install udunits - - f) Install cmake-3.15.2 - brew install cmake - - g) Install xquartz-2.7.11 (optional, only needed for ncview) - brew cask install xquartz - - h) Install ncview-2.1.7 (optional) - brew install ncview - -4. Create a shell setup script ~/setenv_develop_nemsfv3gfs.sh to set the required paths for compiling NEMSfv3gfs. Contents: - -#################################################################################### -#!/bin/bash - -echo "Setting environment variables for develop-NEMSfv3gfs" - -export CC=gcc-8 -export CXX=g++-8 -export FC=gfortran-8 -export F90=gfortran-8 -export F77=gfortran-8 - -export MPICC="mpicc -cc=$CC" -export MPICXX="mpicxx -cxx=$CXX" -export MPIFORT="mpifort -fc=$FC" -export MPIF77="mpif77 -fc=$F77" -export MPIF90="mpif90 -fc=$F90" - -export HDF5=/usr/local -export NETCDF=/usr/local -export ESMFMKFILE=/usr/local/esmf-8.0.0_bs50/lib/esmf.mk -export NCEPLIBS_DIR=/usr/local/NCEPlibs-20190811 -export MKL_DIR=/opt/intel/compilers_and_libraries_2019.4.233/mac/mkl - -#################################################################################### - -5. Source this shell script for compiling the libraries and the model below - . ~/setenv_develop_nemsfv3gfs.sh - -6. Install ESMF 8.0.0_bs50 - - # Download ESMF_8_0_0_beta_snapshot_50 from https://sourceforge.net/p/esmf/esmf/ref/master/tags/ - # to /usr/local/src (creates a directory esmf-esmf-...), rename it to esmf-8.0.0_bs50 and tar it - # up for later use - cd /usr/local/src - mv esmf-esmf-... esmf-8.0.0_bs50 - tar -cvzf esmf-8.0.0_bs50.tar.gz esmf-8.0.0_bs50 - - # Compile and install ESMF - cd esmf-8.0.0_bs50 - export ESMF_DIR=`pwd` - export ESMF_COMPILER=gfortran - export ESMF_CXXCOMPILER=$MPICXX - export ESMF_CXXLINKER=$MPICXX - export ESMF_F90COMPILER=$MPIF90 - export ESMF_F90LINKER=$MPIF90 - export ESMF_BOPT=O - export ESMF_OPTLEVEL=2 - export ESMF_COMM=mpich - export ESMF_MPIRUN=mpirun - export ESMF_NETCDF=1 - export ESMF_NETCDF_INCLUDE=$NETCDF/include - export ESMF_NETCDF_LIBPATH=$NETCDF/lib - export ESMF_NETCDF_LIBS="-lnetcdff -lnetcdf -lmpichf90" - export ESMF_NETCDF=split - export ESMF_INSTALL_PREFIX=/usr/local/esmf-8.0.0_bs50 - export ESMF_INSTALL_BINDIR=bin - export ESMF_INSTALL_LIBDIR=lib - export ESMF_INSTALL_MODDIR=mod - # - make info 2>&1 | tee log.info - make 2>&1 | tee log.make - # "make check" is optional and can take very long time - make check 2>&1 | tee log.check - # SYSTEM TESTS SUMMARY - # Found 45 multi-processor system tests, 38 passed and 7 failed. - # UNIT TESTS SUMMARY - # Found 3466 non-exhaustive multi-processor unit tests, 3394 passed and 72 failed. - # --> ignore and proceed - make install 2>&1 | tee log.install - make installcheck 2>&1 | tee log.installcheck - # - cd $ESMF_INSTALL_PREFIX - # - # Fix wrong path to libesmf.dylib in ESMF binaries - this will hopefully be addressed in future ESMF releases - # - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortran.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_Info - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortran.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_InfoC - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortran.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_WebServController - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortran.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_Regrid - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortran.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_RegridWeightGen - install_name_tool -change $ESMF_DIR/lib/libO/Darwin.gfortran.64.mpich.default/libesmf.dylib $ESMF_INSTALL_PREFIX/lib/libesmf.dylib bin/ESMF_Scrip2Unstruct - # - # Fix wrong ID in libesmf.dylib - this will hopefully be addressed in future ESMF releases - # - install_name_tool -id $ESMF_INSTALL_PREFIX/lib/libesmf.dylib lib/libesmf.dylib - install_name_tool -id $ESMF_INSTALL_PREFIX/lib/libesmf_fullylinked.dylib lib/libesmf_fullylinked.dylib - # - # Clean up - cd /usr/local/src - rm -fr esmf-8.0.0_bs50 - export -n ESMF_DIR - export -n ESMF_COMPILER - export -n ESMF_CXXCOMPILER - export -n ESMF_CXXLINKER - export -n ESMF_F90COMPILER - export -n ESMF_F90LINKER - export -n ESMF_BOPT - export -n ESMF_OPTLEVEL - export -n ESMF_COMM - export -n ESMF_MPIRUN - export -n ESMF_NETCDF - export -n ESMF_NETCDF_INCLUDE - export -n ESMF_NETCDF_LIBPATH - export -n ESMF_NETCDF_LIBS - export -n ESMF_NETCDF - export -n ESMF_INSTALL_PREFIX - export -n ESMF_INSTALL_BINDIR - export -n ESMF_INSTALL_LIBDIR - export -n ESMF_INSTALL_MODDIR - -7. Build external NCEP libraries (use date tag 20190811 to allow for different versions in the future) - - # Obtain source code from gitub and build in /usr/local/src - cd /usr/local/src - git clone https://github.com/NCAR/NCEPlibs.git NCEPlibs-20190811 - cd NCEPlibs-20190811 - # Requires exporting CC, F90, MPIF90 (done by setenv_develop_nemsfv3gfs.sh) - ./make_ncep_libs.sh -s macosx -c gnu -d /usr/local/NCEPlibs-20190811 -o 1 2>&1 | tee log.make - -8. Download and install Intel Math Kernel Library MKL (full package) from https://software.intel.com/en-us/mkl - to /opt/intel (default location) using the installer script (requires sudo/root access) - -9. Pro-tip (optional): have your computer remember your GitHub username and password. - See https://help.github.com/en/articles/caching-your-github-password-in-git. - - git config --global credential.helper osxkeychain - -10. Download the model from GitHub: - cd $HOME - mkdir NEMSfv3gfs - cd NEMSfv3gfs - git clone --branch=gmtb/develop https://github.com/NCAR/NEMSfv3gfs NEMSfv3gfs-gmtb-develop-20190811 - cd NEMSfv3gfs-gmtb-develop-20190811 - git submodule init - git submodule update - -11. Build model. Change to top-level directory of NEMSfv3gfs-gmtb-develop-20190811 - - . ~/setenv_develop_nemsfv3gfs.sh - cd tests - # Note: omit '32BIT=Y' to compile dynamics in double precision (slower to run) - ./compile.sh $PWD/../FV3 macosx.gnu '32BIT=Y CCPP=N' 2>&1 | tee log.compile # without CCPP - ./compile.sh $PWD/../FV3 macosx.gnu '32BIT=Y CCPP=Y' 2>&1 | tee log.compile # with CCPP, dynamic mode - ./compile.sh $PWD/../FV3 macosx.gnu '32BIT=Y CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp' 2>&1 | tee log.compile # with CCPP, static mode, GFS suite - -12. Set up the run directory using the template on Theia or Cheyenne at some location on your machine: - - a) copy the contents of the run directory templates to where you want to run the model, change to this directory - (these folders are read-only, i.e. users might have to add the write-flag after copying/rsyncing them) - - theia: /scratch4/BMC/gmtb/Dom.Heinzeller/rundirs/20190811/macosx/fv3_gfdlmp/ - cheyenne: /glade/p/ral/jntp/GMTB/NEMSfv3gfs/rundirs/20190811/macosx/fv3_gfdlmp/ - - b) edit run_macosx.sh, set variables and change the variable FV3_BUILD_DIR to the top-level directory of your FV3-build - - c) source ~/setenv_develop_nemsfv3gfs.sh and execute the model run using the wrapper run_macosx.sh - . ~/setenv_develop_nemsfv3gfs.sh - ./run_macosx.sh 2>&1 | tee run_macosx.log - # or, with N OpenMP threads (use N=1 for the dynamic CCPP build) - OMP_NUM_THREADS=N ./run_macosx.sh 2>&1 | tee run_macosx.log - - d) go and get yourself a cup of coffee ... diff --git a/doc/README_THEIA.txt b/doc/README_THEIA.txt deleted file mode 100644 index c58d815a4c..0000000000 --- a/doc/README_THEIA.txt +++ /dev/null @@ -1,80 +0,0 @@ -################################################################################ -# Theia/Intel -################################################################################ - -# OFFICIAL BASELINE -# /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20190315 - -# OWN BASELINE -# /scratch4/NCEPDEV/stmp4/$USER/FV3_RT/REGRESSION_TEST_INTEL - -# RUN DIRS - substitute $USER with your username -# /scratch4/NCEPDEV/stmp3/$USER/FV3_RT/rt_... - -export NEMS_COMPILER=intel # optional, default is intel if not set -export ACCNR=... # set account name/number for job scheduler - -# from the top-level directory of NEMSfv3gfs -cd tests - -# Test standard compile -./compile.sh $PWD/../FV3 theia.intel 2>&1 | tee log.compile - -# Regression tests against official baseline -./rt.sh -f 2>&1 | tee rt_full.log - -################################################################################ -# Theia/GNU -################################################################################ - -# OFFICIAL BASELINE -# N/A - -# OWN BASELINE -# /scratch4/NCEPDEV/stmp4/$USER/FV3_RT/REGRESSION_TEST_GNU - -# RUN DIRS -# /scratch4/NCEPDEV/stmp3/$USER/FV3_RT/rt_... - -export NEMS_COMPILER=gnu -export ACCNR=... # set account name/number for job scheduler - -# from the top-level directory of NEMSfv3gfs -cd tests - -# Test standard compile -./compile.sh $PWD/../FV3 theia.gnu 2>&1 | tee log.compile - -# Create new baseline -./rt.sh -f -l rt_gnu_pgi.conf -c fv3 2>&1 | tee rt_create_sub.log - -# Regression tests against new baseline -./rt.sh -f -l rt_gnu_pgi.conf -m 2>&1 | tee rt_sub.log - -################################################################################ -# Theia/PGI -################################################################################ - -# OFFICIAL BASELINE -# N/A - -# OWN BASELINE -# /scratch4/NCEPDEV/stmp4/$USER/FV3_RT/REGRESSION_TEST_PGI - -# RUN DIRS -# /scratch4/NCEPDEV/stmp3/$USER/FV3_RT/rt_... - -export NEMS_COMPILER=pgi -export ACCNR=... # set account name/number for job scheduler - -# from the top-level directory of NEMSfv3gfs -cd tests - -# Test standard compile -./compile.sh $PWD/../FV3 theia.pgi 2>&1 | tee log.compile - -# Create new baseline -./rt.sh -f -l rt_gnu_pgi.conf -c fv3 2>&1 | tee rt_create_sub.log - -# Regression tests against new baseline -./rt.sh -f -l rt_gnu_pgi.conf -m 2>&1 | tee rt_sub.log diff --git a/doc/README_UBUNTU.txt b/doc/README_UBUNTU.txt deleted file mode 100644 index 7e4ead9fbf..0000000000 --- a/doc/README_UBUNTU.txt +++ /dev/null @@ -1,135 +0,0 @@ -# Dom Heinzeller (dom.heinzeller@noaa.gov), 08/21/2019 - -In order to build and run the FV3 trunk (August 2019) with possible CCPP extensions by GMTB on Ubuntu Linux, -the following installation steps are recommended. The version numbers correspond to the default versions in -August 2019 and will change to newer versions in the future. Unless problems occur during the manual builds in -step 4, these differences can be ignored. It is also assumed that the bash shell is used in the following. - -1. Install Ubuntu Linux 18.04.1 LTS or start up cloud instance (e.g. Amazon AWS) with Ubuntu Linux 18.04.1 LTS - -2. Install standard packages as root (sudo su) - - apt update - apt install ssh - apt install gfortran libgfortran-7-dev - apt install g++ libstdc++-7-dev - apt install make - apt install cmake - apt install m4 - apt install ksh - apt install git - apt install python - apt install libxml2-dev - apt install libnetcdff-dev - apt install mpich - ln -s /usr/bin/make /usr/bin/gmake - - export NETCDF=/usr - -3. Install thirdparty libraries - - mkdir -p /usr/local/src && cd /usr/local/src - - # NCEP libraries - git clone https://github.com/NCAR/NCEPlibs.git - mv NCEPlibs NCEPlibs-20190820 - cd NCEPlibs-20190820 - mkdir /usr/local/NCEPlibs-20190820 - ./make_ncep_libs.sh -s linux -c gnu -d /usr/local/NCEPlibs-20190820 -o 1 - cd .. - rm -fr NCEPlibs-20190820 - - export NCEPLIBS_DIR=/usr/local/NCEPlibs-20190820 - - # Download ESMF_8_0_0_beta_snapshot_50 from https://sourceforge.net/p/esmf/esmf/ref/master/tags/ - # to /home/ubuntu/src (creates a directory esmf-esmf-...), rename it to esmf-8.0.0_bs50 and tar it - # up for later use - cd /usr/local/src - mv esmf-esmf-... esmf-8.0.0_bs50 - tar -cvzf esmf-8.0.0_bs50.tar.gz esmf-8.0.0_bs50 - # - # Install esmf-8.0.0_bs50 - tar -xvzf esmf-8.0.0_bs50.tar.gz - cd esmf-8.0.0_bs50 - export ESMF_DIR=`pwd` - export ESMF_INSTALL_PREFIX=/usr/local/esmf-8.0.0_bs50 - export ESMF_CXXCOMPILER=mpicxx - export ESMF_CXXLINKER=mpicxx - export ESMF_F90COMPILER=mpif90 - export ESMF_F90LINKER=mpif90 - export ESMF_COMM=mpich3 - export ESMF_MPIRUN=mpiexec - export ESMF_NETCDF=nc-config - export ESMF_INSTALL_BINDIR=bin - export ESMF_INSTALL_LIBDIR=lib - export ESMF_INSTALL_MODDIR=mod - make info 2>&1 | tee log.info - make 2>&1 | tee log.make - # "make check" is optional and can take very long time - make check 2>&1 | tee log.check - # SYSTEM TESTS SUMMARY - # Found 45 multi-processor system tests, 45 passed and 0 failed. - # UNIT TESTS SUMMARY - # Found 3466 non-exhaustive multi-processor unit tests, 3464 passed and 2 failed. - # --> ignore and proceed - make install 2>&1 | tee log.install - make installcheck 2>&1 | tee log.installcheck - cd .. - rm -fr esmf-8.0.0_bs50 - export -n ESMF_DIR - export -n ESMF_INSTALL_PREFIX - export -n ESMF_CXXCOMPILER - export -n ESMF_CXXLINKER - export -n ESMF_F90COMPILER - export -n ESMF_F90LINKER - export -n ESMF_COMM - export -n ESMF_MPIRUN - export -n ESMF_NETCDF - export -n ESMF_INSTALL_BINDIR - export -n ESMF_INSTALL_LIBDIR - export -n ESMF_INSTALL_MODDIR - - export ESMFMKFILE=/usr/local/esmf-8.0.0_bs50/lib/esmf.mk - -4. Compile NEMSfv3gfs as normal user - - mkdir ~/scratch - - # clone this branch of NEMSfv3gfs to ~/scratch/NEMSfv3gfs - - cd ~/scratch/NEMSfv3gfs/tests - - export NCEPLIBS_DIR=/usr/local/NCEPlibs-20190820 - export NETCDF=/usr - export ESMFMKFILE=/usr/local/esmf-8.0.0_bs50/lib/esmf.mk - export CC=mpicc - export CXX=mpicxx - export F77=mpif77 - export F90=mpif90 - export FC=mpif90 - - ./compile.sh $PWD/../FV3 linux.gnu 'CCPP=N' 2>&1 | tee log.compile # without CCPP - ./compile.sh $PWD/../FV3 linux.gnu 'CCPP=Y' 2>&1 | tee log.compile # with CCPP, dynamic mode - ./compile.sh $PWD/../FV3 linux.gnu 'CCPP=Y STATIC=Y SUITES=FV3_GFS_2017_gfdlmp' 2>&1 | tee log.compile # with CCPP, static mode, GFS suite - -5. Set up the run directory using the template on Theia or Cheyenne at some location on your machine: - - a) copy the contents of the run directory templates to where you want to run the model, change to this directory - (these folders are read-only, i.e. users might have to add the write-flag after copying/rsyncing them) - - theia: /scratch4/BMC/gmtb/Dom.Heinzeller/rundirs/20190811/linux/fv3_gfdlmp/ - cheyenne: /glade/p/ral/jntp/GMTB/NEMSfv3gfs/rundirs/20190811/linux/fv3_gfdlmp/ - - b) edit run_linux.sh, set variables and change the variable FV3_BUILD_DIR to the top-level directory of your FV3-build - - c) set the environment variables as above (consider creating a shell script to source?) and run the model - - export NCEPLIBS_DIR=/usr/local/NCEPlibs-20190820 - export NETCDF=/usr - export ESMFMKFILE=/usr/local/esmf-8.0.0_bs50/lib/esmf.mk - - ./run_linux.sh 2>&1 | tee run_linux.log - # or, with N OpenMP threads (use N=1 for the dynamic CCPP build) - OMP_NUM_THREADS=N ./run_linux.sh 2>&1 | tee run_linux.log - - d) go and get yourself a cup of coffee ... diff --git a/modulefiles/cheyenne.gnu/fv3 b/modulefiles/cheyenne.gnu/fv3 index 5dfa2e778f..ae7cc83c78 100644 --- a/modulefiles/cheyenne.gnu/fv3 +++ b/modulefiles/cheyenne.gnu/fv3 @@ -39,8 +39,9 @@ module load NCEPlibs/9.9.9 module load SIONlib/1.7.4 ## -## set cmake compilers +## load cmake ## +module load cmake/3.16.4 setenv CMAKE_C_COMPILER mpicc setenv CMAKE_CXX_COMPILER mpicxx setenv CMAKE_Fortran_COMPILER mpif90 diff --git a/modulefiles/cheyenne.intel-impi/fv3 b/modulefiles/cheyenne.intel-impi/fv3 index e0160b61dc..6813a383e3 100644 --- a/modulefiles/cheyenne.intel-impi/fv3 +++ b/modulefiles/cheyenne.intel-impi/fv3 @@ -39,8 +39,9 @@ module load NCEPlibs/9.9.9 module load SIONlib/1.7.4 ## -## set cmake compilers +## load cmake ## +module load cmake/3.16.4 setenv CMAKE_C_COMPILER mpicc setenv CMAKE_CXX_COMPILER mpicxx setenv CMAKE_Fortran_COMPILER mpif90 diff --git a/modulefiles/cheyenne.intel/fv3 b/modulefiles/cheyenne.intel/fv3 index f009c137d1..8abc6c1b69 100644 --- a/modulefiles/cheyenne.intel/fv3 +++ b/modulefiles/cheyenne.intel/fv3 @@ -39,8 +39,9 @@ module load NCEPlibs/9.9.9 module load SIONlib/1.7.4 ## -## set cmake compilers +## load cmake ## +module load cmake/3.16.4 setenv CMAKE_C_COMPILER mpicc setenv CMAKE_CXX_COMPILER mpicxx setenv CMAKE_Fortran_COMPILER mpif90 diff --git a/modulefiles/cheyenne.pgi/fv3 b/modulefiles/cheyenne.pgi/fv3 deleted file mode 100644 index c02e509b67..0000000000 --- a/modulefiles/cheyenne.pgi/fv3 +++ /dev/null @@ -1,48 +0,0 @@ -#%Module###################################################################### -## -## NEMS FV3 Prerequisites: Cheyenne/PGI - -proc ModulesHelp {} { - puts stderr "\tcit - loads modules required for building and running FV3 under NEMS on Cheyenne/PGI" -} - -module-whatis "loads NEMS FV3 prerequisites for Cheyenne/PGI" - -# NOTE: the "module purge" and loading of the module command are -# handled by the module-setup.sh (or .csh) script. - -## -## load programming environment -## this typically includes compiler, MPI and job scheduler -## -module load ncarenv/1.2 -module load pgi/17.9 -module load mpt/2.19 -module load ncarcompilers/0.4.1 -module load netcdf/4.4.1.1 - -## -## use pre-compiled EMSF library for pgi-17.9 and mpt-2.15f, because -## building it with mpt-2.19 was a hopeless undertaking -## -set ESMF_DIR /glade/p/ral/jntp/GMTB/tools/esmf-7.1.0r/pgi-17.9/mpt-2.15f -setenv ESMF_DIR ${ESMF_DIR} -setenv ESMF_INC ${ESMF_DIR}/include -setenv ESMF_BIN ${ESMF_DIR}/bin/binO/Linux.pgi.64.mpi.default -setenv ESMF_LIB ${ESMF_DIR}/lib/libO/Linux.pgi.64.mpi.default -setenv ESMF_MOD ${ESMF_DIR}/mod/modO/Linux.pgi.64.mpi.default -setenv ESMFMKFILE ${ESMF_DIR}/lib/libO/Linux.pgi.64.mpi.default/esmf.mk -prepend-path PATH ${ESMF_DIR}/bin/binO/Linux.pgi.64.mpi.default -prepend-path LD_LIBRARY_PATH ${ESMF_DIR}/lib/libO/Linux.pgi.64.mpi.default - -## -## NCEP libraries (download and build manually for the time being; https://github.com/NCAR/NCEPlibs) -## -set NCEPLIBS_DIR /glade/p/ral/jntp/GMTB/tools/NCEPlibs/20190307/pgi-17.9/mpt-2.19 -setenv NCEPLIBS_DIR ${NCEPLIBS_DIR} -setenv NEMSIO_INC ${NCEPLIBS_DIR}/include -setenv NEMSIO_LIB ${NCEPLIBS_DIR}/lib/libnemsio_d.a -setenv BACIO_LIB4 ${NCEPLIBS_DIR}/lib/libbacio_4.a -setenv SP_LIBd ${NCEPLIBS_DIR}/lib/libsp_v2.0.2_d.a -setenv W3EMC_LIBd ${NCEPLIBS_DIR}/lib/libw3emc_d.a -setenv W3NCO_LIBd ${NCEPLIBS_DIR}/lib/libw3nco_d.a diff --git a/modulefiles/stampede.intel/fv3 b/modulefiles/stampede.intel/fv3 index cc2fa9e88a..21fda14b51 100644 --- a/modulefiles/stampede.intel/fv3 +++ b/modulefiles/stampede.intel/fv3 @@ -15,36 +15,24 @@ module-whatis "loads NEMS FV3 prerequisites for Stampede/Intel" ## load programming environment ## this typically includes compiler, MPI and job scheduler ## -module load intel/17.0.4 -module load impi/17.0.3 -module load netcdf/4.3.3.1 -module load hdf5/1.8.16 -module load pnetcdf/1.8.1 -setenv NETCDF /opt/apps/intel17/netcdf/4.3.3.1/x86_64 +module load intel/18.0.2 +module load impi/18.0.2 +module load python2/2.7.15 +module load parallel-netcdf/4.6.2 +setenv NETCDF /opt/apps/intel18/impi18_0/parallel-netcdf/4.6.2/x86_64 ## -## use pre-compiled EMSF library for above compiler / MPI combination +## use pre-compiled EMSF and NCEP libraries from MRW public release above compiler / MPI combination ## # DH* todo - shared directory -set ESMF_DIR /home1/05294/tg845932/softwares/esmf_7_1_0r -setenv ESMF_DIR ${ESMF_DIR} -setenv ESMF_INC ${ESMF_DIR}/include -setenv ESMF_BIN ${ESMF_DIR}/bin/bin0/Linux.intel.64.intelmpi.default -setenv ESMF_LIB ${ESMF_DIR}/lib/lib0/Linux.intel.64.intelmpi.default -setenv ESMF_MOD ${ESMF_DIR}/mod/mod0/Linux.intel.64.intelmpi.default -setenv ESMFMKFILE ${ESMF_DIR}/lib/lib0/Linux.intel.64.intelmpi.default/esmf.mk -prepend-path PATH ${ESMF_DIR}/bin/bin0/Linux.intel.64.intelmpi.default -prepend-path LD_LIBRARY_PATH ${ESMF_DIR}/lib/lib0/Linux.intel.64.intelmpi.default +module use -a /work/06146/tg854455/stampede2/NCEPLIBS-ufs-v1.0.0/modules +module load NCEPLIBS/1.0.0 ## -## NCEP libraries (download and build manually for the time being; https://github.com/climbfuji/NCEPlibs) +## load cmake ## -# DH* todo - shared directory -set NCEPLIBS_DIR /home1/05294/tg845932/softwares/NCEPlibs -setenv NCEPLIBS_DIR ${NCEPLIBS_DIR} -setenv NEMSIO_INC ${NCEPLIBS_DIR}/include -setenv NEMSIO_LIB ${NCEPLIBS_DIR}/lib/libnemsio_d.a -setenv BACIO_LIB4 ${NCEPLIBS_DIR}/lib/libbacio_4.a -setenv SP_LIBd ${NCEPLIBS_DIR}/lib/libsp_v2.0.2_d.a -setenv W3EMC_LIBd ${NCEPLIBS_DIR}/lib/libw3emc_d.a -setenv W3NCO_LIBd ${NCEPLIBS_DIR}/lib/libw3nco_d.a +module load cmake/3.16.1 +setenv CMAKE_C_COMPILER mpiicc +setenv CMAKE_CXX_COMPILER mpiicpc +setenv CMAKE_Fortran_COMPILER mpiifort +setenv CMAKE_Platform stampede.intel diff --git a/tests/fv3_conf/compile_qsub.IN_cheyenne b/tests/fv3_conf/compile_qsub.IN_cheyenne new file mode 100644 index 0000000000..c7a5cf0833 --- /dev/null +++ b/tests/fv3_conf/compile_qsub.IN_cheyenne @@ -0,0 +1,16 @@ +#!/bin/sh -l +#PBS -o out +#PBS -e err +#PBS -N @[JBNME] +#PBS -A @[ACCNR] +#PBS -q @[QUEUE] +#PBS -l select=1:ncpus=8:mpiprocs=1 +#PBS -l walltime=00:30:00 + +set -eux + +echo "Compile started: " `date` + +@[PATHRT]/compile_cmake.sh @[PATHTR] @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR] + +echo "Compile ended: " `date` diff --git a/tests/fv3_conf/fv3_qsub.IN_stampede b/tests/fv3_conf/fv3_slurm.IN_stampede similarity index 100% rename from tests/fv3_conf/fv3_qsub.IN_stampede rename to tests/fv3_conf/fv3_slurm.IN_stampede diff --git a/tests/rt.sh b/tests/rt.sh index 0fd4f3ec94..abcea7f9c2 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -29,11 +29,11 @@ usage() { rt_trap() { [[ ${ROCOTO:-false} == true ]] && rocoto_kill - [[ ${ECFLOW:-false} == true ]] && ecflow_stop cleanup } cleanup() { + [[ ${ECFLOW:-false} == true ]] && ecflow_stop rm -rf ${LOCKDIR} trap 0 exit @@ -149,6 +149,7 @@ elif [[ $MACHINE_ID = gaea.* ]]; then DISKNM=/lustre/f2/pdata/esrl/gsd/ufs/ufs-weather-model/RT # *DH 20190717 QUEUE=debug + COMPILE_QUEUE=debug # DO NOT SET AN ACCOUNT EVERYONE IS NOT A MEMBER OF # USE AN ENVIRONMENT VARIABLE TO SET ACCOUNT # ACCNR=cmp @@ -271,7 +272,8 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then export PYTHONPATH=/glade/p/ral/jntp/tools/ecFlow-5.3.1/lib/python2.7/site-packages ECFLOW_START=/glade/p/ral/jntp/tools/ecFlow-5.3.1/bin/ecflow_start.sh ECF_PORT=$(( $(id -u) + 1500 )) - QUEUE=premium + QUEUE=regular + COMPILE_QUEUE=regular PARTITION= dprefix=/glade/scratch DISKNM=/glade/p/ral/jntp/GMTB/ufs-weather-model/RT @@ -279,6 +281,7 @@ elif [[ $MACHINE_ID = cheyenne.* ]]; then PTMP=$dprefix SCHEDULER=pbs cp fv3_conf/fv3_qsub.IN_cheyenne fv3_conf/fv3_qsub.IN + cp fv3_conf/compile_qsub.IN_cheyenne fv3_conf/compile_qsub.IN elif [[ $MACHINE_ID = stampede.* ]]; then @@ -289,15 +292,16 @@ elif [[ $MACHINE_ID = stampede.* ]]; then export PYTHONPATH= ECFLOW_START= QUEUE=skx-dev + COMPILE_QUEUE=skx-dev PARTITION= - dprefix=$WORK/NEMSfv3gfs/run - DISKNM=$WORK/NEMSfv3gfs/RT - STMP=$dprefix/stmp4 - PTMP=$dprefix/stmp3 - SCHEDULER=sbatch + dprefix=$WORK/ufs-weather-model/run + DISKNM=$WORK/ufs-weather-model/RT + STMP=$dprefix + PTMP=$dprefix + SCHEDULER=slurm MPIEXEC=ibrun MPIEXECOPTS= - cp fv3_conf/fv3_qsub.IN_stampede fv3_conf/fv3_qsub.IN + cp fv3_conf/fv3_slurm.IN_stampede fv3_conf/fv3_slurm.IN else die "Unknown machine ID, please edit detect_machine.sh file" @@ -516,7 +520,7 @@ EOF elif [[ $MACHINE_ID = jet.* ]]; then QUEUE=batch elif [[ $MACHINE_ID = cheyenne.* ]]; then - QUEUE=premium + QUEUE=regular else die "ecFlow is not supported on this machine $MACHINE_ID" fi diff --git a/tests/rt_fv3.sh b/tests/rt_fv3.sh index 74c7c4e696..e7f629144b 100755 --- a/tests/rt_fv3.sh +++ b/tests/rt_fv3.sh @@ -42,15 +42,7 @@ fi # Set up the run directory source ./fv3_run -if [[ $SCHEDULER = 'moab' ]]; then - atparse < $PATHRT/fv3_conf/fv3_msub.IN > job_card -elif [[ $SCHEDULER = 'pbs' ]]; then - NODES=$(( TASKS / TPN )) - if (( NODES * TPN < TASKS )); then - NODES=$(( NODES + 1 )) - fi - atparse < $PATHRT/fv3_conf/fv3_qsub.IN > job_card -elif [[ $SCHEDULER = 'sbatch' ]]; then +if [[ $SCHEDULER = 'pbs' ]]; then NODES=$(( TASKS / TPN )) if (( NODES * TPN < TASKS )); then NODES=$(( NODES + 1 )) diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh index 3f1283b626..252003e2fa 100755 --- a/tests/rt_utils.sh +++ b/tests/rt_utils.sh @@ -1,6 +1,3 @@ -# -# DH* TODO - COMBINE SBATCH AND SLURM? -# set -eu if [[ "$0" = "${BASH_SOURCE[0]}" ]]; then @@ -22,9 +19,7 @@ submit_and_wait() { local test_status='PASS' - if [[ $SCHEDULER = 'moab' ]]; then - msub $job_card - elif [[ $SCHEDULER = 'pbs' ]]; then + if [[ $SCHEDULER = 'pbs' ]]; then qsubout=$( qsub $job_card ) if [[ ${MACHINE_ID} = cheyenne.* ]]; then re='^([0-9]+\.[a-zA-Z0-9\.]+)$' @@ -36,14 +31,7 @@ submit_and_wait() { if [[ ${MACHINE_ID} = cheyenne.* ]]; then qsub_id="${qsub_id%.chadm*}" fi - elif [[ $SCHEDULER = 'sbatch' ]]; then - qsubout=$( sbatch $job_card ) - re='^([0-9]+\.[a-zA-Z0-9]+)$' - qsub_id=0 - [[ "${qsubout}" =~ $re ]] && qsub_id=${BASH_REMATCH[1]} - if [[ ${MACHINE_ID} = stampede.* ]]; then - qsub_id="${qsub_id}" - fi + echo "Job id ${qsub_id}" elif [[ $SCHEDULER = 'slurm' ]]; then slurmout=$( sbatch $job_card ) re='Submitted batch job ([0-9]+)' @@ -67,20 +55,12 @@ submit_and_wait() { until [[ $job_running -eq 1 ]] do echo "TEST ${TEST_NR} ${TEST_NAME} is waiting to enter the queue" - if [[ $SCHEDULER = 'moab' ]]; then - job_running=$( showq -u ${USER} -n | grep ${JBNME} | wc -l) - elif [[ $SCHEDULER = 'pbs' ]]; then + if [[ $SCHEDULER = 'pbs' ]]; then if [[ ${MACHINE_ID} = cheyenne.* ]]; then job_running=$( qstat ${qsub_id} | grep ${qsub_id} | wc -l) else job_running=$( qstat -u ${USER} -n | grep ${JBNME} | wc -l) fi - elif [[ $SCHEDULER = 'sbatch' ]]; then - if [[ ${MACHINE_ID} = stampede.* ]]; then - job_running=$( squeue ${qsub_id} | grep ${qsub_id} | wc -l) - else - job_running=$( squeue -u ${USER} -n | grep ${JBNME} | wc -l) - fi elif [[ $SCHEDULER = 'slurm' ]]; then job_running=$( squeue -u ${USER} -j ${slurm_id} | grep ${slurm_id} | wc -l) elif [[ $SCHEDULER = 'lsf' ]]; then @@ -95,30 +75,8 @@ submit_and_wait() { done # find jobid - if [[ $SCHEDULER = 'moab' ]]; then - : - elif [[ $SCHEDULER = 'pbs' ]]; then - if [[ ${MACHINE_ID} = cheyenne.* ]]; then - jobid=$( qstat ${qsub_id} | grep ${qsub_id} | awk '{print $1}' ) - jobid="${jobid%.chadm*}" - else - jobid=$( qstat -u ${USER} | grep ${JBNME} | awk '{print $1}' ) - fi - trap 'echo "Job ${jobid} killed"; qdel ${jobid}; trap 0; exit' 1 2 3 4 5 6 7 8 10 12 13 15 - if [[ ${qsub_id} != ${jobid} ]]; then - echo "Warning: qsub_id is not equal to jobid" - fi - elif [[ $SCHEDULER = 'sbatch' ]]; then - if [[ ${MACHINE_ID} = stampede.* ]]; then - jobid=$( squeue ${qsub_id} | grep ${qsub_id} | awk '{print $1}' ) - jobid="${jobid}" - else - jobid=$( squeue -u ${USER} | grep ${JBNME} | awk '{print $1}' ) - fi - trap 'echo "Job ${jobid} killed"; qdel ${jobid}; trap 0; exit' 1 2 3 4 5 6 7 8 10 12 13 15 - if [[ ${qsub_id} != ${jobid} ]]; then - echo "Warning: qsub_id is not equal to jobid" - fi + if [[ $SCHEDULER = 'pbs' ]]; then + jobid=${qsub_id} elif [[ $SCHEDULER = 'slurm' ]]; then jobid=${slurm_id} elif [[ $SCHEDULER = 'lsf' ]]; then @@ -136,20 +94,12 @@ submit_and_wait() { sleep 60 & wait $! - if [[ $SCHEDULER = 'moab' ]]; then - job_running=$( showq -u ${USER} -n | grep ${JBNME} | wc -l) - elif [[ $SCHEDULER = 'pbs' ]]; then + if [[ $SCHEDULER = 'pbs' ]]; then if [[ ${MACHINE_ID} = cheyenne.* ]]; then job_running=$( qstat ${qsub_id} | grep ${qsub_id} | wc -l) else job_running=$( qstat -u ${USER} -n | grep ${JBNME} | wc -l) fi - elif [[ $SCHEDULER = 'sbatch' ]]; then - if [[ ${MACHINE_ID} = stampede.* ]]; then - job_running=$( squeue ${qsub_id} | grep ${qsub_id} | wc -l) - else - job_running=$( squeue -u ${USER} -n | grep ${JBNME} | wc -l) - fi elif [[ $SCHEDULER = 'slurm' ]]; then job_running=$( squeue -u ${USER} -j ${slurm_id} | grep ${slurm_id} | wc -l) elif [[ $SCHEDULER = 'lsf' ]]; then @@ -159,19 +109,8 @@ submit_and_wait() { exit 1 fi - if [[ $SCHEDULER = 'moab' ]]; then + if [[ $SCHEDULER = 'pbs' ]]; then - status=$( showq -u ${USER} -n | grep ${JBNME} | awk '{print $3}'); status=${status:--} - if [[ $status = 'Idle' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is waiting in a queue, Status: $status" - elif [[ $status = 'Running' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is running, Status: $status" - elif [[ $status = 'Starting' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is ready to run, Status: $status" - elif [[ $status = 'Completed' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" ; job_running=0 - else echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" - fi - - elif [[ $SCHEDULER = 'pbs' ]]; then - - #status=$( qstat -u ${USER} -n | grep ${JBNME} | awk '{print $"10"}' ); status=${status:--} PJP comment out to speed up regression test if [[ ${MACHINE_ID} = cheyenne.* ]]; then status=$( qstat ${qsub_id} | grep ${qsub_id} | awk '{print $5}' ); status=${status:--} else @@ -221,38 +160,6 @@ submit_and_wait() { echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is ${state}" fi - elif [[ $SCHEDULER = 'sbatch' ]]; then - - #status=$( qstat -u ${USER} -n | grep ${JBNME} | awk '{print $"10"}' ); status=${status:--} PJP comment out to speed up regression test - if [[ ${MACHINE_ID} = stampede.* ]]; then - status=$( squeue ${qsub_id} | grep ${qsub_id} | awk '{print $5}' ); status=${status:--} - else - status=$( squeue -u ${USER} -n | grep ${JBNME} | awk '{print $10}' ); status=${status:--} - fi - if [[ $status = 'Q' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is waiting in a queue, Status: $status jobid ${jobid}" - elif [[ $status = 'H' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is held in a queue, Status: $status" - elif [[ $status = 'R' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is running, Status: $status" - elif [[ $status = 'E' ]] || [[ $status = 'C' ]]; then - if [[ ${MACHINE_ID} = stampede.* ]]; then - exit_status=$( squeue ${jobid} -x -f | grep Exit_status | awk '{print $3}') - else - jobid=$( squeue -u ${USER} | grep ${JBNME} | awk '{print $1}') - exit_status=$( qstat ${jobid} -f | grep exit_status | awk '{print $3}') - fi - if [[ $exit_status != 0 ]]; then - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" >> ${REGRESSIONTEST_LOG} - echo >> ${REGRESSIONTEST_LOG} - echo "Test ${TEST_NR} ${TEST_NAME} FAIL" - echo - test_status='FAIL' - break - fi - echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" - job_running=0 - elif [[ $status = 'C' ]]; then echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" ; job_running=0 - else echo "$n min. TEST ${TEST_NR} ${TEST_NAME} is finished, Status: $status" - fi - elif [[ $SCHEDULER = 'lsf' ]]; then status=$( bjobs ${bsub_id} 2>/dev/null | grep ${bsub_id} | awk '{print $3}' ); status=${status:--} @@ -418,9 +325,7 @@ kill_job() { local -r jobid=$1 - if [[ $SCHEDULER = 'moab' ]]; then - : - elif [[ $SCHEDULER = 'pbs' ]]; then + if [[ $SCHEDULER = 'pbs' ]]; then qdel ${jobid} elif [[ $SCHEDULER = 'slurm' ]]; then scancel ${jobid} diff --git a/tests/run_compile.sh b/tests/run_compile.sh index a6acac8967..53819be257 100755 --- a/tests/run_compile.sh +++ b/tests/run_compile.sh @@ -34,6 +34,8 @@ if [[ $SCHEDULER = 'slurm' ]]; then atparse < $PATHRT/fv3_conf/compile_slurm.IN > job_card elif [[ $SCHEDULER = 'lsf' ]]; then atparse < $PATHRT/fv3_conf/compile_bsub.IN > job_card +elif [[ $SCHEDULER = 'pbs' ]]; then + atparse < $PATHRT/fv3_conf/compile_qsub.IN > job_card fi ################################################################################