diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index abb36922c..b8e872837 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -85,12 +85,16 @@ case "$MACHINE" in ;; "ODIN") + APRUN="srun" ;; "CHEYENNE") nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS )) APRUN="mpirun -np $nprocs" ;; + "STAMPEDE") + APRUN="ibrun" + ;; esac # diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index af64a106a..7b53dab31 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -85,6 +85,7 @@ case "$MACHINE" in ;; "ODIN") + APRUN="srun" ;; "CHEYENNE") @@ -92,6 +93,10 @@ case "$MACHINE" in APRUN="mpirun -np $nprocs" ;; + "STAMPEDE") + APRUN="ibrun" + ;; + esac # #----------------------------------------------------------------------- diff --git a/modulefiles/codes/odin/global_equiv_resol b/modulefiles/codes/odin/global_equiv_resol new file mode 100644 index 000000000..123d202ea --- /dev/null +++ b/modulefiles/codes/odin/global_equiv_resol @@ -0,0 +1,13 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load PrgEnv-intel +module swap intel/19.0.5.281 +module load cray-mpich/7.7.10 +module load cray-libsci +module load cray-netcdf-hdf5parallel +module load cray-parallel-netcdf +module load cray-hdf5-parallel + +export NETCDF=/opt/cray/pe/netcdf-hdf5parallel/4.6.3.2/INTEL/19.0 +export HDF5=/opt/cray/pe/hdf5-parallel/1.10.5.2/INTEL/19.0 diff --git a/modulefiles/codes/odin/mosaic_file b/modulefiles/codes/odin/mosaic_file new file mode 100644 index 000000000..123d202ea --- /dev/null +++ b/modulefiles/codes/odin/mosaic_file @@ -0,0 +1,13 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load PrgEnv-intel +module swap intel/19.0.5.281 +module load cray-mpich/7.7.10 +module load cray-libsci +module load cray-netcdf-hdf5parallel +module load cray-parallel-netcdf +module load cray-hdf5-parallel + +export NETCDF=/opt/cray/pe/netcdf-hdf5parallel/4.6.3.2/INTEL/19.0 +export HDF5=/opt/cray/pe/hdf5-parallel/1.10.5.2/INTEL/19.0 diff --git a/modulefiles/codes/odin/regional_grid b/modulefiles/codes/odin/regional_grid new file mode 100644 index 000000000..2d4b9a9a3 --- /dev/null +++ b/modulefiles/codes/odin/regional_grid @@ -0,0 +1,11 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load PrgEnv-intel +module swap intel/19.0.5.281 +module load cray-mpich/7.7.10 +module load cray-libsci +module load cray-netcdf-hdf5parallel +module load cray-parallel-netcdf +module load cray-hdf5-parallel + diff --git a/modulefiles/codes/stampede/global_equiv_resol b/modulefiles/codes/stampede/global_equiv_resol new file mode 100644 index 000000000..0f78231fa --- /dev/null +++ b/modulefiles/codes/stampede/global_equiv_resol @@ -0,0 +1,10 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load intel/18.0.2 +module load impi/18.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.2 + +setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64 +setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64 diff --git a/modulefiles/codes/stampede/mosaic_file b/modulefiles/codes/stampede/mosaic_file new file mode 100644 index 000000000..bab633db7 --- /dev/null +++ b/modulefiles/codes/stampede/mosaic_file @@ -0,0 +1,12 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# + + +module load intel/18.0.2 +module load impi/18.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.2 + +setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64 +setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64 diff --git a/modulefiles/codes/stampede/regional_grid b/modulefiles/codes/stampede/regional_grid new file mode 100644 index 000000000..0f78231fa --- /dev/null +++ b/modulefiles/codes/stampede/regional_grid @@ -0,0 +1,10 @@ +#%Module##################################################### +## Module file for regional_grid +############################################################# +module load intel/18.0.2 +module load impi/18.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.2 + +setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64 +setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64 diff --git a/modulefiles/tasks/odin/get_extrn_ics b/modulefiles/tasks/odin/get_extrn_ics new file mode 100644 index 000000000..77e861603 --- /dev/null +++ b/modulefiles/tasks/odin/get_extrn_ics @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_ics task. +############################################################# + +#module purge + +#module load hpss + diff --git a/modulefiles/tasks/odin/get_extrn_lbcs b/modulefiles/tasks/odin/get_extrn_lbcs new file mode 100644 index 000000000..e0b1eb8ab --- /dev/null +++ b/modulefiles/tasks/odin/get_extrn_lbcs @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_lbcs task. +############################################################# + +#module purge + +#module load hpss + diff --git a/modulefiles/tasks/odin/make_grid b/modulefiles/tasks/odin/make_grid new file mode 100644 index 000000000..d37cb0e9d --- /dev/null +++ b/modulefiles/tasks/odin/make_grid @@ -0,0 +1,14 @@ +#%Module##################################################### +## Module file for make_grid task. +############################################################# + +#module purge + +module load PrgEnv-intel +module swap intel/19.0.5.281 +module load cray-libsci +module load cray-netcdf-hdf5parallel +module load cray-parallel-netcdf +module load cray-hdf5-parallel +module load gcc/6.1.0 +module load slurm diff --git a/modulefiles/tasks/stampede/get_extrn_ics b/modulefiles/tasks/stampede/get_extrn_ics new file mode 100644 index 000000000..77e861603 --- /dev/null +++ b/modulefiles/tasks/stampede/get_extrn_ics @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_ics task. +############################################################# + +#module purge + +#module load hpss + diff --git a/modulefiles/tasks/stampede/get_extrn_lbcs b/modulefiles/tasks/stampede/get_extrn_lbcs new file mode 100644 index 000000000..e0b1eb8ab --- /dev/null +++ b/modulefiles/tasks/stampede/get_extrn_lbcs @@ -0,0 +1,8 @@ +#%Module##################################################### +## Module file for get_extrn_lbcs task. +############################################################# + +#module purge + +#module load hpss + diff --git a/modulefiles/tasks/stampede/make_grid b/modulefiles/tasks/stampede/make_grid new file mode 100644 index 000000000..aa867cfbf --- /dev/null +++ b/modulefiles/tasks/stampede/make_grid @@ -0,0 +1,11 @@ +#%Module##################################################### +## Module file for make_grid task. +############################################################# + +module purge + +module load intel/18.0.2 +module load impi/18.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.2 +module load python3/3.7.0 diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 37b7bcbf1..48cfb49a2 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -148,6 +148,13 @@ case $MACHINE in APRUN="time" ;; +"STAMPEDE") +# + export APRUN="time" + + ulimit -s unlimited + ulimit -a + ;; esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 5f5179666..a8a39dcc0 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -145,6 +145,11 @@ case $MACHINE in APRUN="time" ;; +"STAMPEDE") + export APRUN="time" + export topo_dir="/work/00315/tg455890/stampede2/regional_fv3/fix_orog" + ;; + esac # #----------------------------------------------------------------------- @@ -317,7 +322,7 @@ ${tmp_dir}" \ ;; -"CHEYENNE" | "HERA" | "JET" | "ODIN") +"CHEYENNE" | "HERA" | "JET" | "ODIN" | "STAMPEDE") $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 48b383f48..938ede5e9 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -151,6 +151,16 @@ case $MACHINE in APRUN="mpirun -np $nprocs" ;; +"ODIN") + nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) + APRUN="srun -n $nprocs" + ;; + +"STAMPEDE") + nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) + APRUN="ibrun -np ${nprocs}" + ;; + *) print_err_msg_exit "\ Run command has not been specified for this machine: diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 6f74dd8b0..10b7fed07 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -120,6 +120,15 @@ case $MACHINE in LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ;; # + +"STAMPEDE") +# + module list + + APRUN="ibrun -np ${PE_MEMBER01}" + #LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + ;; + esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 1832daaad..60758381e 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -128,6 +128,11 @@ case $MACHINE in APRUN="mpirun -np $nprocs" ;; +"STAMPEDE") + nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) + APRUN="ibrun -n $nprocs" + ;; + esac # #----------------------------------------------------------------------- diff --git a/sorc/global_equiv_resol.fd/Makefile_stampede b/sorc/global_equiv_resol.fd/Makefile_stampede new file mode 100644 index 000000000..ee9355051 --- /dev/null +++ b/sorc/global_equiv_resol.fd/Makefile_stampede @@ -0,0 +1,29 @@ +SHELL := bash + +MAKEFLAGS += --warn-undefined-variables + +INC = -I${NETCDF}/include + +LIBS = ${NETCDF}/lib/libnetcdff.a ${NETCDF}/lib/libnetcdf.a \ + ${HDF5}/lib/libhdf5_hl.a ${HDF5}/lib/libhdf5.a ${HDF5}/lib/libsz.a -lz + +FC = mpiifort +FFLAGS = -g -O2 $(INC) + +EXEC = global_equiv_resol + +.PHONY: all +all : $(EXEC) + +$(EXEC): global_equiv_resol.o + $(FC) $(FFLAGS) -o $@ $^ $(LIBS) + +.SUFFIXES: +.SUFFIXES: .f90 .o + +.f90.o: + $(FC) $(FFLAGS) -c $< + +.PHONY: clean +clean: + rm -f *.o *.mod $(EXEC) diff --git a/sorc/mosaic_file.fd/Makefile_stampede b/sorc/mosaic_file.fd/Makefile_stampede new file mode 100644 index 000000000..8a3b1a83a --- /dev/null +++ b/sorc/mosaic_file.fd/Makefile_stampede @@ -0,0 +1,29 @@ +SHELL := bash + +MAKEFLAGS += --warn-undefined-variables + +INC = -I${NETCDF}/include + +LIBS = ${NETCDF}/lib/libnetcdff.a ${NETCDF}/lib/libnetcdf.a \ + ${HDF5}/lib/libhdf5_hl.a ${HDF5}/lib/libhdf5.a ${HDF5}/lib/libsz.a -lz + +FC = mpiifort +FFLAGS = -g -O2 $(INC) + +EXEC = mosaic_file + +.PHONY: all +all : $(EXEC) + +$(EXEC): mosaic_file.o + $(FC) $(FFLAGS) -o $@ $^ $(LIBS) + +.SUFFIXES: +.SUFFIXES: .f90 .o + +.f90.o: + $(FC) $(FFLAGS) -c $< + +.PHONY: clean +clean: + rm -f *.o *.mod $(EXEC) diff --git a/sorc/regional_grid.fd/Makefile_stampede b/sorc/regional_grid.fd/Makefile_stampede new file mode 100644 index 000000000..5f584611b --- /dev/null +++ b/sorc/regional_grid.fd/Makefile_stampede @@ -0,0 +1,29 @@ + + +MAKEFLAGS += --warn-undefined-variables + +INC = -I${NETCDF}/include + +LIBS = ${NETCDF}/lib/libnetcdff.a ${NETCDF}/lib/libnetcdf.a \ + ${HDF5}/lib/libhdf5_hl.a ${HDF5}/lib/libhdf5.a ${HDF5}/lib/libsz.a -lz + +FC = mpiifort +FFLAGS = -g -O2 $(INC) + +REGIONAL_GRID = regional_grid + +.PHONY: all +all : $(REGIONAL_GRID) + +$(REGIONAL_GRID): pkind.o pietc.o pmat.o pmat4.o pmat5.o psym2.o gen_schmidt.o hgrid_ak.o regional_grid.o + $(FC) $(FFLAGS) -o $@ $^ $(LIBS) + +.SUFFIXES: +.SUFFIXES: .f90 .o + +.f90.o: + $(FC) $(FFLAGS) -c $< + +.PHONY: clean +clean: + rm -f *.o *.mod $(REGIONAL_GRID) diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index 1e52e6ae2..fde6b8e2d 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -545,11 +545,14 @@ bination of external model (extrn_mdl_name) and analysis or forecast sysdir="" ;; "ODIN") - sysdir="" + sysdir="$sysbasedir" ;; "CHEYENNE") sysdir="" ;; + "STAMPEDE") + sysdir="$sysbasedir" + ;; *) print_err_msg_exit "\ The system directory in which to look for external model output files diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 5943fa1ee..c434f8d60 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -68,11 +68,14 @@ else EXTRN_MDL_FILES_SYSBASEDIR_ICS="" ;; "ODIN") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="" + EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/ywang/EPIC/GDAS/2019053000_mem001" ;; "CHEYENNE") EXTRN_MDL_FILES_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; + "STAMPEDE") + EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" + ;; *) print_err_msg_exit "\ The system directory in which to look for the files generated by the ex- @@ -223,11 +226,14 @@ else EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" ;; "ODIN") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" + EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/ywang/EPIC/GDAS/2019053000_mem001" ;; "CHEYENNE") EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; + "STAMPEDE") + EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" + ;; *) print_err_msg_exit "\ The system directory in which to look for the files generated by the ex- diff --git a/ush/setup.sh b/ush/setup.sh index 5aff206f4..8fcabfbc9 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -312,7 +312,16 @@ case $MACHINE in QUEUE_HPSS=${QUEUE_HPSS:-"regular"} QUEUE_HPSS_TAG="queue" # pbspro does not support "partition" tag QUEUE_FCST=${QUEUE_FCST:-"regular"} + ;; +# +"STAMPEDE") # + NCORES_PER_NODE=68 + SCHED="slurm" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} + QUEUE_HPSS=${QUEUE_HPSS:-"development"} + QUEUE_FCST=${QUEUE_FCST:-"normal"} + ;; esac # #----------------------------------------------------------------------- @@ -670,13 +679,18 @@ case $MACHINE in "ODIN") FIXgsm="/scratch/ywang/fix/theia_fix/fix_am" - SFC_CLIMO_INPUT_DIR="/scratch1/NCEPDEV/da/George.Gayno/ufs_utils.git/climo_fields_netcdf" + SFC_CLIMO_INPUT_DIR="/scratch/ywang/fix/climo_fields_netcdf" ;; "CHEYENNE") FIXgsm="/glade/p/ral/jntp/UFS_CAM/fix/fix_am" SFC_CLIMO_INPUT_DIR="/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf" ;; +"STAMPEDE") + FIXgsm="/work/00315/tg455890/stampede2/regional_fv3/fix_am" + SFC_CLIMO_INPUT_DIR="/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf" + ;; + *) print_err_msg_exit "\ Directories have not been specified for this machine: diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 59caeec39..793c568c8 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -1,6 +1,6 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE") +valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE" "STAMPEDE") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_PREDEF_GRID_NAME=( \ "EMC_CONUS_3km" \