Revert "Revert "Added support on Odin and Stampede (NOAA-EMC#242)" (N…

…OAA-EMC#243)" (NOAA-EMC#244) This reverts commit c57ce32.
EricJames-NOAA · Jun 26, 2020 · 4b1cbab · 4b1cbab
1 parent c57ce32
commit 4b1cbab
Show file tree

Hide file tree

Showing 26 changed files with 287 additions and 6 deletions.
diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS
@@ -85,12 +85,16 @@ case "$MACHINE" in
     ;;
 
   "ODIN")
+    APRUN="srun"
     ;;
 
   "CHEYENNE")
     nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS ))
     APRUN="mpirun -np $nprocs"
     ;;
+  "STAMPEDE")
+    APRUN="ibrun"
+    ;;
 
 esac
 #

diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS
@@ -85,13 +85,18 @@ case "$MACHINE" in
     ;;
 
   "ODIN")
+    APRUN="srun"
     ;;
 
   "CHEYENNE")
     nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS ))
     APRUN="mpirun -np $nprocs"
     ;;
 
+  "STAMPEDE")
+    APRUN="ibrun"
+    ;;
+
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/modulefiles/codes/odin/global_equiv_resol b/modulefiles/codes/odin/global_equiv_resol
@@ -0,0 +1,13 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load PrgEnv-intel
+module swap intel/19.0.5.281
+module load cray-mpich/7.7.10
+module load cray-libsci
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+
+export NETCDF=/opt/cray/pe/netcdf-hdf5parallel/4.6.3.2/INTEL/19.0
+export HDF5=/opt/cray/pe/hdf5-parallel/1.10.5.2/INTEL/19.0
diff --git a/modulefiles/codes/odin/mosaic_file b/modulefiles/codes/odin/mosaic_file
@@ -0,0 +1,13 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load PrgEnv-intel
+module swap intel/19.0.5.281
+module load cray-mpich/7.7.10
+module load cray-libsci
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+
+export NETCDF=/opt/cray/pe/netcdf-hdf5parallel/4.6.3.2/INTEL/19.0
+export HDF5=/opt/cray/pe/hdf5-parallel/1.10.5.2/INTEL/19.0
diff --git a/modulefiles/codes/odin/regional_grid b/modulefiles/codes/odin/regional_grid
@@ -0,0 +1,11 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load PrgEnv-intel
+module swap intel/19.0.5.281
+module load cray-mpich/7.7.10
+module load cray-libsci
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+
diff --git a/modulefiles/codes/stampede/global_equiv_resol b/modulefiles/codes/stampede/global_equiv_resol
@@ -0,0 +1,10 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load intel/18.0.2
+module load impi/18.0.2
+module load hdf5/1.10.4
+module load netcdf/4.6.2
+
+setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64
+setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64
diff --git a/modulefiles/codes/stampede/mosaic_file b/modulefiles/codes/stampede/mosaic_file
@@ -0,0 +1,12 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+
+
+module load intel/18.0.2
+module load impi/18.0.2
+module load hdf5/1.10.4
+module load netcdf/4.6.2
+
+setenv  NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64
+setenv  HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64
diff --git a/modulefiles/codes/stampede/regional_grid b/modulefiles/codes/stampede/regional_grid
@@ -0,0 +1,10 @@
+#%Module#####################################################
+## Module file for regional_grid
+#############################################################
+module load intel/18.0.2
+module load impi/18.0.2
+module load hdf5/1.10.4
+module load netcdf/4.6.2
+
+setenv NETCDF /opt/apps/intel18/netcdf/4.6.2/x86_64
+setenv HDF5 /opt/apps/intel18/hdf5/1.10.4/x86_64
diff --git a/modulefiles/tasks/odin/get_extrn_ics b/modulefiles/tasks/odin/get_extrn_ics
@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_ics task.
+#############################################################
+
+#module purge
+
+#module load hpss
+
diff --git a/modulefiles/tasks/odin/get_extrn_lbcs b/modulefiles/tasks/odin/get_extrn_lbcs
@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_lbcs task.
+#############################################################
+
+#module purge
+
+#module load hpss
+
diff --git a/modulefiles/tasks/odin/make_grid b/modulefiles/tasks/odin/make_grid
@@ -0,0 +1,14 @@
+#%Module#####################################################
+## Module file for make_grid task.
+#############################################################
+
+#module purge
+
+module load PrgEnv-intel
+module swap intel/19.0.5.281
+module load cray-libsci
+module load cray-netcdf-hdf5parallel
+module load cray-parallel-netcdf
+module load cray-hdf5-parallel
+module load gcc/6.1.0
+module load slurm
diff --git a/modulefiles/tasks/stampede/get_extrn_ics b/modulefiles/tasks/stampede/get_extrn_ics
@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_ics task.
+#############################################################
+
+#module purge
+
+#module load hpss
+
diff --git a/modulefiles/tasks/stampede/get_extrn_lbcs b/modulefiles/tasks/stampede/get_extrn_lbcs
@@ -0,0 +1,8 @@
+#%Module#####################################################
+## Module file for get_extrn_lbcs task.
+#############################################################
+
+#module purge
+
+#module load hpss
+
diff --git a/modulefiles/tasks/stampede/make_grid b/modulefiles/tasks/stampede/make_grid
@@ -0,0 +1,11 @@
+#%Module#####################################################
+## Module file for make_grid task.
+#############################################################
+
+module purge
+
+module load intel/18.0.2
+module load impi/18.0.2
+module load hdf5/1.10.4
+module load netcdf/4.6.2
+module load python3/3.7.0
diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh
@@ -148,6 +148,13 @@ case $MACHINE in
   APRUN="time"
   ;;
 
+"STAMPEDE")
+#
+  export APRUN="time"
+
+  ulimit -s unlimited
+  ulimit -a
+  ;;
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh
@@ -145,6 +145,11 @@ case $MACHINE in
   APRUN="time"
   ;;
 
+"STAMPEDE")
+  export APRUN="time"
+  export topo_dir="/work/00315/tg455890/stampede2/regional_fv3/fix_orog"
+  ;;
+
 esac
 #
 #-----------------------------------------------------------------------
@@ -317,7 +322,7 @@ ${tmp_dir}" \
   ;;
 
 
-"CHEYENNE" | "HERA" | "JET" | "ODIN")
+"CHEYENNE" | "HERA" | "JET" | "ODIN" | "STAMPEDE")
   $APRUN "${exec_fp}" < "${input_redirect_fn}" || \
     print_err_msg_exit "\
 Call to executable (exec_fp) that generates the raw orography file returned 

diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh
@@ -151,6 +151,16 @@ case $MACHINE in
   APRUN="mpirun -np $nprocs"
   ;;
 
+"ODIN")
+  nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO ))
+  APRUN="srun -n $nprocs"
+  ;;
+
+"STAMPEDE")
+  nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO ))
+  APRUN="ibrun -np ${nprocs}"
+  ;;
+
 *)
   print_err_msg_exit "\
 Run command has not been specified for this machine:

diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh
@@ -120,6 +120,15 @@ case $MACHINE in
   LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
   ;;
 #
+
+"STAMPEDE")
+#
+  module list
+
+  APRUN="ibrun -np ${PE_MEMBER01}"
+  #LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+  ;;
+
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh
@@ -128,6 +128,11 @@ case $MACHINE in
   APRUN="mpirun -np $nprocs"
   ;;
 
+"STAMPEDE")
+  nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST ))
+  APRUN="ibrun -n $nprocs"
+  ;;
+
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/sorc/global_equiv_resol.fd/Makefile_stampede b/sorc/global_equiv_resol.fd/Makefile_stampede
@@ -0,0 +1,29 @@
+SHELL := bash
+
+MAKEFLAGS += --warn-undefined-variables
+
+INC  = -I${NETCDF}/include
+
+LIBS = ${NETCDF}/lib/libnetcdff.a ${NETCDF}/lib/libnetcdf.a \
+       ${HDF5}/lib/libhdf5_hl.a ${HDF5}/lib/libhdf5.a ${HDF5}/lib/libsz.a -lz
+
+FC = mpiifort
+FFLAGS = -g -O2 $(INC)
+
+EXEC = global_equiv_resol
+
+.PHONY: all
+all : $(EXEC)
+
+$(EXEC): global_equiv_resol.o
+	$(FC) $(FFLAGS) -o $@ $^ $(LIBS)
+
+.SUFFIXES:
+.SUFFIXES: .f90 .o
+
+.f90.o:
+	$(FC) $(FFLAGS) -c $<
+
+.PHONY: clean
+clean:
+	rm -f *.o *.mod $(EXEC)
diff --git a/sorc/mosaic_file.fd/Makefile_stampede b/sorc/mosaic_file.fd/Makefile_stampede
@@ -0,0 +1,29 @@
+SHELL := bash
+
+MAKEFLAGS += --warn-undefined-variables
+
+INC  = -I${NETCDF}/include
+
+LIBS = ${NETCDF}/lib/libnetcdff.a ${NETCDF}/lib/libnetcdf.a \
+       ${HDF5}/lib/libhdf5_hl.a ${HDF5}/lib/libhdf5.a ${HDF5}/lib/libsz.a -lz
+
+FC = mpiifort
+FFLAGS = -g -O2 $(INC)
+
+EXEC = mosaic_file
+
+.PHONY: all
+all : $(EXEC)
+
+$(EXEC): mosaic_file.o
+	$(FC) $(FFLAGS) -o $@ $^ $(LIBS)
+
+.SUFFIXES:
+.SUFFIXES: .f90 .o
+
+.f90.o:
+	$(FC) $(FFLAGS) -c $<
+
+.PHONY: clean
+clean:
+	rm -f *.o *.mod $(EXEC)
diff --git a/sorc/regional_grid.fd/Makefile_stampede b/sorc/regional_grid.fd/Makefile_stampede
@@ -0,0 +1,29 @@
+
+
+MAKEFLAGS += --warn-undefined-variables
+
+INC  = -I${NETCDF}/include
+
+LIBS = ${NETCDF}/lib/libnetcdff.a ${NETCDF}/lib/libnetcdf.a \
+       ${HDF5}/lib/libhdf5_hl.a ${HDF5}/lib/libhdf5.a ${HDF5}/lib/libsz.a -lz 
+
+FC = mpiifort
+FFLAGS = -g -O2 $(INC)
+
+REGIONAL_GRID = regional_grid
+
+.PHONY: all
+all : $(REGIONAL_GRID)
+
+$(REGIONAL_GRID): pkind.o pietc.o pmat.o pmat4.o pmat5.o psym2.o gen_schmidt.o hgrid_ak.o regional_grid.o
+	$(FC) $(FFLAGS) -o $@ $^ $(LIBS)
+
+.SUFFIXES:
+.SUFFIXES: .f90 .o
+
+.f90.o:
+	$(FC) $(FFLAGS) -c $<
+
+.PHONY: clean
+clean:
+	rm -f *.o *.mod $(REGIONAL_GRID)
diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh
@@ -545,11 +545,14 @@ bination of external model (extrn_mdl_name) and analysis or forecast
       sysdir=""
       ;;
     "ODIN")
-      sysdir=""
+      sysdir="$sysbasedir"
       ;;
     "CHEYENNE")
       sysdir=""
       ;;
+    "STAMPEDE")
+      sysdir="$sysbasedir"
+      ;;
     *)
       print_err_msg_exit "\
 The system directory in which to look for external model output files 

diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh
@@ -68,11 +68,14 @@ else
       EXTRN_MDL_FILES_SYSBASEDIR_ICS=""
       ;;
     "ODIN")
-      EXTRN_MDL_FILES_SYSBASEDIR_ICS=""
+      EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/ywang/EPIC/GDAS/2019053000_mem001"
       ;;
     "CHEYENNE")
       EXTRN_MDL_FILES_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS"
       ;;
+    "STAMPEDE")
+      EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001"
+      ;;
     *)
       print_err_msg_exit "\
 The system directory in which to look for the files generated by the ex-
@@ -223,11 +226,14 @@ else
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS=""
       ;;
     "ODIN")
-      EXTRN_MDL_FILES_SYSBASEDIR_LBCS=""
+      EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/ywang/EPIC/GDAS/2019053000_mem001"
       ;;
     "CHEYENNE")
       EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS"
       ;;
+    "STAMPEDE")
+      EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001"
+      ;;
     *)
       print_err_msg_exit "\
 The system directory in which to look for the files generated by the ex-