Ported RTs to S4.

DavidHuber-NOAA · Feb 28, 2023 · 7d05325 · 7d05325
1 parent d266ddd
commit 7d05325
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 17 deletions.
diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua
@@ -1,14 +1,13 @@
 help([[
 ]])
 
-local hpc_ver=os.getenv("hpc_ver") or "1.1.0"
-local hpc_intel_ver=os.getenv("hpc_intel_ver") or "18.0.4"
-local hpc_impi_ver=os.getenv("hpc_impi_ver") or "18.0.4"
+prepend_path("MODULEPATH", "/data/prod/hpc-stack/modulefiles/stack")
+local hpc_ver=os.getenv("hpc_ver") or "1.2.0"
+local hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1"
+local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1"
 local miniconda_ver=os.getenv("miniconda_ver") or "3.8-s4"
 local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
 
-prepend_path("MODULEPATH", "/data/prod/hpc-stack/modulefiles/stack")
-
 load("license_intel/S4")
 load(pathJoin("hpc", hpc_ver))
 load(pathJoin("hpc-intel", hpc_intel_ver))
@@ -20,8 +19,13 @@ load("gsi_common")
 
 load(pathJoin("prod_util", prod_util_ver))
 
-pushenv("CFLAGS", "-march=ivybridge")
-pushenv("FFLAGS", "-march=ivybridge")
+load("intel/2023.0")
+pushenv("CFLAGS", "-xHOST")
+pushenv("FFLAGS", "-xHOST")
+pushenv("FC", "mpiifort -fc=ifx")
+pushenv("CC", "mpiicc")
+pushenv("MPI_C", "mpiicc")
+pushenv("MPI_Fortran", "mpiifort -fc=ifx")
 
 pushenv("GSI_BINARY_SOURCE_DIR", "/data/prod/glopara/fix/gsi/20221128")
 

diff --git a/regression/regression_param.sh b/regression/regression_param.sh
@@ -29,6 +29,11 @@ case $machine in
     ;;
 	Cheyenne)
 	   sub_cmd="sub_ncar -a p48503002 -q economy -d $PWD"
+    ;;
+	S4)
+	   sub_cmd="sub_s4"
+           memnode=188
+           numcore=32
     ;;
     *) # EXIT out for unresolved machine
         echo "unknown $machine"
@@ -64,6 +69,9 @@ case $regtest in
         elif [[ "$machine" = "Discover" ]]; then
            topts[1]="0:30:00" ; popts[1]="36/2"  ; ropts[1]="/1"
            topts[2]="0:30:00" ; popts[2]="72/3"  ; ropts[2]="/2"
+        elif [[ "$machine" = "S4" ]]; then
+           topts[1]="0:90:00" ; popts[1]="12/5/" ; ropts[1]="/1"
+           topts[2]="0:90:00" ; popts[2]="12/9/" ; ropts[2]="/2"
         fi
 
         if [ "$debug" = ".true." ] ; then
@@ -94,6 +102,9 @@ case $regtest in
         elif [[ "$machine" = "Discover" ]]; then
             topts[1]="2:00:00" ; popts[1]="48/2"  ; ropts[1]="/1"
             topts[2]="2:00:00" ; popts[2]="60/3"  ; ropts[2]="/2"
+        elif [[ "$machine" = "S4" ]]; then
+           topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         fi
 
         if [ "$debug" = ".true." ] ; then
@@ -107,6 +118,8 @@ case $regtest in
            elif [[ "$machine" = "wcoss2" ]]; then
               popts[1]="28/4/"
               topts[1]="3:00:00"
+           elif [[ "$machine" = "S4" ]]; then
+              popts[1]="12/5/"
            fi
         fi
 
@@ -134,6 +147,9 @@ case $regtest in
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
            topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
+        elif [[ "$machine" = "S4" ]]; then
+           topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
+           topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
         fi
 
         if [ "$debug" = ".true." ] ; then
@@ -158,6 +174,9 @@ case $regtest in
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
+        elif [[ "$machine" = "S4" ]]; then
+           topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
         fi
 
         if [ "$debug" = ".true." ] ; then
@@ -182,6 +201,9 @@ case $regtest in
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1"
+        elif [[ "$machine" = "S4" ]]; then
+           topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2"
         fi
 
         if [ "$debug" = ".true." ] ; then
@@ -209,6 +231,9 @@ case $regtest in
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1"
            topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2"
+        elif [[ "$machine" = "S4" ]]; then
+           topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1"
+           topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/2"
         fi
 
         if [ "$debug" = ".true." ] ; then
@@ -236,6 +261,9 @@ case $regtest in
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="10/10/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="14/14/" ; ropts[2]="/2"
+        elif [[ "$machine" = "S4" ]]; then
+           topts[1]="0:20:00" ; popts[1]="6/6/" ; ropts[1]="/1"
+           topts[2]="0:20:00" ; popts[2]="8/8/" ; ropts[2]="/2"
         fi
 
         if [ "$debug" = ".true." ] ; then
@@ -263,6 +291,9 @@ case $regtest in
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1"
            topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2"
+        elif [[ "$machine" = "S4" ]]; then
+           topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+           topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         fi
 
         if [ "$debug" = ".true." ] ; then
@@ -329,6 +360,12 @@ elif [[ "$machine" = "wcoss2" ]]; then
    export FORT_BUFFERED=true
    export FI_OFI_RXM_SAR_LIMIT=3145728
    export APRUN="mpiexec -n \$ntasks -ppn \$ppn --cpu-bind core --depth \$threads"
+elif [[ "$machine" = "S4" ]]; then
+   export OMP_STACKSIZE=1024M
+   export MPI_BUFS_PER_PROC=256
+   export MPI_BUFS_PER_HOST=256
+   export MPI_GROUP_MAX=256
+   export APRUN="srun"
 elif [[ "$machine" = "Discover" ]]; then
    export APRUN="mpiexec_mpt -np \$SLURM_NTASKS"
 fi
diff --git a/regression/regression_var.sh b/regression/regression_var.sh
@@ -45,6 +45,8 @@ elif [[ -d /work ]]; then # Orion
   export machine="Orion"
 elif [[ -d /lfs/h2 ]]; then # wcoss2
   export machine="wcoss2"
+elif [[ -d /data ]]; then # wcoss2
+  export machine="s4"
 fi
 echo "Running Regression Tests on '$machine'";
 
@@ -109,13 +111,6 @@ case $machine in
 
     export local_or_default="${local_or_default:-/scratch1/NCEPDEV/da/$LOGNAME}"
     export noscrub="/scratch1/NESDIS/nesdis-rdo2/$LOGNAME"
-#    if [ -d $local_or_default ]; then
-#      export noscrub="$local_or_default/noscrub"
-#    elif [ -d /scratch1/NCEPDEV/global/$LOGNAME ]; then
-#      export noscrub="/scratch1/NCEPDEV/global/$LOGNAME/noscrub"
-#     elif [ -d /scratch2/BMC/gsienkf/$LOGNAME ]; then
-#      export noscrub="/scratch2/BMC/gsienkf/$LOGNAME"
-#    fi
 
     export group="${group:-global}"
     export queue="${queue:-batch}"
@@ -125,15 +120,12 @@ case $machine in
 
     export ptmp="/scratch1/NESDIS/nesdis-rdo2/David.Huber/para/ptmp/$ptmpName"
 
-##  export fixcrtm="${CRTM_FIX:-/scratch1/NCEPDEV/da/Michael.Lueken/CRTM_REL-2.2.3/crtm_v2.2.3/fix_update}"
     export casesdir="/scratch1/NCEPDEV/da/Russ.Treadon/CASES/regtest"
 
     export check_resource="no"
 
     export accnt="nesdis-rdo1"
 
-    #  On Hera, there are no scrubbers to remove old contents from stmp* directories.
-    #  After completion of regression tests, will remove the regression test subdirecories
     export clean=".false."
   ;;
   Jet)
@@ -172,6 +164,27 @@ case $machine in
     export queue="compute"
     export clean=".false."
   ;;
+  S4)
+
+    export local_or_default="${local_or_default:-/data/users/$LOGNAME}"
+    export noscrub="/data/users/$LOGNAME"
+
+    export group="${group:-}"
+    export queue="${queue:-s4}"
+    if [[ "$cmaketest" = "false" ]]; then
+      export basedir="/data/users/dhuber/save/git/gsi"
+    fi
+
+    export ptmp="/scratch/users/$LOGNAME/ptmp/$ptmpName"
+
+    export casesdir="/data/users/dhuber/save/gsi_data/CASES/regtest"
+
+    export check_resource="no"
+
+    export accnt="star"
+
+    export clean=".false."
+  ;;
   *)
     echo "Regression tests are not setup on '$machine', ABORT!"
     exit 1