Support coupling on AWS

env/AWSPW.env

@@ -33,7 +33,153 @@ else
     exit 2
 fi
 
-if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
+
+    export POE="NO"
+    export BACK="NO"
+    export sys_tp="HERA"
+    export launcher_PREP="srun"
+
+elif [[ "${step}" = "prepsnowobs" ]]; then
+
+    export APRUN_CALCFIMS="${APRUN}"
+
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${APRUN}"
+
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
+elif [[ "${step}" = "atmanlvar" ]]; then
+
+    export NTHREADS_ATMANLVAR=${NTHREADSmax}
+    export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}"
+
+elif [[ "${step}" = "atmensanlletkf" ]]; then
+
+    export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
+    export APRUN_ATMENSANLLETKF="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}"
+
+elif [[ "${step}" = "atmensanlfv3inc" ]]; then
+
+    export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
+    export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"    
+
+elif [[ "${step}" = "aeroanlrun" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    export NTHREADS_AEROANL=${NTHREADSmax}
+    export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}"
+
+elif [[ "${step}" = "atmanlfv3inc" ]]; then
+
+    export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
+    export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
+
+elif [[ "${step}" = "prepobsaero" ]]; then
+
+    export NTHREADS_PREPOBSAERO=${NTHREADS1}
+    export APRUN_PREPOBSAERO="${APRUN} --cpus-per-task=${NTHREADS_PREPOBSAERO}"
+
+elif [[ "${step}" = "snowanl" ]]; then
+
+    export NTHREADS_SNOWANL=${NTHREADSmax}
+    export APRUN_SNOWANL="${APRUN} --cpus-per-task=${NTHREADS_SNOWANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
+elif [[ "${step}" = "marinebmat" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_MARINEBMAT="${APRUN}"
+
+elif [[ "${step}" = "marinebmat" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_MARINEBMAT="${APRUN}"
+
+elif [[ "${step}" = "ocnanalrun" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${APRUN}"
+
+elif [[ "${step}" = "ocnanalchkpt" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${APRUN}"
+
+elif [[ "${step}" = "ocnanalecen" ]]; then
+
+    export NTHREADS_OCNANALECEN=${NTHREADSmax}
+    export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+
+elif [[ "${step}" = "marineanalletkf" ]]; then
+
+    export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
+    export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    export NTHREADS_GSI=${NTHREADSmax}
+    export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-12}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    ntasks_cycle=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    export NTHREADS_GAUSFCANL=1
+    ntasks_gausfcanl=${ntasks_gausfcanl:-1}
+    export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
+
+elif [[ "${step}" = "sfcanl" ]]; then
+
+    export NTHREADS_CYCLE=${threads_per_task:-14}
+    export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}"
+
+elif [[ "${step}" = "eobs" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export NTHREADS_GSI=${NTHREADSmax}
+    export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}"
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+elif [[ "${step}" = "eupd" ]]; then
+
+    export NTHREADS_ENKF=${NTHREADSmax}
+    export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}} --cpus-per-task=${NTHREADS_ENKF}"
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+
+    export launcher="srun --mpi=pmi2 -l"
 
     export launcher="srun --mpi=pmi2 -l"
 
@@ -52,12 +198,26 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
 
 elif [[ "${step}" = "post" ]]; then
 
-    export NTHREADS_NP=${NTHREADS1}
-    export APRUN_NP="${APRUN}"
+    export NTHREADS_UPP=${NTHREADS1}
+    export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}"
+
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
+elif [[ "${step}" = "atmos_products" ]]; then
 
-    export NTHREADS_DWN=${threads_per_task_dwn:-1}
-    [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
-    export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
 
 elif [[ "${step}" = "atmos_products" ]]; then
 
@@ -71,33 +231,56 @@ elif [[ "${step}" = "oceanice_products" ]]; then
 elif [[ "${step}" = "ecen" ]]; then
 
     export NTHREADS_ECEN=${NTHREADSmax}
-    export APRUN_ECEN="${APRUN}"
+    export APRUN_ECEN="${APRUN} --cpus-per-task=${NTHREADS_ECEN}"
 
     export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
     [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
     export APRUN_CHGRES="time"
 
     export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
-    export APRUN_CALCINC="${APRUN}"
+    export APRUN_CALCINC="${APRUN} --cpus-per-task=${NTHREADS_CALCINC}"
 
 elif [[ "${step}" = "esfc" ]]; then
 
-    export NTHREADS_ESFC=${NTHREADSmax}
-    export APRUN_ESFC="${APRUN}"
+    export NTHREADS_ESFC=${threads_per_task_esfc:-${max_threads_per_task}}
+    export APRUN_ESFC="${APRUN} --cpus-per-task=${NTHREADS_ESFC}"
 
     export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
-    export APRUN_CYCLE="${APRUN}"
+    export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}"
 
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}
-    export APRUN_EPOS="${APRUN}"
+    export APRUN_EPOS="${APRUN} --cpus-per-task=${NTHREADS_EPOS}"
+
+elif [[ "${step}" = "postsnd" ]]; then
+
+    export CFP_MP="YES"
+
+    export NTHREADS_POSTSND=${NTHREADS1}
+    export APRUN_POSTSND="${APRUN} --cpus-per-task=${NTHREADS_POSTSND}"
+
+    export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
+    [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}
+    export APRUN_POSTSNDCFP="${launcher} -n ${ntasks_postsndcfp} ${mpmd_opt}"
+
+elif [[ "${step}" = "awips" ]]; then
+
+    export NTHREADS_AWIPS=${NTHREADS1}
+    export APRUN_AWIPSCFP="${APRUN} ${mpmd_opt}"
+
+elif [[ "${step}" = "gempak" ]]; then
+
+    export CFP_MP="YES"
+
+    export NTHREADS_GEMPAK=${NTHREADS1}
+    [[ ${NTHREADS_GEMPAK} -gt ${max_threads_per_task} ]] && export NTHREADS_GEMPAK=${max_threads_per_task}
 
 elif [[ "${step}" = "fit2obs" ]]; then
 
     export NTHREADS_FIT2OBS=${NTHREADS1}
-    export MPIRUN="${APRUN}"
+    export MPIRUN="${APRUN} --cpus-per-task=${NTHREADS_FIT2OBS}"
 
 fi

parm/config/gfs/config.base

@@ -488,6 +488,7 @@ if [[ "${machine}" =~ "PW" ]]; then
    export DO_TRACKER="NO"
    export DO_GENESIS="NO"
    export DO_METP="NO"
+   export DO_WAVE="NO"
 fi
 
 echo "END: config.base"

scripts/exgfs_wave_post_pnt.sh

@@ -393,14 +393,14 @@ source "${USHgfs}/preamble.sh"
     then
       export dtspec=3600.
       # Construct the wave_outp_spec (spec) command to run on each buoy in buoy_lst.txt
-      sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}\/spec_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}"
+      sed "s|^\(.*\)$|${escaped_USHgfs}/wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}/spec_\1.out 2>\&1|" buoy_lst.txt >> "tmpcmdfile.${FH3}"
     fi
 
     if [ "$DOBLL_WAV" = 'YES' ]
     then
       export dtspec=3600.
       # Construct the wave_outp_spec (bull) command to run on each buoy in buoy_lst.txt
-      sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}\/bull_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}"
+      sed "s|^\(.*\)$|${escaped_USHgfs}/wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}/bull_\1.out 2>\&1|" buoy_lst.txt >> "tmpcmdfile.${FH3}"
     fi
 
     split -n l/1/10  tmpcmdfile.$FH3 > cmdfile.${FH3}.01