From cfcbb14884f89689397be46a314012ba34218f75 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 11 Dec 2023 11:10:36 -0600 Subject: [PATCH 1/4] Initial port to Hercules #101 --- README.md | 2 +- modulefiles/hercules-run.lua | 21 +++++++++++++++++++ modulefiles/hercules.lua | 16 ++++++++++++++ parm/Mon_config | 14 ++++++++++++- .../data_extract/ush/ConMon_DE.sh | 3 ++- .../image_gen/ush/mk_horz_hist.sh | 13 +++++++----- .../image_gen/ush/mk_time_vert.sh | 15 +++++++------ src/Minimization_Monitor/README | 2 +- src/Ozone_Monitor/README | 2 +- src/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh | 4 ++-- src/Ozone_Monitor/image_gen/ush/mk_horiz.sh | 2 +- src/Ozone_Monitor/image_gen/ush/mk_summary.sh | 6 ++++-- src/Ozone_Monitor/image_gen/ush/mk_time.sh | 6 ++++-- src/Radiance_Monitor/README | 2 +- .../data_extract/ush/RadMon_DE_glb.sh | 3 ++- .../image_gen/ush/mk_angle_plots.sh | 13 +++++++----- .../image_gen/ush/mk_bcoef_plots.sh | 2 +- .../image_gen/ush/mk_bcor_plots.sh | 10 +++++---- .../image_gen/ush/mk_time_plots.sh | 12 ++++++----- ush/detect_machine.sh | 3 +++ ush/module-setup.sh | 9 +++++++- 21 files changed, 119 insertions(+), 41 deletions(-) create mode 100644 modulefiles/hercules-run.lua create mode 100644 modulefiles/hercules.lua diff --git a/README.md b/README.md index 009a06f0..08ae4faf 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ GSI Monitoring Tools These tools monitor the Gridpoint Statistical Interpolation (GSI) package's data assimiliation, detecting and reporting missing data sources, low observational counts, and high penalty values. These machines -are supported: wcoss2, hera, orion, jet, cheyenne, s4. +are supported: wcoss2, hera, hercules, orion, jet, cheyenne, s4. Suite includes: ``` diff --git a/modulefiles/hercules-run.lua b/modulefiles/hercules-run.lua new file mode 100644 index 00000000..6772bfbd --- /dev/null +++ b/modulefiles/hercules-run.lua @@ -0,0 +1,21 @@ +help([[ +]]) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" +local grads_ver=os.getenv("grads_ver") or "2.2.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" + +-- wgrib2 is a newer version on Hercules +setenv("wgrib2_ver", "3.1.1") + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("grads", grads_ver)) +load(pathJoin("prod_util", prod_util_ver)) + +load("common-run") + +whatis("Description: GSI Monitoring run-time environment on Hercules") diff --git a/modulefiles/hercules.lua b/modulefiles/hercules.lua new file mode 100644 index 00000000..dc75c004 --- /dev/null +++ b/modulefiles/hercules.lua @@ -0,0 +1,16 @@ +help([[ +]]) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) +load(pathJoin("cmake", cmake_ver)) + +load("common") + +whatis("Description: GSI Monitoring environment on Hercules with Intel Compilers") diff --git a/parm/Mon_config b/parm/Mon_config index 60b9cc55..c2afc845 100644 --- a/parm/Mon_config +++ b/parm/Mon_config @@ -30,7 +30,8 @@ export MY_MACHINE=$machine_id # run-time modules. If the run-time mod idea passes muster # then I'll open an issue for the change to those machines. # -if [[ $MY_MACHINE = "wcoss2" || $MY_MACHINE = "hera" || $MY_MACHINE = "orion" || $MY_MACHINE = "s4" || $MY_MACHINE = "jet" ]]; then +if [[ $MY_MACHINE = "wcoss2" || $MY_MACHINE = "hera" || $MY_MACHINE = "orion" || \ + $MY_MACHINE = "hercules" || $MY_MACHINE = "s4" || $MY_MACHINE = "jet" ]]; then source $DIR_ROOT/ush/module-setup.sh module use $DIR_ROOT/modulefiles module load ${MACHINE_ID}-run @@ -96,6 +97,17 @@ case $MY_MACHINE in export SUB="sbatch" #/opt/slurm/bin/sbatch export SERVICE_PARTITION="orion" + tankdir="/work/noaa/da/$USER/nbns" + ptmp="/work2/noaa/stmp/$USER" + stmp="/work/noaa/stmp/$USER" + queue="" + project="" + account="da-cpu" + + hercules) + export SUB="sbatch" #/opt/slurm/bin/sbatch + export SERVICE_PARTITION="hercules" + tankdir="/work/noaa/da/$USER/nbns" ptmp="/work2/noaa/stmp/$USER" stmp="/work/noaa/stmp/$USER" diff --git a/src/Conventional_Monitor/data_extract/ush/ConMon_DE.sh b/src/Conventional_Monitor/data_extract/ush/ConMon_DE.sh index af5210cd..e7a5a335 100755 --- a/src/Conventional_Monitor/data_extract/ush/ConMon_DE.sh +++ b/src/Conventional_Monitor/data_extract/ush/ConMon_DE.sh @@ -216,7 +216,8 @@ if [ -s $cnvstat -a -s $pgrbf00 -a -s $pgrbf06 ]; then rm -f ${logfile} fi - if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" || $MY_MACHINE = "orion" ]]; then + if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" || + $MY_MACHINE = "orion" || $MY_MACHINE = "hercules" ]]; then $SUB -A $ACCOUNT --ntasks=1 --time=00:30:00 \ -p ${SERVICE_PARTITION} -J ${jobname} -o $C_LOGDIR/DE.${PDY}.${CYC}.log \ ${HOMEgdas_conmon}/jobs/JGDAS_ATMOS_CONMON diff --git a/src/Conventional_Monitor/image_gen/ush/mk_horz_hist.sh b/src/Conventional_Monitor/image_gen/ush/mk_horz_hist.sh index 11cc3b33..d13a160c 100755 --- a/src/Conventional_Monitor/image_gen/ush/mk_horz_hist.sh +++ b/src/Conventional_Monitor/image_gen/ush/mk_horz_hist.sh @@ -56,8 +56,9 @@ rm -f ${errfile} fi - if [[ $MY_MACHINE == "hera" || $MY_MACHINE == "s4" || \ - $MY_MACHINE == "jet" || $MY_MACHINE == "orion" ]]; then + if [[ $MY_MACHINE == "hera" || $MY_MACHINE == "s4" || + $MY_MACHINE == "jet" || $MY_MACHINE == "orion" || + $MY_MACHINE == "hercules" ]]; then ${SUB} -A ${ACCOUNT} --ntasks=1 --time=00:20:00 \ -p ${SERVICE_PARTITION} -J ${jobname} -o ${logfile} ${plot_hist} @@ -83,8 +84,9 @@ rm -f ${errfile} fi - if [[ $MY_MACHINE == "hera" || $MY_MACHINE == "s4" || \ - $MY_MACHINE == "jet" || $MY_MACHINE == "orion" ]]; then + if [[ $MY_MACHINE == "hera" || $MY_MACHINE == "s4" || + $MY_MACHINE == "jet" || $MY_MACHINE == "orion" || + $MY_MACHINE == "hercules" ]]; then ${SUB} -A ${ACCOUNT} --ntasks=1 --time=00:20:00 \ -p ${SERVICE_PARTITION} -J ${jobname} -o ${logfile} ${plot_horz} @@ -111,7 +113,8 @@ fi if [[ ${MY_MACHINE} == "hera" || ${MY_MACHINE} == "s4" || \ - ${MY_MACHINE} == "jet" || ${MY_MACHINE} == "orion" ]]; then + ${MY_MACHINE} == "jet" || ${MY_MACHINE} == "orion" || + ${MY_MACHINE} == "hercules" ]]; then ${SUB} -A ${ACCOUNT} --ntasks=1 --time=01:30:00 \ -p ${SERVICE_PARTITION} -J ${jobname} -o ${logfile} ${plot_horz_uv} diff --git a/src/Conventional_Monitor/image_gen/ush/mk_time_vert.sh b/src/Conventional_Monitor/image_gen/ush/mk_time_vert.sh index c4a27765..74a59c78 100755 --- a/src/Conventional_Monitor/image_gen/ush/mk_time_vert.sh +++ b/src/Conventional_Monitor/image_gen/ush/mk_time_vert.sh @@ -27,8 +27,9 @@ echo "--> mk_time_vert.sh" rm -f $errfile fi - if [[ ${MY_MACHINE} == "hera" || ${MY_MACHINE} == "s4" || \ - ${MY_MACHINE} == "jet" || ${MY_MACHINE} == "orion" ]]; then + if [[ ${MY_MACHINE} == "hera" || ${MY_MACHINE} == "s4" || + ${MY_MACHINE} == "jet" || ${MY_MACHINE} == "orion" || + ${MY_MACHINE} == "hercules" ]]; then ${SUB} -A ${ACCOUNT} --ntasks=1 --time=00:15:00 \ -p ${SERVICE_PARTITION} -J ${jobname} -o ${logfile} ${pltfile} @@ -55,8 +56,9 @@ echo "--> mk_time_vert.sh" rm -f $errfile fi - if [[ ${MY_MACHINE} == "hera" || ${MY_MACHINE} == "s4" || \ - ${MY_MACHINE} == "jet" || ${MY_MACHINE} == "orion" ]]; then + if [[ ${MY_MACHINE} == "hera" || ${MY_MACHINE} == "s4" || + ${MY_MACHINE} == "jet" || ${MY_MACHINE} == "orion" || + ${MY_MACHINE} == "hercules" ]]; then if [[ ${type} == "uv" || ${type} == "u" || ${type} == "v" ]]; then walltime="02:30:00" else @@ -99,8 +101,9 @@ echo "--> mk_time_vert.sh" rm -f $errfile fi - if [[ ${MY_MACHINE} == "hera" || ${MY_MACHINE} == "s4" || \ - ${MY_MACHINE} == "jet" || ${MY_MACHINE} = "orion" ]]; then + if [[ ${MY_MACHINE} == "hera" || ${MY_MACHINE} == "s4" || + ${MY_MACHINE} == "jet" || ${MY_MACHINE} = "orion" || + ${MY_MACHINE} == "hercules" ]]; then if [[ ${type} == "uv" || ${type} == "u" || ${type} == "v" ]]; then walltime="00:50:00" else diff --git a/src/Minimization_Monitor/README b/src/Minimization_Monitor/README index a89c3d65..fcde059e 100644 --- a/src/Minimization_Monitor/README +++ b/src/Minimization_Monitor/README @@ -2,7 +2,7 @@ README MinMon package The Minimization Monitor (MinMon) provides a means to visualize and report on the performance of the GSI minimization function. The package -is supported on wcoss2, hera, orion, cheyenne, jet, and s4 machines. +is supported on wcoss2, hera, orion, hercules, cheyenne, jet, and s4 machines. The package is organized in two main processes: data_extract and image_gen (image generation). The data extract piece is now located in the diff --git a/src/Ozone_Monitor/README b/src/Ozone_Monitor/README index 7d53bfd3..2a668ad3 100644 --- a/src/Ozone_Monitor/README +++ b/src/Ozone_Monitor/README @@ -3,7 +3,7 @@ README OznMon package The OznMon (ozone monitoring) package can be used to extract information from ozone diagnostic files and generate image plots to visualize the results. The package also may optionally perform data validation and error checking. -The package is supported on wcoss2, hera, orion, cheyenne, jet, and s4 +The package is supported on wcoss2, hera, hercules, orion, cheyenne, jet, and s4 machines. The package is organized in two processes, the data_extract and image_gen diff --git a/src/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh b/src/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh index 322dbcb1..3e41cfb9 100755 --- a/src/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh +++ b/src/Ozone_Monitor/data_xtrct/ush/OznMon_CP.sh @@ -203,8 +203,8 @@ if compgen -G "${OZN_DATA_DIR}/time/*${PDATE}*.ieee_d*" > /dev/null; then $SUB --account=${ACCOUNT} --time=10 -J ${jobname} -D . \ -o ${logfile} --ntasks=1 --mem=5g ${job} - elif [[ $MY_MACHINE = "orion" ]]; then - echo submit job on orion + elif [[ $MY_MACHINE = "orion" || $MY_MACHINE = "hercules" ]]; then + echo submit job on orion or hercules $SUB --account=${ACCOUNT} --time=10 -J ${jobname} -D . \ -o ${logfile} --ntasks=1 --mem=5g ${job} fi diff --git a/src/Ozone_Monitor/image_gen/ush/mk_horiz.sh b/src/Ozone_Monitor/image_gen/ush/mk_horiz.sh index 806a358a..38d769bc 100755 --- a/src/Ozone_Monitor/image_gen/ush/mk_horiz.sh +++ b/src/Ozone_Monitor/image_gen/ush/mk_horiz.sh @@ -51,7 +51,7 @@ for dsrc in ${data_source}; do fi if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || ${MY_MACHINE} = "s4" || - ${MY_MACHINE} = "orion" ]]; then + ${MY_MACHINE} = "orion" || ${MY_MACHINE} = "hercules" ]]; then echo "$ctr ${OZN_IG_SCRIPTS}/plot_horiz.sh $type $suffix '$list' $dsrc" >> $cmdfile else echo "${OZN_IG_SCRIPTS}/plot_horiz.sh $type $suffix '$list' $dsrc" >> $cmdfile diff --git a/src/Ozone_Monitor/image_gen/ush/mk_summary.sh b/src/Ozone_Monitor/image_gen/ush/mk_summary.sh index d43437b9..f6e1aab7 100755 --- a/src/Ozone_Monitor/image_gen/ush/mk_summary.sh +++ b/src/Ozone_Monitor/image_gen/ush/mk_summary.sh @@ -68,7 +68,8 @@ for ptype in ${data_source}; do if [[ $type != "omi_aura" && $type != "gome_metop-a" && \ $type != "gome_metop-b" && $type != "ompstc8_npp" && $type != "ompstc8_n20" ]]; then if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || - ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" ]]; then + ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" || + ${MY_MACHINE} = "hercules" ]]; then echo "${ctr} ${OZN_IG_SCRIPTS}/plot_summary.sh $type $ptype" >> $cmdfile else echo "${OZN_IG_SCRIPTS}/plot_summary.sh $type $ptype" >> $cmdfile @@ -95,7 +96,8 @@ for ptype in ${data_source}; do fi - if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" ]]; then + if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" || + ${MY_MACHINE} = "hercules" ]]; then $SUB --account ${ACCOUNT} -n $ctr -o ${logf} -D . -J ${job} --time=10 \ --wrap "srun -l --multi-prog ${cmdfile}" diff --git a/src/Ozone_Monitor/image_gen/ush/mk_time.sh b/src/Ozone_Monitor/image_gen/ush/mk_time.sh index 4b539070..5b4f2f32 100755 --- a/src/Ozone_Monitor/image_gen/ush/mk_time.sh +++ b/src/Ozone_Monitor/image_gen/ush/mk_time.sh @@ -50,7 +50,8 @@ for dsrc in ${data_source}; do >$cmdfile for type in ${SATYPE}; do if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || - ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" ]]; then + ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" || + ${MY_MACHINE} = "hercules" ]]; then echo "${ctr} ${OZN_IG_SCRIPTS}/plot_time.sh $type $suffix '$list' $dsrc" >> $cmdfile ((ctr=ctr+1)) else @@ -71,7 +72,8 @@ for dsrc in ${data_source}; do rm -f $errf fi - if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" ]]; then + if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "s4" || + ${MY_MACHINE} = "orion" || ${MY_MACHINE} = "hercules" ]]; then $SUB --account ${ACCOUNT} -n $ctr -o ${logf} -D . -J ${job} --time=10 \ --wrap "srun -l --multi-prog ${cmdfile}" diff --git a/src/Radiance_Monitor/README b/src/Radiance_Monitor/README index 98c97988..ed37da28 100644 --- a/src/Radiance_Monitor/README +++ b/src/Radiance_Monitor/README @@ -4,7 +4,7 @@ The RadMon (radiance monitoring) package can be used to extract data from radiance diagnostic files and visualize the results by plotting the data using javascript or optionally GrADS. The package also may optionally perform data validation and error checking. The package is supported on wcoss2, hera, -orion, cheyenne, jet, and s4 machines. +orion, hercules, cheyenne, jet, and s4 machines. The package is organized in two main processes, data_extract and image_gen (image generation). There is also an nwprod directory, which contains the lower diff --git a/src/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh b/src/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh index 0935f7b3..596daba4 100755 --- a/src/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh +++ b/src/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh @@ -236,7 +236,8 @@ if [[ -e ${radstat} && -e ${biascr} ]]; then fi - if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "orion" ]]; then + if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "orion" || + $MY_MACHINE = "hercules" ]]; then $SUB --account=${ACCOUNT} --time=10 -J ${jobname} -D . \ -o ${logfile} --ntasks=1 --mem=5g ${job} diff --git a/src/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh b/src/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh index 3acf5a8a..1b5410ed 100755 --- a/src/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh +++ b/src/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh @@ -232,7 +232,8 @@ while [[ $ctr -le ${satarr_len} ]]; do # to the cmdfile # if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || - ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" ]]; then + ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" || + ${MY_MACHINE} = "hercules" ]]; then echo "${itemctr} ${IG_SCRIPTS}/plot_angle.sh ${type} ${suffix} '${list}'" >> ${cmdfile} else echo "${IG_SCRIPTS}/plot_angle.sh ${type} ${suffix} '${list}'" >> ${cmdfile} @@ -252,7 +253,8 @@ while [[ $ctr -le ${satarr_len} ]]; do errfile=${R_LOGDIR}/plot_angle_${suffix}_${jobctr}.err echo "TASKS= $tasks" - if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" ]]; then + if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" || + ${MY_MACHINE} = "hercules" ]]; then $SUB --account ${ACCOUNT} -n ${itemctr} -o ${logfile} -D . -J ${jobname} --time=30:00 \ --wrap "srun -l --multi-prog ${cmdfile}" @@ -314,9 +316,10 @@ for sat in ${big_satlist}; do -V -l walltime=60:00 -N ${jobname} ${cmdfile} #--------------------------------------------------- - # hera|jet|s4|orion, submit 1 job for each sat/list item + # hera|jet|s4|orion|hercules, submit 1 job for each sat/list item elif [[ $MY_MACHINE = "hera" || $MY_MACHINE = "jet" || \ - $MY_MACHINE = "s4" || $MY_MACHINE = "orion" ]]; then + $MY_MACHINE = "s4" || $MY_MACHINE = "orion" || + $MY_MACHINE = "hercules" ]]; then ii=0 logfile=${R_LOGDIR}/plot_angle_${sat}.log @@ -338,7 +341,7 @@ for sat in ${big_satlist}; do $SUB --account ${ACCOUNT} -n $ii -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ --mem=0 --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "orion" ]]; then + elif [[ $MY_MACHINE = "orion" || $MY_MACHINE = "hercules" ]]; then $SUB --account ${ACCOUNT} -n $ii -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ -p ${SERVICE_PARTITION} --mem=0 --wrap "srun -l --multi-prog ${cmdfile}" diff --git a/src/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh b/src/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh index 1ef90de0..a8e5c8d4 100755 --- a/src/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh +++ b/src/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh @@ -164,7 +164,7 @@ if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then $SUB --account $ACCOUNT --ntasks=1 --mem=5g --time=1:00:00 -J ${jobname} \ -o ${logfile} -D . $IG_SCRIPTS/plot_bcoef.sh -elif [[ $MY_MACHINE = "orion" ]]; then +elif [[ $MY_MACHINE = "orion" || $MY_MACHINE = "hercules" ]]; then $SUB --account $ACCOUNT --ntasks=1 --mem=5g --time=20 -J ${jobname} \ -p ${SERVICE_PARTITION} -o ${logfile} -D . $IG_SCRIPTS/plot_bcoef.sh diff --git a/src/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh b/src/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh index 3acd742e..3412967a 100755 --- a/src/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh +++ b/src/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh @@ -187,7 +187,8 @@ rm -f ${logfile} ctr=0 for sat in ${SATLIST}; do if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "jet" || - $MY_MACHINE = "s4" || $MY_MACHINE = "orion" ]]; then + $MY_MACHINE = "s4" || $MY_MACHINE = "orion" || + $MY_MACHINE = "hercules" ]]; then echo "${ctr} $IG_SCRIPTS/plot_bcor.sh $sat $suffix '$plot_list'" >> $cmdfile else echo "$IG_SCRIPTS/plot_bcor.sh $sat $suffix '$plot_list'" >> $cmdfile @@ -208,7 +209,7 @@ if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then $SUB --account ${ACCOUNT} -n $ctr -o ${logfile} -D . -J ${jobname} \ --time=2:00:00 --wrap "srun -l --multi-prog ${cmdfile}" -elif [[ $MY_MACHINE = "orion" ]]; then +elif [[ $MY_MACHINE = "orion" || $MY_MACHINE = "hercules" ]]; then $SUB --account ${ACCOUNT} -n $ctr -o ${logfile} -D . -J ${jobname} --time=2:00:00 \ -p ${SERVICE_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" @@ -245,7 +246,8 @@ for sat in ${bigSATLIST}; do ctr=0 for var in $plot_list; do if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "jet" || - $MY_MACHINE = "s4" || $MY_MACHINE = "orion" ]]; then + $MY_MACHINE = "s4" || $MY_MACHINE = "orion" || + $MY_MACHINE = "hercules" ]]; then echo "$ctr $IG_SCRIPTS/plot_bcor.sh $sat $var $var" >> $cmdfile else echo "$IG_SCRIPTS/plot_bcor.sh $sat $var $var" >> $cmdfile @@ -265,7 +267,7 @@ for sat in ${bigSATLIST}; do $SUB --account ${ACCOUNT} -n $ctr -o ${logfile} -D . -J ${jobname} \ --time=1:00:00 --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "orion" ]]; then + elif [[ $MY_MACHINE = "orion" || $MY_MACHINE = "hercules" ]]; then $SUB --account ${ACCOUNT} -n $ctr -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ -p ${SERVICE_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}" diff --git a/src/Radiance_Monitor/image_gen/ush/mk_time_plots.sh b/src/Radiance_Monitor/image_gen/ush/mk_time_plots.sh index ed6dae04..30916f3b 100755 --- a/src/Radiance_Monitor/image_gen/ush/mk_time_plots.sh +++ b/src/Radiance_Monitor/image_gen/ush/mk_time_plots.sh @@ -189,7 +189,7 @@ if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "s4" ]]; then ${SUB} --account ${ACCOUNT} --ntasks=1 --mem=5g --time=1:00:00 -J ${jobname} \ -o ${logfile} ${IG_SCRIPTS}/plot_summary.sh -elif [[ ${MY_MACHINE} = "orion" ]]; then +elif [[ ${MY_MACHINE} = "orion" || ${MY_MACHINE} = "hercules" ]]; then ${SUB} --account ${ACCOUNT} --ntasks=1 --mem=5g --time=20:00 -J ${jobname} \ -o ${logfile} ${IG_SCRIPTS}/plot_summary.sh @@ -247,7 +247,8 @@ ctr=0 for sat in ${SATLIST}; do if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || - ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" ]]; then + ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" || + ${MY_MACHINE} = "hercules" ]]; then echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile else echo "$IG_SCRIPTS/plot_time.sh $sat $suffix '$list'" >> $cmdfile @@ -261,7 +262,7 @@ if [[ $MY_MACHINE = "hera" || $MY_MACHINE = "s4" ]]; then $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ --wrap "srun -l --multi-prog ${cmdfile}" -elif [[ $MY_MACHINE = "orion" ]]; then +elif [[ $MY_MACHINE = "orion" || $MY_MACHINE = "hercules" ]]; then $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:00:00 \ -p $SERVICE_PARTITION --wrap "srun -l --multi-prog ${cmdfile}" @@ -299,7 +300,8 @@ for sat in ${bigSATLIST}; do ctr=0 for var in $list; do if [[ ${MY_MACHINE} = "hera" || ${MY_MACHINE} = "jet" || - ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" ]]; then + ${MY_MACHINE} = "s4" || ${MY_MACHINE} = "orion" || + ${MY_MACHINE} = "hercules" ]]; then echo "${ctr} $IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile else echo "$IG_SCRIPTS/plot_time.sh $sat $var $var" >> $cmdfile @@ -317,7 +319,7 @@ for sat in ${bigSATLIST}; do $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=4:00:00 \ --wrap "srun -l --multi-prog ${cmdfile}" - elif [[ $MY_MACHINE = "orion" ]]; then + elif [[ $MY_MACHINE = "orion" || $MY_MACHINE = "hercules" ]]; then $SUB --account ${ACCOUNT} -n ${ctr} -o ${logfile} -D . -J ${jobname} --time=1:30:00 \ -p $SERVICE_PARTITION --wrap "srun -l --multi-prog ${cmdfile}" diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index ecd1ad53..7b317bd8 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -1,5 +1,6 @@ #!/bin/bash +echo $(hostname -f) case $(hostname -f) in adecflow0[12].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn @@ -25,6 +26,8 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 + Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 + cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 chadmin[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 469fd4a3..a62d95be 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -18,7 +18,14 @@ elif [[ $MACHINE_ID = hera* ]] ; then elif [[ $MACHINE_ID = orion* ]] ; then # We are on Orion if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/init/bash + source /apps/lmod/lmod/init/bash + fi + module purge + +elif [[ $MACHINE_ID = hercules* ]] ; then + # We are on Hercules + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/other/lmod/lmod/init/bash fi module purge From dd9534a9605834898d9a629d7f592de6dd014864 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 18 Dec 2023 14:06:50 -0600 Subject: [PATCH 2/4] Update detect_machine.sh for Hercules. #107 --- ush/detect_machine.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 7b317bd8..9454998f 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -1,6 +1,5 @@ #!/bin/bash -echo $(hostname -f) case $(hostname -f) in adecflow0[12].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn @@ -26,7 +25,7 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 - Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 + [Hh]ercules-login-[1-4].[Hh][Pp][Cc].[Mm]s[Ss]tate.[Ed]du) MACHINE_ID=hercules ;; ### hercules1-4 cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 From f9bbb04cfe42375a784c18e9309ef5eda34a76b1 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Mar 2024 12:30:28 -0600 Subject: [PATCH 3/4] Rename Hercules modules. --- modulefiles/{hercules-run.lua => hercules.intel-run.lua} | 0 modulefiles/{hercules.lua => hercules.intel.lua} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename modulefiles/{hercules-run.lua => hercules.intel-run.lua} (100%) rename modulefiles/{hercules.lua => hercules.intel.lua} (100%) diff --git a/modulefiles/hercules-run.lua b/modulefiles/hercules.intel-run.lua similarity index 100% rename from modulefiles/hercules-run.lua rename to modulefiles/hercules.intel-run.lua diff --git a/modulefiles/hercules.lua b/modulefiles/hercules.intel.lua similarity index 100% rename from modulefiles/hercules.lua rename to modulefiles/hercules.intel.lua From 1340d3f4a5adb87b02ce244e601021a0e8c8b816 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 1 Mar 2024 12:31:26 -0600 Subject: [PATCH 4/4] Upgrade Hercules to spack-stack v1.6.0; remove perl module Use system perl instead. New Perl modules were installed during the last system maintainance period. --- modulefiles/hercules.intel-run.lua | 2 +- modulefiles/hercules.intel.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modulefiles/hercules.intel-run.lua b/modulefiles/hercules.intel-run.lua index 6772bfbd..434f0880 100644 --- a/modulefiles/hercules.intel-run.lua +++ b/modulefiles/hercules.intel-run.lua @@ -1,7 +1,7 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" diff --git a/modulefiles/hercules.intel.lua b/modulefiles/hercules.intel.lua index dc75c004..e2ffedb5 100644 --- a/modulefiles/hercules.intel.lua +++ b/modulefiles/hercules.intel.lua @@ -1,7 +1,7 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"