Ref NOAA-EMC#310

Add error file output to submitted jobs.
EdwardSafford-NOAA · Feb 24, 2022 · fc3dfb1 · fc3dfb1
1 parent 920e34c
commit fc3dfb1
Show file tree

Hide file tree

Showing 10 changed files with 42 additions and 21 deletions.
diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_CP_glb.sh
@@ -192,8 +192,8 @@ if [[  -d ${DATA_LOCATION} ]]; then
         -o ${logfile} --ntasks=1 --mem=5g ${job}
 
    elif [[ $MY_MACHINE = "wcoss2" ]]; then
-      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-           -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/CP.${PDY}.${CYC}.err \
+	   -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
    fi
 else
    echo "Unable to locate DATA_LOCATION: ${DATA_LOCATION}"

diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_glb.sh
@@ -248,8 +248,8 @@ if [[ -e ${radstat} && -e ${biascr} ]]; then
         -o ${logfile} --ntasks=1 --mem=5g ${job} 
 
    elif [[ $MY_MACHINE = "wcoss2" ]]; then
-      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-        -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${pdy}.${cyc}.err \
+        -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
    fi
 
 else  # radstat and/or biascr not found

diff --git a/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh b/util/Radiance_Monitor/data_extract/ush/RadMon_DE_rgn.sh
@@ -306,8 +306,8 @@ elif [[ $MY_MACHINE = "hera" ]]; then
         -j oe -o ${logfile} ${job}
 
 elif [[ $MY_MACHINE = "wcoss2" ]]; then
-   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-        -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
+   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/DE.${PDY}.${cyc}.err \
+        -V -l select=1:mem=5000M -l walltime=20:00 -N ${jobname} ${job}
 fi
 
 

diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_glb.sh
@@ -335,8 +335,8 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then
          cmdfile=transfer_cmd
          echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile
 
-         $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \
-              -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
+         $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err 
+	      -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
       else
          $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \
               -R affinity[core] -J ${jobname} -cwd ${PWD} -b $run_time ${job}

diff --git a/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh b/util/Radiance_Monitor/image_gen/ush/RadMon_IG_rgn.sh
@@ -286,8 +286,8 @@ if [[ $RUN_TRANSFER -eq 1 ]]; then
 	    cmdfile=transfer_cmd
 	    echo "${IG_SCRIPTS}/Transfer.sh --nosrc ${RADMON_SUFFIX}" >$cmdfile
 
-	    $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -V \
-	         -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
+	    $SUB -q $transfer_queue -A $ACCOUNT -o ${transfer_log} -e ${LOGdir}/Transfer_${RADMON_SUFFIX}.err 
+	         -V -l select=1:mem=500M -l walltime=45:00 -N ${jobname} ${cmdfile}
 
 	 else
             $SUB -P $PROJECT -q $transfer_queue -o ${transfer_log} -M 80 -W 0:45 \

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_angle_plots.sh
@@ -200,8 +200,8 @@ elif [[ ${MY_MACHINE} = "wcoss_c" ]]; then
         -J ${jobname} -cwd ${PWD} $cmdfile
 
 elif [[ $MY_MACHINE = "wcoss2" ]]; then
-   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-        -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile}
+   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_angle_${suffix}.err \
+	-V -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile}
 fi
 
 
@@ -218,7 +218,7 @@ echo "starting big_satlist"
 for sat in ${big_satlist}; do
    echo processing $sat in $big_satlist
 
-   if [[ ${MY_MACHINE} = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then 	
+   if [[ ${MY_MACHINE} = "wcoss_d" || $MY_MACHINE = "wcoss_c" || $MY_MACHINE = "wcoss2" ]]; then 	
 
       cmdfile=${PLOT_WORK_DIR}/cmdfile_pangle_${sat}
       if [[ -e ${cmdfile} ]]; then
@@ -242,6 +242,10 @@ for sat in ${big_satlist}; do
       elif [[ $MY_MACHINE = "wcoss_c" ]]; then
          $SUB -q $JOB_QUEUE -P $PROJECT -o ${logfile} -M 600 -W ${wall_tm} \
               -J ${jobname} -cwd ${PWD} $cmdfile
+
+      elif [[ $MY_MACHINE = "wcoss2" ]]; then
+         $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_angle_${suffix}.err \
+              -V -l select=1:mem=1g -l walltime=30:00 -N ${jobname} ${cmdfile}
       fi
 
 

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_bcoef_plots.sh
@@ -135,7 +135,7 @@ elif [[ $MY_MACHINE = "jet" ]]; then
         -p ${RADMON_PARTITION} -o ${logfile} -D . $IG_SCRIPTS/plot_bcoef.sh
 
 elif [[ $MY_MACHINE = "wcoss2" ]]; then
-   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
+   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e $LOGdir/plot_bcoef.err -V \
         -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} $IG_SCRIPTS/plot_bcoef.sh
 fi
 

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_bcor_plots.sh
@@ -189,6 +189,10 @@ elif [[ $MY_MACHINE = "jet" ]]; then
    $SUB --account ${ACCOUNT} -n $ctr  -o ${logfile} -D . -J ${jobname} \
         -p ${RADMON_PARTITION} --time=2:00:00 --wrap "srun -l --multi-prog ${cmdfile}"
 
+elif [[ $MY_MACHINE = "wcoss2" ]]; then
+   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_bcor_${suffix}.err \
+        -V -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile}
+
 fi
 
 
@@ -246,8 +250,8 @@ for sat in ${bigSATLIST}; do
            -p ${RADMON_PARTITION} --time=1:00:00 --wrap "srun -l --multi-prog ${cmdfile}"
 
    elif [[ $MY_MACHINE = "wcoss2" ]]; then
-      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
-           -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile}
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_bcor_${suffix}.err \
+	   -V -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile}
    fi
 
    echo "submitted $sat"

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_horiz_plots.sh
@@ -160,7 +160,7 @@ done
 #  submit the plot jobs
 #
 
-if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then
+if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" || $MY_MACHINE = "wcoss2" ]]; then
    cmdfile="./cmdfile_horiz_${RADMON_SUFFIX}_${PID}"
    logfile=${LOGdir}/horiz_${PID}.log
    rm -f $cmdfile
@@ -177,9 +177,11 @@ if [[ $MY_MACHINE = "wcoss_d" || $MY_MACHINE = "wcoss_c" ]]; then
    if [[ $MY_MACHINE = "wcoss_d" ]]; then
       $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \
            -W 0:45 -J ${jobname} -cwd ${PWD} $cmdfile
-   else
+
+   elif [[ $MY_MACHINE = "wcoss_c" ]]; then
       $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 0:45 \
            -J ${jobname} -cwd ${PWD} $cmdfile
+
    fi
 
 else							# hera|jet|s4
@@ -225,9 +227,14 @@ for sat in ${bigSATLIST}; do
    if [[ $MY_MACHINE = "wcoss_d" ]]; then
       $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \
            -W 2:45 -J ${jobname} -cwd ${PWD} $cmdfile
+
    elif [[ $MY_MACHINE = "wcoss_c" ]]; then
       $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 2:45 \
            -J ${jobname} -cwd ${PWD} $cmdfile
+
+   elif [[ $MY_MACHINE = "wcoss2" ]]; then
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/horiz_${PID}.err \
+           -V -l select=1:mem=1g -l walltime=2:00:00 -N ${jobname} ${cmdfile}
    else
       $SUB -A $ACCOUNT -l procs=${ntasks},walltime=2:00:00 -N ${jobname} \
            -V -j oe -o $LOGdir/horiz_${PID}.log $cmdfile
@@ -248,9 +255,15 @@ for sat in ${bigSATLIST}; do
    if [[ $MY_MACHINE = "wcoss_d" ]]; then
       $SUB -q $JOB_QUEUE -P $PROJECT -R affinity[core] -M 500 -o ${logfile} \
            -W 2:45 -J ${jobname} -cwd ${PWD} $cmdfile
+
    elif [[ $MY_MACHINE = "wcoss_c" ]]; then
       $SUB -q $JOB_QUEUE -P $PROJECT -M 500 -o ${logfile} -W 2:45 \
            -J ${jobname} -cwd ${PWD} $cmdfile
+
+   elif [[ $MY_MACHINE = "wcoss2" ]]; then
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/horiz_${PID}.err \
+           -V -l select=1:mem=1g -l walltime=2:00:00 -N ${jobname} ${cmdfile}
+
    else
       $SUB -A $ACCOUNT -l procs=${ntasks},walltime=2:00:00 -N ${jobname} \
            -V -j oe -o $LOGdir/horiz_${PID}.log $cmdfile

diff --git a/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh b/util/Radiance_Monitor/image_gen/ush/mk_time_plots.sh
@@ -152,7 +152,7 @@ elif [[ ${MY_MACHINE} = "jet" ]]; then
           --partition ${RADMON_PARTITION} -o ${logfile} ${IG_SCRIPTS}/plot_summary.sh
 
 elif [[ $MY_MACHINE = "wcoss2" ]]; then
-   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
+   $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_summary.err -V \
           -l select=1:mem=1g -l walltime=10:00 -N ${jobname} ${IG_SCRIPTS}/plot_summary.sh
 fi
 
@@ -231,7 +231,7 @@ list="count penalty omgnbc total omgbc"
            -J ${jobname} -cwd ${PWD} ${cmdfile}
 
    elif [[ $MY_MACHINE = "wcoss2" ]]; then
-      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
+      $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_time_${suffix}.err -V \
            -l select=1:mem=1g -l walltime=1:00:00 -N ${jobname} ${cmdfile}
    fi
 
@@ -287,7 +287,7 @@ list="count penalty omgnbc total omgbc"
               -p ${RADMON_PARTITION} --wrap "srun -l --multi-prog ${cmdfile}"
 
       elif [[ $MY_MACHINE = "wcoss2" ]]; then
-         $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -V \
+         $SUB -q $JOB_QUEUE -A $ACCOUNT -o ${logfile} -e ${LOGdir}/plot_time_${sat}.err -V \
               -l select=1:mem=1g -l walltime=1:30:00 -N ${jobname} ${cmdfile}
       fi