ci: Switch to /usr/bin/time for measuring time + memory in physmon (#…

…3007)

I'm hoping this circumvents the problem of missing memory and time measurements. They're possibly less accurate, but less buggy for now. Should produce the same CSV as before, and I still use the custom python tool to convert them into a combined CSV that's published on the tracker.

.github/workflows/builds.yml

@@ -161,7 +161,7 @@ jobs:
 
     steps:
       - name: Install git lfs
-        run: apt-get update && apt-get install -y git-lfs
+        run: apt-get update && apt-get install -y git-lfs time
 
       - uses: actions/checkout@v3
         with:

.gitlab-ci.yml

@@ -221,7 +221,7 @@ linux_physmon:
     expire_in: 1 week
 
   script:
-    - apt-get update && apt-get install -y git-lfs
+    - apt-get update && apt-get install -y git-lfs time
 
     - git clone $CLONE_URL src
     - cd src

CI/physmon/phys_perf_mon.sh

@@ -1,7 +1,18 @@
 #!/bin/bash
 
 set -e
-set -x
+
+# helper function to selectively print and run commands without a subshell
+function run() {
+    set -x
+    "$@"
+    { set +x;   } 2> /dev/null
+}
+
+export run
+
+
+shopt -s extglob
 
 
 mode=${1:-all}
@@ -22,12 +33,74 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}"  )" &> /dev/null && pwd
 histcmp_results=$outdir/histcmp_results.csv
 echo -n "" > $histcmp_results
 
-SPYRAL_BIN=${SPYRAL_BIN:=spyral}
-SPYRAL="${SPYRAL_BIN} run -i 0.05 --summary"
-
-mkdir -p ${outdir}/memory
+memory_dir=${outdir}/memory
+mkdir -p "$memory_dir"
+
+if [ "$(uname)" == "Darwin" ]; then
+    function measure {
+        label=$1
+        slug=$2
+        shift
+        shift
+        echo "Measure Darwin $label ($slug)" >&2
+        tmp=$(mktemp)
+        echo "+ $@" >&2
+        /usr/bin/time -l -o "$tmp" "$@"
+
+        of="${memory_dir}/mem_${slug}.csv"
+        {
+            echo "# spyral-label: $label"
+            echo "# spyral-cmd: $*"
+            echo "time,rss,vms"
+            # in bytes
+            grep -E "real" "$tmp" | awk '{printf $1}'
+            printf ","
+            grep -E "maximum resident set size" "$tmp" | awk '{printf $1}'
+            printf ",0\n"
+        } > "$of"
+    }
+    export measure
+elif [ "$(uname)" == "Linux" ]; then
+    function measure {
+        label=$1
+        slug=$2
+        shift
+        shift
+        echo "Measure Linux $label ($slug)" >&2
+        tmp=$(mktemp)
+        echo "+ $@" >&2
+        /usr/bin/time -v -o  "$tmp" "$@"
+        # in kbytes
+        max_rss=$(grep "Maximum resident set size (kbytes):" "$tmp" | awk '{printf $(NF)}')
+        max_rss=$(( 1000*max_rss ))
+        wall_time=$(grep "Elapsed (wall clock)" "$tmp" | awk '{printf $(NF)}')
+        echo $max_rss
+        wall_time=$(python3 -c "i='${wall_time}';p=i.split(':');p = p if len(p) == 3 else ['0', *p];t=float(p[0])*60*60 + float(p[1])*60 + float(p[2]);print(t)")
+        echo $wall_time
+
+        of="${memory_dir}/mem_${slug}.csv"
+        {
+            echo "# spyral-label: $label"
+            echo "# spyral-cmd: $*"
+            echo "time,rss,vms"
+            echo "${wall_time},${max_rss},0"
+        } > "$of"
+    }
+    export measure
+else
+    function measure {
+        echo "Not measuring because unknown environment"
+        shift
+        shift
+        "$@"
+    }
+    export measure
+fi
 
-set +e
+if [ -n "$CI" ]; then
+    echo "CI mode, do not abort immediately on failure"
+    set +e
+fi
 ec=0
 
 source $SCRIPT_DIR/setup.sh
@@ -38,18 +111,16 @@ function run_physmon_gen() {
 
     script=CI/physmon/workflows/physmon_${slug}.py
 
-    $SPYRAL -l "$title" -o "$outdir/memory/mem_${slug}.csv" -- ${script} $outdir 2>&1 > $outdir/run_${slug}.log
+    measure "$title" "$slug" ${script} $outdir 2>&1 > $outdir/run_${slug}.log
 
     this_ec=$?
     ec=$(($ec | $this_ec))
 
     if [ $this_ec -ne 0 ]; then
       echo "::error::🟥 Dataset generation failed: ${script} -> ec=$this_ec"
     else
-      echo "::notice::✅ Dataset generation succeeded: ${script}"   
+      echo "::notice::✅ Dataset generation succeeded: ${script}"
     fi
-
-    $SPYRAL_BIN plot $outdir/memory/mem_${slug}.csv --output $outdir/memory
 }
 
 echo "::group::Generate validation dataset"
@@ -95,7 +166,7 @@ function run_histcmp() {
       ec=1
     fi
 
-    histcmp $a $b \
+    run histcmp $a $b \
         --label-reference=reference \
         --label-monitored=monitored \
         --title="$title" \
@@ -124,7 +195,7 @@ function full_chain() {
         config="CI/physmon/default.yml"
     fi
     echo $config
-    
+
     if [ $suffix != truth_smeared ]; then
       run_histcmp \
         $outdir/performance_seeding_${suffix}.root \
@@ -133,15 +204,15 @@ function full_chain() {
         seeding_${suffix} \
         -c $config
     fi
-    
+
     run_histcmp \
         $outdir/performance_ckf_${suffix}.root \
         $refdir/performance_ckf_${suffix}.root \
         "CKF ${suffix}" \
         ckf_${suffix} \
         -c $config
 
-    Examples/Scripts/generic_plotter.py \
+    run Examples/Scripts/generic_plotter.py \
         $outdir/performance_ivf_${suffix}.root \
         vertexing \
         $outdir/performance_ivf_${suffix}_hist.root \
@@ -158,7 +229,7 @@ function full_chain() {
         "IVF ${suffix}" \
         ivf_${suffix}
 
-    Examples/Scripts/generic_plotter.py \
+    run Examples/Scripts/generic_plotter.py \
         $outdir/performance_amvf_${suffix}.root \
         vertexing \
         $outdir/performance_amvf_${suffix}_hist.root \
@@ -176,7 +247,7 @@ function full_chain() {
         amvf_${suffix}
 
     if [ $suffix == seeded ]; then
-	    Examples/Scripts/generic_plotter.py \
+	    run Examples/Scripts/generic_plotter.py \
             $outdir/performance_amvf_gridseeder_${suffix}.root \
             vertexing \
             $outdir/performance_amvf_gridseeder_${suffix}_hist.root \
@@ -194,7 +265,7 @@ function full_chain() {
             amvf_gridseeder_${suffix}
     fi
 
-    Examples/Scripts/generic_plotter.py \
+    run Examples/Scripts/generic_plotter.py \
         $outdir/tracksummary_ckf_${suffix}.root \
         tracksummary \
         $outdir/tracksummary_ckf_${suffix}_hist.root \
@@ -218,7 +289,7 @@ function simulation() {
 
     config="CI/physmon/simulation_config.yml"
 
-    Examples/Scripts/generic_plotter.py \
+    run Examples/Scripts/generic_plotter.py \
         $outdir/particles_${suffix}.root \
         particles \
         $outdir/particles_${suffix}_hist.root \
@@ -274,15 +345,15 @@ if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
         "Ambisolver " \
         ambi_ttbar
 
-    Examples/Scripts/generic_plotter.py \
+    run Examples/Scripts/generic_plotter.py \
         $outdir/performance_amvf_ttbar.root \
         vertexing \
         $outdir/performance_amvf_ttbar_hist.root \
         --silent \
         --config CI/physmon/vertexing_ttbar_config.yml
     ec=$(($ec | $?))
 
-    Examples/Scripts/generic_plotter.py \
+    run Examples/Scripts/generic_plotter.py \
         $outdir/tracksummary_ckf_ttbar.root \
         tracksummary \
         $outdir/tracksummary_ckf_ttbar_hist.root \
@@ -307,7 +378,7 @@ if [[ "$mode" == "all" || "$mode" == "fullchains" ]]; then
         "AMVF ttbar" \
         amvf_ttbar
 
-    Examples/Scripts/generic_plotter.py \
+    run Examples/Scripts/generic_plotter.py \
         $outdir/performance_amvf_gridseeder_ttbar.root \
         vertexing \
         $outdir/performance_amvf_gridseeder_ttbar_hist.root \
@@ -353,7 +424,7 @@ if [[ "$mode" == "all" || "$mode" == "gx2f" ]]; then
 fi
 
 if [[ "$mode" == "all" || "$mode" == "vertexing" ]]; then
-    Examples/Scripts/vertex_mu_scan.py \
+    run Examples/Scripts/vertex_mu_scan.py \
         $outdir/performance_vertexing_*mu*.root \
         $outdir/vertexing_mu_scan.pdf
 
@@ -365,7 +436,7 @@ if [[ "$mode" == "all" || "$mode" == "simulation" ]]; then
     simulation geant4
 fi
 
-CI/physmon/summary.py $histcmp_results \
+run CI/physmon/summary.py $histcmp_results \
   --md $outdir/summary.md \
   --html $outdir/summary.html
 ec=$(($ec | $?))